From d009da06d6053030cc92d4b274341584010d4c16 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 29 Apr 2026 13:03:14 +0300 Subject: [PATCH 1/7] support s3:// document sources --- haiku_rag_slim/haiku/rag/client/__init__.py | 10 +- haiku_rag_slim/haiku/rag/client/documents.py | 144 +++++++++- haiku_rag_slim/haiku/rag/config/__init__.py | 2 + haiku_rag_slim/haiku/rag/config/models.py | 10 + haiku_rag_slim/haiku/rag/s3.py | 42 +++ haiku_rag_slim/pyproject.toml | 2 + pyproject.toml | 9 + tests/test_s3_source.py | 261 +++++++++++++++++++ uv.lock | 82 +++++- 9 files changed, 549 insertions(+), 13 deletions(-) create mode 100644 haiku_rag_slim/haiku/rag/s3.py create mode 100644 tests/test_s3_source.py diff --git a/haiku_rag_slim/haiku/rag/client/__init__.py b/haiku_rag_slim/haiku/rag/client/__init__.py index cb95771e..1413757f 100644 --- a/haiku_rag_slim/haiku/rag/client/__init__.py +++ b/haiku_rag_slim/haiku/rag/client/__init__.py @@ -210,10 +210,18 @@ class HaikuRAG: title: str | None = None, metadata: dict | None = None, uri: str | None = None, + storage_options: dict[str, str] | None = None, ) -> Document | list[Document]: from haiku.rag.client.documents import create_document_from_source - return await create_document_from_source(self, source, title, metadata, uri=uri) + return await create_document_from_source( + self, + source, + title, + metadata, + uri=uri, + storage_options=storage_options, + ) async def update_document( self, diff --git a/haiku_rag_slim/haiku/rag/client/documents.py b/haiku_rag_slim/haiku/rag/client/documents.py index e104cce3..a0b0003a 100644 --- a/haiku_rag_slim/haiku/rag/client/documents.py +++ b/haiku_rag_slim/haiku/rag/client/documents.py @@ -197,6 +197,7 @@ async def create_document_from_source( title: str | None = None, metadata: dict | None = None, uri: str | None = None, + storage_options: dict[str, str] | None = None, ) -> Document | list[Document]: """Create or update document(s) from a file path, directory, or URL. @@ -221,6 +222,15 @@ async def create_document_from_source( return await _create_or_update_document_from_url( client, source_str, title=title, metadata=metadata, uri=uri ) + elif parsed_url.scheme == "s3": + return await _create_or_update_document_from_s3( + client, + source_str, + storage_options=storage_options, + title=title, + metadata=metadata, + uri=uri, + ) elif parsed_url.scheme == "file": source_path = Path(parsed_url.path) else: @@ -441,6 +451,138 @@ async def _create_or_update_document_from_url( ) +async def _create_or_update_document_from_s3( + client: "HaikuRAG", + url: str, + *, + storage_options: dict[str, str] | None = None, + title: str | None = None, + metadata: dict | None = None, + uri: str | None = None, +) -> Document: + """Create or update a document from an s3:// URL. + + Two-stage change detection: + - HeadObject ETag matches stored metadata["etag"] → skip GetObject and re-chunk. + - ETag differs but content MD5 matches → refresh etag only, no re-chunk. + - Otherwise download, convert, chunk, embed. + + ``uri`` overrides the s3:// URL as the stored document identifier. + """ + from haiku.rag.client.processing import get_extension_from_content_type_or_url + from haiku.rag.embeddings import embed_chunks + from haiku.rag.s3 import make_s3_session + + metadata = metadata or {} + stored_uri = uri if uri is not None else url + + parsed = urlparse(url) + bucket = parsed.netloc + key = parsed.path.lstrip("/") + if not bucket or not key: + raise ValueError(f"Invalid S3 URI: {url}") + + converter = get_converter(client._config) + supported_extensions = converter.supported_extensions + + session, client_kwargs = make_s3_session(storage_options) + + async with session.client("s3", **client_kwargs) as s3: + head = await s3.head_object(Bucket=bucket, Key=key) + etag = head["ETag"].strip('"') + content_type = (head.get("ContentType") or "").lower() + if not content_type: + content_type = "application/octet-stream" + + existing_doc = await client.get_document_by_uri(stored_uri) + if existing_doc and existing_doc.metadata.get("etag") == etag: + updated = False + if title is not None and title != existing_doc.title: + existing_doc.title = title + updated = True + + merged_metadata = {**(existing_doc.metadata or {}), **metadata} + if merged_metadata != existing_doc.metadata: + existing_doc.metadata = merged_metadata + updated = True + + if updated: + return await client.document_repository.update(existing_doc) + return existing_doc + + file_extension = get_extension_from_content_type_or_url(url, content_type) + if file_extension not in supported_extensions: + raise ValueError( + f"Unsupported content type/extension: {content_type}/{file_extension}" + ) + + get_resp = await s3.get_object(Bucket=bucket, Key=key) + body = await get_resp["Body"].read() + + md5_hash = hashlib.md5(body, usedforsecurity=False).hexdigest() + + with tempfile.NamedTemporaryFile( + mode="wb", suffix=file_extension, delete=False + ) as temp_file: + temp_file.write(body) + temp_file.flush() + temp_path = Path(temp_file.name) + + metadata.update({"contentType": content_type, "md5": md5_hash, "etag": etag}) + + if existing_doc and existing_doc.metadata.get("md5") == md5_hash: + temp_path.unlink(missing_ok=True) + merged_metadata = {**(existing_doc.metadata or {}), **metadata} + updated = False + if merged_metadata != existing_doc.metadata: + existing_doc.metadata = merged_metadata + updated = True + if title is not None and title != existing_doc.title: + existing_doc.title = title + updated = True + if updated: + return await client.document_repository.update(existing_doc) + return existing_doc + + try: + docling_document = await client.convert(temp_path) + chunks = await client.chunk(docling_document) + embedded_chunks = await embed_chunks(chunks, client._config) + finally: + temp_path.unlink(missing_ok=True) + + stored_content = docling_document.export_to_markdown() + + if existing_doc: + existing_doc.content = stored_content + existing_doc.metadata = metadata + existing_doc.set_docling(docling_document) + if title is not None: + existing_doc.title = title + elif existing_doc.title is None: + existing_doc.title = await resolve_title( + client._config, docling_document, stored_content + ) + return await _update_document_with_chunks( + client, existing_doc, embedded_chunks, docling_document + ) + else: + if title is None: + title = await resolve_title( + client._config, docling_document, stored_content + ) + document = Document( + content=stored_content, + uri=stored_uri, + title=title, + metadata=metadata, + ) + document.set_docling(docling_document) + return await _store_document_with_chunks( + client, document, embedded_chunks, docling_document + ) + + async def update_document( client: "HaikuRAG", document_id: str, @@ -525,7 +667,7 @@ def check_source_accessible(uri: str) -> bool: try: if parsed_url.scheme == "file": return Path(parsed_url.path).exists() - elif parsed_url.scheme in ("http", "https"): + elif parsed_url.scheme in ("http", "https", "s3"): return True return False except Exception: diff --git a/haiku_rag_slim/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py index 75e637bc..e5567237 100644 --- a/haiku_rag_slim/haiku/rag/config/__init__.py +++ b/haiku_rag_slim/haiku/rag/config/__init__.py @@ -18,6 +18,7 @@ from haiku.rag.config.models import ( QAConfig, RerankingConfig, ResearchConfig, + S3MonitorEntry, StorageConfig, ) @@ -37,6 +38,7 @@ __all__ = [ "QAConfig", "RerankingConfig", "ResearchConfig", + "S3MonitorEntry", "StorageConfig", "find_config_file", "generate_default_config", diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index 29d36407..a46cedc3 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -51,11 +51,21 @@ class StorageConfig(BaseModel): vacuum_retention_seconds: int = 86400 +class S3MonitorEntry(BaseModel): + uri: str + storage_options: dict[str, str] = Field(default_factory=dict) + poll_interval: int = 300 + ignore_patterns: list[str] = [] + include_patterns: list[str] = [] + delete_orphans: bool = False + + class MonitorConfig(BaseModel): directories: list[Path] = [] ignore_patterns: list[str] = [] include_patterns: list[str] = [] delete_orphans: bool = False + s3: list[S3MonitorEntry] = [] class LanceDBConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/s3.py b/haiku_rag_slim/haiku/rag/s3.py new file mode 100644 index 00000000..9c9d489f --- /dev/null +++ b/haiku_rag_slim/haiku/rag/s3.py @@ -0,0 +1,42 @@ +from typing import Any + + +def make_s3_session( + storage_options: dict[str, str] | None, +) -> tuple[Any, dict[str, Any]]: + """Build an aioboto3 Session and S3 client kwargs from LanceDB-style options. + + Accepts the same dict shape as `LanceDBConfig.storage_options` so users can + copy-paste their LanceDB credentials. Recognized keys: aws_access_key_id, + aws_secret_access_key, aws_session_token, region (or region_name), endpoint + (or endpoint_url), allow_http. Empty/missing keys fall back to the boto3 + default credential chain (environment variables, IAM role, AWS profile). + """ + try: + import aioboto3 # type: ignore[import-not-found] # ty: ignore[unresolved-import] + except ImportError as e: + raise ImportError( + "aioboto3 is required for s3:// sources. " + "Install with: pip install haiku.rag-slim[s3]" + ) from e + + storage_options = storage_options or {} + session_kwargs: dict[str, Any] = {} + client_kwargs: dict[str, Any] = {} + + for key in ("aws_access_key_id", "aws_secret_access_key", "aws_session_token"): + if key in storage_options: + session_kwargs[key] = storage_options[key] + + region = storage_options.get("region") or storage_options.get("region_name") + if region: + session_kwargs["region_name"] = region + + endpoint = storage_options.get("endpoint") or storage_options.get("endpoint_url") + if endpoint: + client_kwargs["endpoint_url"] = endpoint + + if str(storage_options.get("allow_http", "")).lower() == "true": + client_kwargs["use_ssl"] = False + + return aioboto3.Session(**session_kwargs), client_kwargs diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 6705793e..a8d3789d 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -43,6 +43,8 @@ dependencies = [ [project.optional-dependencies] # Document processing docling = ["docling>=2.91.0", "opencv-python-headless>=4.13.0.92"] +# S3 / object-storage monitoring +s3 = ["aioboto3>=13"] # Embedding providers voyageai = ["pydantic-ai-slim[voyageai]"] # Rerankers diff --git a/pyproject.toml b/pyproject.toml index fe94c4c4..2539c311 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ haiku-rag = "haiku.rag.cli:cli" [project.optional-dependencies] tui = ["textual>=1.0.0"] +s3 = ["haiku.rag-slim[s3]==0.45.0"] [build-system] requires = ["hatchling"] @@ -57,6 +58,14 @@ exclude = [ [tool.hatch.build.targets.wheel] only-include = ["/README.md"] +[tool.uv] +conflicts = [ + [ + { package = "haiku.rag-slim", extra = "s3" }, + { package = "pydantic-ai-slim", extra = "bedrock" }, + ], +] + [tool.uv.workspace] members = ["haiku_rag_slim", "evaluations"] diff --git a/tests/test_s3_source.py b/tests/test_s3_source.py new file mode 100644 index 00000000..c5f0ba7b --- /dev/null +++ b/tests/test_s3_source.py @@ -0,0 +1,261 @@ +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from haiku.rag.client import HaikuRAG + + +@pytest.fixture +def fake_aioboto3(monkeypatch): + """Install a fake aioboto3 module in sys.modules. + + Returns the module itself; callers configure `Session` to control behavior. + """ + fake = MagicMock() + monkeypatch.setitem(sys.modules, "aioboto3", fake) + return fake + + +@pytest.fixture +def fake_s3_client(fake_aioboto3): + """Configure fake_aioboto3.Session to return a controllable S3 client. + + Returns a MagicMock representing the S3 client (head_object, get_object). + Tests override its return values per-case. + """ + s3_client = MagicMock() + s3_client.head_object = AsyncMock() + s3_client.get_object = AsyncMock() + + client_ctx = AsyncMock() + client_ctx.__aenter__.return_value = s3_client + client_ctx.__aexit__.return_value = None + + session = MagicMock() + session.client.return_value = client_ctx + + fake_aioboto3.Session.return_value = session + return s3_client + + +def _streaming_body(data: bytes) -> AsyncMock: + body = AsyncMock() + body.read.return_value = data + return body + + +def test_make_s3_session_translates_lancedb_keys(fake_aioboto3): + from haiku.rag.s3 import make_s3_session + + storage_options = { + "endpoint": "http://seaweed:8333", + "region": "us-east-1", + "aws_access_key_id": "AKIA", + "aws_secret_access_key": "secret", + "allow_http": "true", + } + + session, client_kwargs = make_s3_session(storage_options) + + fake_aioboto3.Session.assert_called_once_with( + aws_access_key_id="AKIA", + aws_secret_access_key="secret", + region_name="us-east-1", + ) + assert client_kwargs == { + "endpoint_url": "http://seaweed:8333", + "use_ssl": False, + } + assert session is fake_aioboto3.Session.return_value + + +def test_make_s3_session_accepts_native_aliases(fake_aioboto3): + from haiku.rag.s3 import make_s3_session + + _, client_kwargs = make_s3_session( + {"region_name": "eu-west-1", "endpoint_url": "https://s3"} + ) + + fake_aioboto3.Session.assert_called_once_with(region_name="eu-west-1") + assert client_kwargs == {"endpoint_url": "https://s3"} + + +def test_make_s3_session_no_options_uses_default_chain(fake_aioboto3): + from haiku.rag.s3 import make_s3_session + + session, client_kwargs = make_s3_session(None) + + fake_aioboto3.Session.assert_called_once_with() + assert client_kwargs == {} + + +def test_make_s3_session_allow_http_only_when_truthy(fake_aioboto3): + from haiku.rag.s3 import make_s3_session + + _, ck = make_s3_session({"allow_http": "false"}) + assert "use_ssl" not in ck + + _, ck = make_s3_session({"allow_http": "True"}) + assert ck["use_ssl"] is False + + +def test_make_s3_session_missing_aioboto3_raises_actionable_error(monkeypatch): + monkeypatch.setitem(sys.modules, "aioboto3", None) + + from haiku.rag.s3 import make_s3_session + + with pytest.raises(ImportError, match=r"haiku\.rag-slim\[s3\]"): + make_s3_session({}) + + +@pytest.mark.asyncio +async def test_create_document_from_s3_new(fake_s3_client, temp_db_path): + text = b"S3 hosted content" + fake_s3_client.head_object.return_value = { + "ETag": '"abc123"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + + async with HaikuRAG(temp_db_path, create=True) as client: + doc = await client.create_document_from_source("s3://my-bucket/folder/file.txt") + + assert doc.uri == "s3://my-bucket/folder/file.txt" + assert doc.metadata["etag"] == "abc123" # quotes stripped + assert doc.metadata["md5"] # real content MD5 + assert doc.metadata["md5"] != "abc123" + fake_s3_client.head_object.assert_awaited_once() + fake_s3_client.get_object.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_create_document_from_s3_skips_when_etag_unchanged( + fake_s3_client, temp_db_path +): + text = b"S3 hosted content" + fake_s3_client.head_object.return_value = { + "ETag": '"abc123"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + + async with HaikuRAG(temp_db_path, create=True) as client: + first = await client.create_document_from_source("s3://my-bucket/file.txt") + + # Second call with the same ETag must not GetObject. + fake_s3_client.get_object.reset_mock() + # Re-arm body so a stray call would still produce something readable. + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + second = await client.create_document_from_source("s3://my-bucket/file.txt") + + assert second.id == first.id + assert fake_s3_client.head_object.await_count == 2 + fake_s3_client.get_object.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk( + fake_s3_client, temp_db_path +): + """Multipart re-upload of same content: etag changes, MD5 doesn't. + + Expected: GetObject runs to verify, but no re-chunk; only metadata.etag updates. + """ + text = b"S3 hosted content" + fake_s3_client.head_object.return_value = { + "ETag": '"abc123"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + + async with HaikuRAG(temp_db_path, create=True) as client: + first = await client.create_document_from_source("s3://my-bucket/file.txt") + original_md5 = first.metadata["md5"] + original_updated_at = first.updated_at + + # Same bytes, different ETag (simulating a multipart re-upload). + fake_s3_client.head_object.return_value = { + "ETag": '"def456-2"', # multipart-style ETag + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + + second = await client.create_document_from_source("s3://my-bucket/file.txt") + + assert second.id == first.id + assert second.metadata["md5"] == original_md5 # MD5 unchanged + assert second.metadata["etag"] == "def456-2" # ETag refreshed + assert second.updated_at >= original_updated_at + # GetObject ran once (initial create) plus once more for the etag-changed compare. + assert fake_s3_client.get_object.await_count == 2 + + +@pytest.mark.asyncio +async def test_create_document_from_s3_etag_changed_md5_changed_rechunks( + fake_s3_client, temp_db_path +): + fake_s3_client.head_object.return_value = { + "ETag": '"abc123"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(b"original text")} + + async with HaikuRAG(temp_db_path, create=True) as client: + first = await client.create_document_from_source("s3://my-bucket/file.txt") + + fake_s3_client.head_object.return_value = { + "ETag": '"new999"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = { + "Body": _streaming_body(b"different text now") + } + + second = await client.create_document_from_source("s3://my-bucket/file.txt") + + assert second.id == first.id + assert second.metadata["md5"] != first.metadata["md5"] + assert second.metadata["etag"] == "new999" + assert "different text now" in second.content + + +@pytest.mark.asyncio +async def test_create_document_from_s3_rejects_invalid_uri( + fake_s3_client, temp_db_path +): + async with HaikuRAG(temp_db_path, create=True) as client: + with pytest.raises(ValueError, match="Invalid S3 URI"): + await client.create_document_from_source("s3://only-bucket-no-key") + + +@pytest.mark.asyncio +async def test_create_document_from_s3_passes_storage_options( + fake_aioboto3, fake_s3_client, temp_db_path +): + fake_s3_client.head_object.return_value = { + "ETag": '"abc"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(b"hello")} + + async with HaikuRAG(temp_db_path, create=True) as client: + await client.create_document_from_source( + "s3://bucket/key.txt", + storage_options={ + "endpoint": "http://seaweed:8333", + "region": "us-east-1", + "allow_http": "true", + "aws_access_key_id": "AKIA", + "aws_secret_access_key": "secret", + }, + ) + + fake_aioboto3.Session.assert_called_with( + aws_access_key_id="AKIA", + aws_secret_access_key="secret", + region_name="us-east-1", + ) + fake_aioboto3.Session.return_value.client.assert_called_with( + "s3", endpoint_url="http://seaweed:8333", use_ssl=False + ) diff --git a/uv.lock b/uv.lock index 59c2370a..cf73fe76 100644 --- a/uv.lock +++ b/uv.lock @@ -872,14 +872,14 @@ wheels = [ [[package]] name = "docling" -version = "2.92.0" +version = "2.93.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docling-slim", extra = ["standard"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/87/343d558da23e0626ba027d2e27da25f1c9022964807bfe80778205550a65/docling-2.92.0.tar.gz", hash = "sha256:e8e393ce1a9520c6a61acc67977a6c2d7c2588d98c7446a935b01f9877af9f93", size = 8727, upload-time = "2026-04-29T07:41:25.438Z" } +sdist = { url = "https://files.pythonhosted.org/packages/df/e7/41a7ef58935b914332a7924e0a9d817d50214201aeca7507944404229161/docling-2.93.0.tar.gz", hash = "sha256:c8b4455466d9a6314c27cf84debfdf491532298f65364ec3449acaad517bfea6", size = 8726, upload-time = "2026-05-07T11:55:38.745Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/ad/e3c776dac9639ba7a9fff4c3fe3687258b6eb651d83c4885fad71b3b6aa4/docling-2.92.0-py3-none-any.whl", hash = "sha256:bd85e102e34bb5a90d3be5c2179855fedbdeb9f45318dfba340bf54f1b3268b2", size = 4829, upload-time = "2026-04-29T07:41:23.992Z" }, + { url = "https://files.pythonhosted.org/packages/47/6f/3befa6e4cd42f3a9cf284165d18ca1039ab676ab36da327242aa701e6b63/docling-2.93.0-py3-none-any.whl", hash = "sha256:30a2dc2db733d6a24095e624cc133ce67e9edc46e778f982cf719e137bcee200", size = 4827, upload-time = "2026-05-07T11:55:37.457Z" }, ] [[package]] @@ -969,7 +969,7 @@ wheels = [ [[package]] name = "docling-slim" -version = "2.92.0" +version = "2.93.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -981,9 +981,9 @@ dependencies = [ { name = "requests" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/45/ff4d565ccf69694b2ab3b0fcc7cd403243b8650c735bb1a42a48ba82bcaf/docling_slim-2.92.0.tar.gz", hash = "sha256:f54a2159a46cf00f4738888594c5a81372048d8a0d1a15dd279a7390641a04fa", size = 387431, upload-time = "2026-04-29T07:40:05.913Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/51/2ee874abcd62b990f0a86abec2e3b87b4cc00731b675ed446fefff6199d9/docling_slim-2.93.0.tar.gz", hash = "sha256:2962f4fc5bdf9dd6d67d6f36f09334f7187039985c0fd4b2d4b1d375e4799157", size = 390036, upload-time = "2026-05-07T11:54:14.562Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/91/12/bdf2579d85e43e5944a146c1c4eab708103784a043fb12d793b524c1cd02/docling_slim-2.92.0-py3-none-any.whl", hash = "sha256:e6b7ea5b955c5c47e9bc36f828268b2b78c32f544842036d43d529746783e380", size = 503467, upload-time = "2026-04-29T07:40:03.714Z" }, + { url = "https://files.pythonhosted.org/packages/f3/3f/f2195f79a62fd6cd10c768c04c758c9d6bd0c804a46175d0f3f10a784fca/docling_slim-2.93.0-py3-none-any.whl", hash = "sha256:98e3db67f7976f051f132e6a6e04f73e7fcd4017877eddf9e849e0969c39fbe7", size = 506046, upload-time = "2026-05-07T11:54:10.884Z" }, ] [package.optional-dependencies] @@ -1445,6 +1445,9 @@ dependencies = [ ] [package.optional-dependencies] +s3 = [ + { name = "haiku-rag-slim", extra = ["s3"] }, +] tui = [ { name = "textual" }, ] @@ -1455,6 +1458,7 @@ dev = [ { name = "haiku-rag-evals" }, { name = "mkdocs" }, { name = "mkdocs-material" }, + { name = "obstore" }, { name = "pre-commit" }, { name = "pydantic-ai-slim", extra = ["anthropic", "bedrock", "google", "groq"] }, { name = "pytest" }, @@ -1469,9 +1473,10 @@ dev = [ [package.metadata] requires-dist = [ { name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "tui"], editable = "haiku_rag_slim" }, + { name = "haiku-rag-slim", extras = ["s3"], marker = "extra == 's3'", editable = "haiku_rag_slim" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=1.0.0" }, ] -provides-extras = ["tui"] +provides-extras = ["tui", "s3"] [package.metadata.requires-dev] dev = [ @@ -1479,6 +1484,7 @@ dev = [ { name = "haiku-rag-evals", editable = "evaluations" }, { name = "mkdocs", specifier = ">=1.6.1" }, { name = "mkdocs-material", specifier = ">=9.7.6" }, + { name = "obstore", specifier = ">=0.9,<0.10" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pydantic-ai-slim", extras = ["anthropic"] }, { name = "pydantic-ai-slim", extras = ["bedrock"] }, @@ -1571,6 +1577,9 @@ mistral = [ mxbai = [ { name = "mxbai-rerank" }, ] +s3 = [ + { name = "obstore" }, +] tui = [ { name = "textual" }, { name = "textual-image" }, @@ -1596,6 +1605,7 @@ requires-dist = [ { name = "jsonpatch", specifier = ">=1.33" }, { name = "lancedb", specifier = "==0.30.2" }, { name = "mxbai-rerank", marker = "extra == 'mxbai'", specifier = ">=0.1.6" }, + { name = "obstore", marker = "extra == 's3'", specifier = ">=0.9,<0.10" }, { name = "opencv-python-headless", marker = "extra == 'docling'", specifier = ">=4.13.0.92" }, { name = "pathspec", specifier = ">=1.0.4" }, { name = "pydantic", specifier = ">=2.12.5" }, @@ -1620,7 +1630,7 @@ requires-dist = [ { name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a11" }, { name = "zstandard", marker = "python_full_version < '3.14'", specifier = ">=0.23.0" }, ] -provides-extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] +provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] [[package]] name = "haiku-skills" @@ -2878,6 +2888,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, ] +[[package]] +name = "obstore" +version = "0.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/7a/3a37b0bf0da898478029fcc511a0d2a7252689b1f29e46db7ae74a219c74/obstore-0.9.4.tar.gz", hash = "sha256:e2b93f1372c59da2c7e74122fc6dc4b713d84fd4528b5b500ef7f548425496b5", size = 124167, upload-time = "2026-04-22T19:51:05.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/25/4449a0066796b91e282d7604a66387bba399b14752598c748ea9557c4c32/obstore-0.9.4-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0d17cd04e7f22960050a85f8daa6e274d693e8fb3b97b81eeaa293c6f9e62eb4", size = 4090743, upload-time = "2026-04-22T19:49:26.461Z" }, + { url = "https://files.pythonhosted.org/packages/93/91/639fe5f5644593b9f4bea66f8f29c7bfd4de3b3381fb74b4f7df678f505f/obstore-0.9.4-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4beec92710fb8826fb357baf28fb79a91ee07dcdfe73777207aa762164aaa35", size = 3876313, upload-time = "2026-04-22T19:49:28.107Z" }, + { url = "https://files.pythonhosted.org/packages/ce/71/d6675f845ebe1e3927f2dce6a2a4d5a393359274762ee00c5e6855d5f468/obstore-0.9.4-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d523c8c365ab60afb8d232614a00a92bea439a9f5c55b92486c23a47af038a1e", size = 4029950, upload-time = "2026-04-22T19:49:30.279Z" }, + { url = "https://files.pythonhosted.org/packages/0e/3a/5915a173f5c6a95f9ec186a7e29b0ce6a23bd9b04c2b0b29a351dbe2baf6/obstore-0.9.4-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee0483619088337ee365cb344fceee337e2670ec4de2a1da92ac7f6b2220f18e", size = 4129455, upload-time = "2026-04-22T19:49:31.934Z" }, + { url = "https://files.pythonhosted.org/packages/b5/a9/63c31d2d436c06c4d39ed5cb154fe54202b303854532ec09537c4ce0755b/obstore-0.9.4-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:83da348bf0a7dd84e5839c0cd54d79dcd08e0729c394e566f73a605b93b9e998", size = 4416727, upload-time = "2026-04-22T19:49:34.016Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fa/23c5c6db02be0e13abcbe01c1ca94c5f7876e8c58e74cb9ac2b57b068866/obstore-0.9.4-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f282a17200bcc37b8d7a1d02a146ed41812eb6e76fd0a4c9a154f02da1b8031f", size = 4311520, upload-time = "2026-04-22T19:49:35.905Z" }, + { url = "https://files.pythonhosted.org/packages/86/f0/49f6b02dab9c05e3fd79d6129e4d9e7e9874d6e5e05369ca3b3b80a48aaa/obstore-0.9.4-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d29dcfceaa0a205ded2263d29a2a3aa206819d549e0325c1f2106f79e2658584", size = 4220536, upload-time = "2026-04-22T19:49:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/50/ab/d0bfd6d68422e7d8f2204d91736c7e62767e0576ad749da442a71e7773b2/obstore-0.9.4-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:caecb912723ab8e9da8da26def249d66da4318959df2bafc0a55af64f3255902", size = 4105099, upload-time = "2026-04-22T19:49:40.384Z" }, + { url = "https://files.pythonhosted.org/packages/66/3b/f595d0ee354f9daa69438991f8818602f34bc59498c8468456a02d45fb27/obstore-0.9.4-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c1c06fec8837595a2829b5f7536d0d01e940ce10b07ad2a8594fec1cfd0b7d5", size = 4294206, upload-time = "2026-04-22T19:49:42.016Z" }, + { url = "https://files.pythonhosted.org/packages/60/54/3c5af2d59258aaa9e5bef05320658ea6e9b1f3897a3a977bf7f54a0b6ec1/obstore-0.9.4-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c132795a789ec5ade31bf4d5b55ed321fb41d9749e9145520bf19063e1da5f7b", size = 4265047, upload-time = "2026-04-22T19:49:43.983Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/a8ba1feb81b9833b253147839da40405ec6bfa51feb3abfe909c800208a5/obstore-0.9.4-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:c6e342360a5d0ae71486bc5f8311778aa144ec1a905c23593f8ef57b5bceae24", size = 4255361, upload-time = "2026-04-22T19:49:45.864Z" }, + { url = "https://files.pythonhosted.org/packages/15/f7/3ccc0288111e057f8ba3d99bee14f95d9e9bb00acaf6e9700e0eb4cd82c3/obstore-0.9.4-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aeb6f7e7e862550f5020a10692ef6f02d5ba4912dba08942eb59bb7d73f93fe0", size = 4439378, upload-time = "2026-04-22T19:49:47.581Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/3ac8b5772743c60064f3c7e02d27f346dbb58feaa99a49ee09798d1cfb00/obstore-0.9.4-cp311-abi3-win_amd64.whl", hash = "sha256:a58ef942292841f99d69ac11d19d05544c835447c8c09dacbfb7409c6374c4a1", size = 4191594, upload-time = "2026-04-22T19:49:49.308Z" }, + { url = "https://files.pythonhosted.org/packages/9d/81/8f6b6509f8df603261cdb5ddb521c49891457775669c6ad857812bf4a7c1/obstore-0.9.4-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:fff17f59390ed307afcd1fb18c56076c1f911dd9f5c2636b7d7133c4d07f8c3f", size = 4071300, upload-time = "2026-04-22T19:49:51.386Z" }, + { url = "https://files.pythonhosted.org/packages/ab/fe/0c74ddf3ab9b24ef356925bfb613bc7846f869220361a784b63f754d8563/obstore-0.9.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4527c4c7889f1bd1f1952017d74774870e14e199d6b50b9e72f291f9498d898c", size = 3870593, upload-time = "2026-04-22T19:49:53.481Z" }, + { url = "https://files.pythonhosted.org/packages/73/fa/260ec94f9a7b4f4c8afbdd016710bed0736615488d3ac0c5620f9179bfcd/obstore-0.9.4-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a57c2016e3e569de35050f95c679ffe61813c4e3cb6d6028c4c3f57231021eb4", size = 4023990, upload-time = "2026-04-22T19:49:55.644Z" }, + { url = "https://files.pythonhosted.org/packages/8d/84/5b8e2b9607fb93c96a39a4cfa6d37bd3049ebf7265d0e9f8afa938bf32fe/obstore-0.9.4-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd5327cee4fb3578b51beb1c92915cc3a05ffe794be40f50bd68d27e97d78c5c", size = 4119971, upload-time = "2026-04-22T19:49:57.745Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2b/e6c093acb7e62009d5b1678d82839903287c29d4a6e1dfbea8fbf41313d5/obstore-0.9.4-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12b1e6105eafe02d8973dbeb2d274eeac2271c67f1126ffa16f18ddea8dd5443", size = 4407147, upload-time = "2026-04-22T19:49:59.928Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3d/5c93a9adee8f045b89d5f21b337f53667499db770bda129f805723ab14e4/obstore-0.9.4-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b0d378248fda4e36652808d73eaaeb7e67154427e6c724248c9b0b9b03e70a6", size = 4312215, upload-time = "2026-04-22T19:50:01.534Z" }, + { url = "https://files.pythonhosted.org/packages/9a/de/507f60b4e6a8c0cad9f93a51a7b28132c9db49e20aadbcd542fa2abc57c4/obstore-0.9.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78fb77c346abd2bcdfa071d7166be2bdc38c28573ae5a230746df6158a5593e", size = 4216936, upload-time = "2026-04-22T19:50:03.244Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ff/612bd5f8258349bfe9e8c349d184b5ea3333038d4cce0d003eefafb2160c/obstore-0.9.4-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:f4e5a6dfe6877fb599868d560d6fcf4d7416cadbdf3bd947254b53830c2f11c0", size = 4105091, upload-time = "2026-04-22T19:50:05.038Z" }, + { url = "https://files.pythonhosted.org/packages/e3/73/b083b99e7bc0b529bee7b4437cafd7cc7d9f59c10995a48b6c26447fdf7f/obstore-0.9.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f8114a2b84268c991232d89b105d9239299b6afb56e4941a61c09f3a89033022", size = 4292570, upload-time = "2026-04-22T19:50:06.823Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cd/3c4555f98db9a49432bc0afa68bfc33dd47bdfa3699c915b4b0e887577e3/obstore-0.9.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:9d7b959f5f74532a142fb449c0bef5814dfe3fa5c43c31ac4284a15221a75aaf", size = 4261946, upload-time = "2026-04-22T19:50:08.789Z" }, + { url = "https://files.pythonhosted.org/packages/96/f8/bdc66df3d0dfdcfb3931a585a7fb3b74336619baf6d3540b1425b424232b/obstore-0.9.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a8e9101fc2659dd938e7ae06512075bc0a8f02ab28d2ee438d6fca8b4f3bdfba", size = 4245595, upload-time = "2026-04-22T19:50:10.765Z" }, + { url = "https://files.pythonhosted.org/packages/d7/22/1aa58ea676293e5b888391c8433ff6ab8f66622aae30427287f9daac6d46/obstore-0.9.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:538384255545b5c575497fcab26389c8f01707402b6ddcdd73b769b66311635d", size = 4436599, upload-time = "2026-04-22T19:50:12.585Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9e/b52f2c97be27952d488cf1980af0c635f9947003e5744e3e1dc6252f0040/obstore-0.9.4-cp313-cp313t-win_amd64.whl", hash = "sha256:eef1c772657bb1293adad0d671ca1ff1e1dcae84ec4dfbf1a34e47c2a1f134ac", size = 4180463, upload-time = "2026-04-22T19:50:14.288Z" }, + { url = "https://files.pythonhosted.org/packages/19/76/c53583f95c6811057abd3116756dca46785318d564a0e99c207cbb2d8938/obstore-0.9.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e009e7437770c85beae4c32cb79f662f0a9922676ef127e943d107a5c082d38d", size = 4071302, upload-time = "2026-04-22T19:50:15.967Z" }, + { url = "https://files.pythonhosted.org/packages/2f/23/ac3b9c05a09b3d5f178ed6f288c5d6913df8f7386059590194e0fee65d15/obstore-0.9.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ac5f3ad314bd4592fe484b79c229518be7bb5f6218bed33c20742026d5caf860", size = 3870813, upload-time = "2026-04-22T19:50:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ec/c3458e0f24d2d1a4f185f541905b07e51c91b3fec589b1600c77d511e585/obstore-0.9.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:db79d5ebc4177360565ffcec4abd49930cf052cdbeb94e3a3ece2e2d08f087d0", size = 4024237, upload-time = "2026-04-22T19:50:19.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/6cf468a200e491fdc6c04075e2fbbac1707bbecd243f0f56ae1e75d052ed/obstore-0.9.4-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05b565d89c3115fb74385852dd628e12f6645a1bba97523dceae016b538a3f33", size = 4119635, upload-time = "2026-04-22T19:50:21.605Z" }, + { url = "https://files.pythonhosted.org/packages/81/fb/b44d002767fa5af95ab4ca8e16c3a9057fc11f13de03f498b99adf0c4e50/obstore-0.9.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dfc4fc98403d8fbb316eb04257c8122b6f1dda37e80869491fdacf60a815e4c", size = 4406906, upload-time = "2026-04-22T19:50:23.654Z" }, + { url = "https://files.pythonhosted.org/packages/4b/18/9a75ad5082cd581c4a55f0e62bedf4b030a8b53824976fc1f030eff225b3/obstore-0.9.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c69af620fd3d06a8cfb62d25faf1adb6ccc97cc572f47ee04dddcde5a5e5444e", size = 4311826, upload-time = "2026-04-22T19:50:25.458Z" }, + { url = "https://files.pythonhosted.org/packages/8d/03/b0f945b31f40364a7ed4dbc5677abc66331fcf478732f4d643e17e56bb13/obstore-0.9.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa78c0230e0b9d49b25ed18980e1751331ddfe05782d6ce97579a9ccda8229ea", size = 4217086, upload-time = "2026-04-22T19:50:27.266Z" }, + { url = "https://files.pythonhosted.org/packages/1a/26/bdd85264c806802086f21d73cc7c95a5baca5feeeac4bce8acb97142163f/obstore-0.9.4-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:c828719f0bb310a9cf0e0f08cb62a0b8cc550138617cb03ac897900aec9d3d47", size = 4105560, upload-time = "2026-04-22T19:50:29.324Z" }, + { url = "https://files.pythonhosted.org/packages/6a/36/4a4a6a398e5f145edd1886388ebe5e6f6bbaf74950a5dea1a6ceae63e6b5/obstore-0.9.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:49a0455519f284b6bc2e0694298114926aff1d1f3d5d344e9163e03b446826cc", size = 4292582, upload-time = "2026-04-22T19:50:31.028Z" }, + { url = "https://files.pythonhosted.org/packages/39/4c/9caa197cd2eba726e9a5285db34027049b9527a23e1a7e08479678ad6a4a/obstore-0.9.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf437309fc0fe852591ae50405300490229f876ea06574651fd753ca3fd23f25", size = 4261613, upload-time = "2026-04-22T19:50:32.868Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/a3fbe6fb3ee1c57fd4943ddbb21848eea3925b77e0789614c857d86b795e/obstore-0.9.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d83dbd20b6a5d42e35794ef64046de39040854829ec4f1eb2f6dfb54df48cc3d", size = 4245638, upload-time = "2026-04-22T19:50:35.009Z" }, + { url = "https://files.pythonhosted.org/packages/56/a7/d18e168f318327d63512dfa7cf3b5e89ed9bfba6d6a8917ad7d4700b8657/obstore-0.9.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5a0c337f37f30a2d66555d69bf3abd840457a279c57ede93bd02e014721ed364", size = 4437226, upload-time = "2026-04-22T19:50:36.635Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ce/66aadd155db1e273c6ec2236c0fb904666d10c2e3b791b40624c272e586c/obstore-0.9.4-cp314-cp314t-win_amd64.whl", hash = "sha256:24e37a1c713c95a964e119f8ef879415a495432162e74e80ed29d645aeeca114", size = 4180746, upload-time = "2026-04-22T19:50:38.396Z" }, +] + [[package]] name = "omegaconf" version = "2.3.0" @@ -3835,16 +3895,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.14.0" +version = "2.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/42/98/c8345dccdc31de4228c039a98f6467a941e39558da41c1744fbe29fa5666/pydantic_settings-2.14.0.tar.gz", hash = "sha256:24285fd4b0e0c06507dd9fdfd331ee23794305352aaec8fc4eb92d4047aeb67d", size = 235709, upload-time = "2026-04-20T13:37:40.293Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/01/dd/bebff3040138f00ae8a102d426b27349b9a49acc310fcae7f92112d867e3/pydantic_settings-2.14.0-py3-none-any.whl", hash = "sha256:fc8d5d692eb7092e43c8647c1c35a3ecd00e040fcf02ed86f4cb5458ca62182e", size = 60940, upload-time = "2026-04-20T13:37:38.586Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, ] [[package]] From 37099988a681bc9063b7ecb99477a342bd435956 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 29 Apr 2026 13:21:26 +0300 Subject: [PATCH 2/7] poll S3 prefixes with S3Watcher and wire into serve --- haiku_rag_slim/haiku/rag/app.py | 16 +- haiku_rag_slim/haiku/rag/monitor.py | 107 +++++++- tests/test_s3_monitor.py | 392 ++++++++++++++++++++++++++++ 3 files changed, 513 insertions(+), 2 deletions(-) create mode 100644 tests/test_s3_monitor.py diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index 65281277..a6ecd55f 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -21,7 +21,7 @@ from rich.syntax import Syntax from haiku.rag.client import HaikuRAG, RebuildMode from haiku.rag.config import AppConfig, Config from haiku.rag.mcp import create_mcp_server -from haiku.rag.monitor import FileWatcher +from haiku.rag.monitor import FileWatcher, S3Watcher from haiku.rag.store.models.document import Document if TYPE_CHECKING: @@ -821,6 +821,20 @@ class HaikuRAGApp: # pragma: no cover monitor_task = asyncio.create_task(monitor.observe()) tasks.append(monitor_task) + if self.config.monitor.s3: + from haiku.rag.converters import get_converter + + supported_extensions = get_converter( + self.config + ).supported_extensions + for entry in self.config.monitor.s3: + s3_watcher = S3Watcher( + client=client, + entry=entry, + supported_extensions=supported_extensions, + ) + tasks.append(asyncio.create_task(s3_watcher.observe())) + # Start MCP server if enabled if enable_mcp: server = create_mcp_server( diff --git a/haiku_rag_slim/haiku/rag/monitor.py b/haiku_rag_slim/haiku/rag/monitor.py index a8ca8bdb..7cabdfd4 100644 --- a/haiku_rag_slim/haiku/rag/monitor.py +++ b/haiku_rag_slim/haiku/rag/monitor.py @@ -2,13 +2,15 @@ import asyncio import logging from pathlib import Path from typing import TYPE_CHECKING +from urllib.parse import urlparse import pathspec from watchfiles import Change, DefaultFilter, awatch from haiku.rag.client import HaikuRAG -from haiku.rag.config import AppConfig, Config +from haiku.rag.config import AppConfig, Config, S3MonitorEntry from haiku.rag.store.models.document import Document +from haiku.rag.utils import escape_sql_string if TYPE_CHECKING: pass @@ -215,3 +217,106 @@ class FileWatcher: logger.info(f"Deleted document {existing_doc.id} for {file}") except Exception as e: logger.error(f"Failed to delete document for {file}: {e}") + + +class S3Watcher: + """Polls an S3 prefix and keeps documents in sync with the index. + + Uses ListObjectsV2 ETags as the cheap-skip key. When a key's listing + ETag differs from the stored `metadata["etag"]`, delegates to + `client.create_document_from_source` which performs the full + HeadObject + GetObject + MD5 compare two-stage detection. + """ + + def __init__( + self, + client: HaikuRAG, + entry: S3MonitorEntry, + supported_extensions: list[str], + ) -> None: + from haiku.rag.s3 import make_s3_session + + parsed = urlparse(entry.uri) + if not parsed.netloc: + raise ValueError(f"Invalid S3 monitor URI: {entry.uri}") + + self.client = client + self.entry = entry + self.bucket = parsed.netloc + self.prefix = parsed.path.lstrip("/") + self.uri_prefix = f"s3://{self.bucket}/{self.prefix}" + self._make_s3_session = make_s3_session + self.filter = FileFilter( + ignore_patterns=entry.ignore_patterns or None, + include_patterns=entry.include_patterns or None, + supported_extensions=supported_extensions, + ) + + async def observe(self) -> None: + logger.info( + f"Watching S3 {self.entry.uri} (poll_interval={self.entry.poll_interval}s)" + ) + await self.refresh() + while True: + await asyncio.sleep(self.entry.poll_interval) + try: + await self.refresh() + except Exception as e: + logger.error(f"S3 watcher refresh failed for {self.entry.uri}: {e}") + + async def refresh(self) -> None: + uris_seen: dict[str, str] = {} + session, client_kwargs = self._make_s3_session(self.entry.storage_options) + + async with session.client("s3", **client_kwargs) as s3: + paginator = s3.get_paginator("list_objects_v2") + async for page in paginator.paginate( + Bucket=self.bucket, Prefix=self.prefix + ): + for obj in page.get("Contents", []): + key = obj["Key"] + if not self.filter.include_file(key): + continue + uri = f"s3://{self.bucket}/{key}" + uris_seen[uri] = obj["ETag"].strip('"') + + existing_etags = await self._existing_etags_under_prefix() + + for uri, etag in uris_seen.items(): + if existing_etags.get(uri) == etag: + continue + await self._upsert_object(uri) + + if self.entry.delete_orphans: + await self._delete_orphans(set(uris_seen.keys()), existing_etags) + + async def _existing_etags_under_prefix(self) -> dict[str, str]: + safe_prefix = escape_sql_string(self.uri_prefix) + docs = await self.client.list_documents(filter=f"uri LIKE '{safe_prefix}%'") + return { + doc.uri: (doc.metadata or {}).get("etag", "") for doc in docs if doc.uri + } + + async def _upsert_object(self, uri: str) -> Document | None: + try: + result = await self.client.create_document_from_source( + uri, storage_options=self.entry.storage_options + ) + doc = result if isinstance(result, Document) else result[0] + logger.info(f"Upserted document {doc.id} from {uri}") + return doc + except Exception as e: + logger.error(f"Failed to upsert document from {uri}: {e}") + return None + + async def _delete_orphans( + self, uris_seen: set[str], existing_etags: dict[str, str] + ) -> None: + for uri in existing_etags.keys() - uris_seen: + try: + doc = await self.client.get_document_by_uri(uri) + if doc and doc.id: + await self.client.delete_document(doc.id) + logger.info(f"Deleted orphaned document {doc.id} for {uri}") + except Exception as e: + logger.error(f"Failed to delete orphan {uri}: {e}") diff --git a/tests/test_s3_monitor.py b/tests/test_s3_monitor.py new file mode 100644 index 00000000..0f946b1f --- /dev/null +++ b/tests/test_s3_monitor.py @@ -0,0 +1,392 @@ +import asyncio +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from haiku.rag.client import HaikuRAG +from haiku.rag.config import AppConfig, MonitorConfig, S3MonitorEntry +from haiku.rag.store.models.document import Document + + +@pytest.fixture +def fake_aioboto3(monkeypatch): + fake = MagicMock() + monkeypatch.setitem(sys.modules, "aioboto3", fake) + return fake + + +@pytest.fixture +def s3_paginator(): + """Return (paginator_mock, set_pages) — `set_pages([page, ...])` rewires the iterator.""" + pages: list[dict] = [] + + async def _paginate(**_kwargs): + for page in pages: + yield page + + paginator = MagicMock() + paginator.paginate.side_effect = lambda **kw: _paginate(**kw) + + def set_pages(new_pages): + pages.clear() + pages.extend(new_pages) + + return paginator, set_pages + + +@pytest.fixture +def fake_s3_client(fake_aioboto3, s3_paginator): + paginator, set_pages = s3_paginator + s3_client = MagicMock() + s3_client.get_paginator.return_value = paginator + + client_ctx = AsyncMock() + client_ctx.__aenter__.return_value = s3_client + client_ctx.__aexit__.return_value = None + + session = MagicMock() + session.client.return_value = client_ctx + fake_aioboto3.Session.return_value = session + + return s3_client, set_pages + + +def _entry(**kwargs): + base = { + "uri": "s3://my-bucket/incoming/", + "poll_interval": 60, + "delete_orphans": False, + "ignore_patterns": [], + "include_patterns": [], + "storage_options": {}, + } + base.update(kwargs) + return S3MonitorEntry(**base) + + +def _doc(uri: str, etag: str, doc_id: str | None = None) -> Document: + return Document( + id=doc_id or uri, + content="...", + uri=uri, + metadata={"etag": etag, "md5": "deadbeef"}, + ) + + +@pytest.mark.asyncio +async def test_s3_watcher_refresh_upserts_new_objects(fake_s3_client): + s3, set_pages = fake_s3_client + set_pages( + [ + { + "Contents": [ + {"Key": "incoming/a.txt", "ETag": '"abc"'}, + {"Key": "incoming/b.txt", "ETag": '"def"'}, + ] + } + ] + ) + + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [] + rag.create_document_from_source.return_value = Document( + id="x", content="...", uri="s3://my-bucket/incoming/a.txt" + ) + + watcher = S3Watcher( + client=rag, entry=_entry(), supported_extensions=[".txt", ".md", ".pdf"] + ) + await watcher.refresh() + + assert rag.create_document_from_source.await_count == 2 + called_uris = {c.args[0] for c in rag.create_document_from_source.await_args_list} + assert called_uris == { + "s3://my-bucket/incoming/a.txt", + "s3://my-bucket/incoming/b.txt", + } + + +@pytest.mark.asyncio +async def test_s3_watcher_skips_unchanged_etag(fake_s3_client): + s3, set_pages = fake_s3_client + set_pages([{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}]) + + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [_doc("s3://my-bucket/incoming/a.txt", "abc")] + + watcher = S3Watcher(client=rag, entry=_entry(), supported_extensions=[".txt"]) + await watcher.refresh() + + rag.create_document_from_source.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_s3_watcher_upserts_when_etag_differs(fake_s3_client): + s3, set_pages = fake_s3_client + set_pages([{"Contents": [{"Key": "incoming/a.txt", "ETag": '"new"'}]}]) + + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [_doc("s3://my-bucket/incoming/a.txt", "old")] + rag.create_document_from_source.return_value = Document( + id="x", content="...", uri="s3://my-bucket/incoming/a.txt" + ) + + watcher = S3Watcher(client=rag, entry=_entry(), supported_extensions=[".txt"]) + await watcher.refresh() + + rag.create_document_from_source.assert_awaited_once_with( + "s3://my-bucket/incoming/a.txt", storage_options={} + ) + + +@pytest.mark.asyncio +async def test_s3_watcher_strips_etag_quotes(fake_s3_client): + s3, set_pages = fake_s3_client + set_pages([{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}]) + + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [ + _doc("s3://my-bucket/incoming/a.txt", "abc") # already stripped in storage + ] + + watcher = S3Watcher(client=rag, entry=_entry(), supported_extensions=[".txt"]) + await watcher.refresh() + + rag.create_document_from_source.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_s3_watcher_deletes_orphans_when_enabled(fake_s3_client): + s3, set_pages = fake_s3_client + set_pages([{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}]) + + from haiku.rag.monitor import S3Watcher + + a_doc = _doc("s3://my-bucket/incoming/a.txt", "abc", doc_id="a-id") + orphan = _doc("s3://my-bucket/incoming/old.txt", "stale", doc_id="orphan-id") + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [a_doc, orphan] + rag.get_document_by_uri.return_value = orphan + + watcher = S3Watcher( + client=rag, + entry=_entry(delete_orphans=True), + supported_extensions=[".txt"], + ) + await watcher.refresh() + + rag.delete_document.assert_awaited_once_with("orphan-id") + + +@pytest.mark.asyncio +async def test_s3_watcher_does_not_delete_orphans_when_disabled(fake_s3_client): + s3, set_pages = fake_s3_client + set_pages([{"Contents": []}]) + + from haiku.rag.monitor import S3Watcher + + orphan = _doc("s3://my-bucket/incoming/old.txt", "stale", doc_id="orphan-id") + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [orphan] + + watcher = S3Watcher( + client=rag, + entry=_entry(delete_orphans=False), + supported_extensions=[".txt"], + ) + await watcher.refresh() + + rag.delete_document.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_s3_watcher_orphan_scope_is_per_entry(fake_s3_client): + """A doc under a different bucket prefix must not be touched.""" + s3, set_pages = fake_s3_client + set_pages([{"Contents": []}]) + + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [] # filter scopes to my-bucket + + watcher = S3Watcher( + client=rag, + entry=_entry(delete_orphans=True), + supported_extensions=[".txt"], + ) + await watcher.refresh() + + rag.list_documents.assert_awaited_once() + filter_kwarg = rag.list_documents.await_args.kwargs["filter"] + assert filter_kwarg == "uri LIKE 's3://my-bucket/incoming/%'" + + +@pytest.mark.asyncio +async def test_s3_watcher_applies_include_and_ignore_patterns(fake_s3_client): + s3, set_pages = fake_s3_client + set_pages( + [ + { + "Contents": [ + {"Key": "incoming/keep.md", "ETag": '"1"'}, + {"Key": "incoming/draft.md", "ETag": '"2"'}, + {"Key": "incoming/skip.txt", "ETag": '"3"'}, + ] + } + ] + ) + + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [] + rag.create_document_from_source.return_value = Document( + id="x", content="...", uri="s3://my-bucket/incoming/keep.md" + ) + + watcher = S3Watcher( + client=rag, + entry=_entry(include_patterns=["*.md"], ignore_patterns=["draft*"]), + supported_extensions=[".md", ".txt"], + ) + await watcher.refresh() + + assert rag.create_document_from_source.await_count == 1 + assert ( + rag.create_document_from_source.await_args.args[0] + == "s3://my-bucket/incoming/keep.md" + ) + + +@pytest.mark.asyncio +async def test_s3_watcher_observe_survives_transient_list_failure(fake_s3_client): + """First refresh succeeds; second refresh raises; loop survives and recovers.""" + s3, set_pages = fake_s3_client + + pages_initial = [{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}] + pages_after = [{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}] + + set_pages(pages_initial) + paginate_calls = {"n": 0} + + async def paginate_side_effect(**_kwargs): + paginate_calls["n"] += 1 + if paginate_calls["n"] == 2: + raise RuntimeError("transient list failure") + for page in pages_after if paginate_calls["n"] > 1 else pages_initial: + yield page + + s3.get_paginator.return_value.paginate.side_effect = paginate_side_effect + + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [] + rag.create_document_from_source.return_value = Document( + id="x", content="...", uri="s3://my-bucket/incoming/a.txt" + ) + + watcher = S3Watcher( + client=rag, + entry=_entry(poll_interval=0), + supported_extensions=[".txt"], + ) + task = asyncio.create_task(watcher.observe()) + + # Let three iterations run: initial refresh, transient failure, recovery. + for _ in range(20): + await asyncio.sleep(0) + if paginate_calls["n"] >= 3: + break + + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + assert paginate_calls["n"] >= 3 # loop kept going past the failure + + +@pytest.mark.asyncio +async def test_s3_watcher_invalid_uri_rejected(): + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + with pytest.raises(ValueError, match="Invalid S3 monitor URI"): + S3Watcher( + client=rag, + entry=S3MonitorEntry(uri="s3://"), + supported_extensions=[".txt"], + ) + + +@pytest.mark.asyncio +async def test_serve_starts_one_s3_task_per_entry(monkeypatch, fake_s3_client): + """`serve` wires one S3Watcher task per MonitorConfig.s3 entry.""" + from haiku.rag import app as app_module + + captured_tasks: list = [] + original_create_task = asyncio.create_task + + def tracking_create_task(coro, *args, **kwargs): + captured_tasks.append(coro) + return original_create_task(coro, *args, **kwargs) + + monkeypatch.setattr(app_module.asyncio, "create_task", tracking_create_task) + + config = AppConfig( + monitor=MonitorConfig( + s3=[ + S3MonitorEntry(uri="s3://bucket-a/x/"), + S3MonitorEntry(uri="s3://bucket-b/y/"), + ] + ) + ) + + fw_observe_calls = {"n": 0} + + async def fake_fw_observe(self): + fw_observe_calls["n"] += 1 + + monkeypatch.setattr(app_module.FileWatcher, "observe", fake_fw_observe) + + s3_observe_calls = {"n": 0} + + async def fake_s3_observe(self): + s3_observe_calls["n"] += 1 + + monkeypatch.setattr(app_module.S3Watcher, "observe", fake_s3_observe) + + # Provide a dummy supported_extensions to skip docling import. + class _Conv: + supported_extensions = [".txt"] + + monkeypatch.setattr("haiku.rag.converters.get_converter", lambda cfg: _Conv()) + + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + db_path = Path(tmp) / "db.lancedb" + app = app_module.HaikuRAGApp(db_path=db_path, config=config) + async with HaikuRAG(db_path, config=config, create=True): + pass # create the database + + await app.serve(enable_monitor=True, enable_mcp=False) + + # Both S3 entries should have triggered observe(), plus the FileWatcher. + assert fw_observe_calls["n"] == 1 + assert s3_observe_calls["n"] == 2 From f87d894b017fbaa4c0856f7054867a0a8dbb2fe3 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 29 Apr 2026 14:19:55 +0300 Subject: [PATCH 3/7] add SeaweedFS integration tests for S3Watcher --- scripts/run-integration-tests.sh | 24 +++++ tests/test_s3_integration.py | 167 ++++++++++++++++++++++++++++++- 2 files changed, 190 insertions(+), 1 deletion(-) create mode 100755 scripts/run-integration-tests.sh diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh new file mode 100755 index 00000000..86c168c6 --- /dev/null +++ b/scripts/run-integration-tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Run integration tests against a local SeaweedFS instance. +# +# Brings up SeaweedFS via docker compose, runs every test marked `integration`, +# and tears down on exit. Extra args are forwarded to pytest: +# +# ./scripts/run-integration-tests.sh +# ./scripts/run-integration-tests.sh tests/test_s3_integration.py::test_s3_watcher_initial_sweep +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMPOSE_FILE="$REPO_ROOT/tests/docker/docker-compose.s3.yml" + +cleanup() { + echo "==> Tearing down SeaweedFS" + docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "==> Bringing up SeaweedFS" +docker compose -f "$COMPOSE_FILE" up -d --wait + +echo "==> Running integration tests" +uv run pytest -m integration "$@" diff --git a/tests/test_s3_integration.py b/tests/test_s3_integration.py index c13cf029..9dfe90be 100644 --- a/tests/test_s3_integration.py +++ b/tests/test_s3_integration.py @@ -3,6 +3,7 @@ # Stop after: # docker compose -f tests/docker/docker-compose.s3.yml down -v +import importlib.util import socket from uuid import uuid4 @@ -10,9 +11,11 @@ import pytest from haiku.rag.app import HaikuRAGApp from haiku.rag.client import HaikuRAG -from haiku.rag.config.models import AppConfig, LanceDBConfig +from haiku.rag.config.models import AppConfig, LanceDBConfig, S3MonitorEntry from haiku.rag.store.engine import Store +HAS_AIOBOTO3 = importlib.util.find_spec("aioboto3") is not None + S3_ENDPOINT = "http://localhost:8333" S3_BUCKET = "test-bucket" S3_STORAGE_OPTIONS = { @@ -149,3 +152,165 @@ async def test_app_info_empty_db(tmp_path, capsys): out = capsys.readouterr().out assert "Database is empty" in out + + +# ----------------------- S3 watcher integration tests ----------------------- # +# These exercise S3Watcher against the live SeaweedFS instance. Documents are +# uploaded as raw S3 objects under a unique per-test prefix; the watcher's +# refresh() is invoked directly so tests stay deterministic. LanceDB stays +# local — these tests verify the watcher path, not LanceDB-on-S3. + + +_aioboto3_required = pytest.mark.skipif( + not HAS_AIOBOTO3, + reason="aioboto3 not installed (uv sync --extra s3)", +) + + +async def _put_object(prefix: str, key: str, body: bytes) -> None: + import aioboto3 # type: ignore[import-not-found] # ty: ignore[unresolved-import] + + session = aioboto3.Session( + aws_access_key_id=S3_STORAGE_OPTIONS["aws_access_key_id"], + aws_secret_access_key=S3_STORAGE_OPTIONS["aws_secret_access_key"], + region_name=S3_STORAGE_OPTIONS["region"], + ) + async with session.client( + "s3", endpoint_url=S3_STORAGE_OPTIONS["endpoint"], use_ssl=False + ) as s3: + try: + await s3.create_bucket(Bucket=S3_BUCKET) + except Exception: + pass # bucket already exists + await s3.put_object(Bucket=S3_BUCKET, Key=f"{prefix}/{key}", Body=body) + + +async def _delete_object(prefix: str, key: str) -> None: + import aioboto3 # type: ignore[import-not-found] # ty: ignore[unresolved-import] + + session = aioboto3.Session( + aws_access_key_id=S3_STORAGE_OPTIONS["aws_access_key_id"], + aws_secret_access_key=S3_STORAGE_OPTIONS["aws_secret_access_key"], + region_name=S3_STORAGE_OPTIONS["region"], + ) + async with session.client( + "s3", endpoint_url=S3_STORAGE_OPTIONS["endpoint"], use_ssl=False + ) as s3: + await s3.delete_object(Bucket=S3_BUCKET, Key=f"{prefix}/{key}") + + +def _watcher_entry(prefix: str, **overrides) -> S3MonitorEntry: + return S3MonitorEntry( + uri=overrides.pop("uri", f"s3://{S3_BUCKET}/{prefix}/"), + storage_options=overrides.pop("storage_options", S3_STORAGE_OPTIONS), + include_patterns=overrides.pop("include_patterns", ["*.txt"]), + delete_orphans=overrides.pop("delete_orphans", False), + poll_interval=overrides.pop("poll_interval", 60), + **overrides, + ) + + +@_aioboto3_required +@pytest.mark.asyncio +async def test_s3_watcher_initial_sweep(tmp_path): + from haiku.rag.monitor import S3Watcher + + prefix = f"watcher-init-{uuid4().hex[:8]}" + await _put_object(prefix, "alpha.txt", b"alpha content") + await _put_object(prefix, "beta.txt", b"beta content") + + async with HaikuRAG(tmp_path / "db.lancedb", create=True) as rag: + watcher = S3Watcher( + client=rag, + entry=_watcher_entry(prefix), + supported_extensions=[".txt", ".md", ".pdf"], + ) + await watcher.refresh() + + docs = await rag.list_documents() + uris = sorted(d.uri or "" for d in docs) + + assert uris == [ + f"s3://{S3_BUCKET}/{prefix}/alpha.txt", + f"s3://{S3_BUCKET}/{prefix}/beta.txt", + ] + + +@_aioboto3_required +@pytest.mark.asyncio +async def test_s3_watcher_detects_new_object(tmp_path): + from haiku.rag.monitor import S3Watcher + + prefix = f"watcher-new-{uuid4().hex[:8]}" + await _put_object(prefix, "first.txt", b"first content") + + async with HaikuRAG(tmp_path / "db.lancedb", create=True) as rag: + watcher = S3Watcher( + client=rag, + entry=_watcher_entry(prefix), + supported_extensions=[".txt"], + ) + await watcher.refresh() + assert await rag.count_documents() == 1 + + await _put_object(prefix, "second.txt", b"second content") + await watcher.refresh() + assert await rag.count_documents() == 2 + + +@_aioboto3_required +@pytest.mark.asyncio +async def test_s3_watcher_detects_modified_object(tmp_path): + from haiku.rag.monitor import S3Watcher + + prefix = f"watcher-mod-{uuid4().hex[:8]}" + uri = f"s3://{S3_BUCKET}/{prefix}/file.txt" + + await _put_object(prefix, "file.txt", b"original content") + + async with HaikuRAG(tmp_path / "db.lancedb", create=True) as rag: + watcher = S3Watcher( + client=rag, + entry=_watcher_entry(prefix), + supported_extensions=[".txt"], + ) + await watcher.refresh() + + first = await rag.get_document_by_uri(uri) + assert first is not None + first_md5 = first.metadata["md5"] + + await _put_object(prefix, "file.txt", b"new content body") + await watcher.refresh() + + second = await rag.get_document_by_uri(uri) + assert second is not None + assert second.id == first.id + assert second.metadata["md5"] != first_md5 + assert "new content body" in second.content + + +@_aioboto3_required +@pytest.mark.asyncio +async def test_s3_watcher_orphan_deletion(tmp_path): + from haiku.rag.monitor import S3Watcher + + prefix = f"watcher-orphan-{uuid4().hex[:8]}" + await _put_object(prefix, "kept.txt", b"keep me") + await _put_object(prefix, "doomed.txt", b"will be deleted") + + async with HaikuRAG(tmp_path / "db.lancedb", create=True) as rag: + watcher = S3Watcher( + client=rag, + entry=_watcher_entry(prefix, delete_orphans=True), + supported_extensions=[".txt"], + ) + await watcher.refresh() + assert await rag.count_documents() == 2 + + await _delete_object(prefix, "doomed.txt") + await watcher.refresh() + + docs = await rag.list_documents() + assert len(docs) == 1 + assert docs[0].uri == f"s3://{S3_BUCKET}/{prefix}/kept.txt" From ef82e6eb083fb05dbb0be46cc83cf0f54b26204a Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 29 Apr 2026 14:38:53 +0300 Subject: [PATCH 4/7] swap aioboto3 for obstore in the S3 client path --- haiku_rag_slim/haiku/rag/client/documents.py | 60 ++--- haiku_rag_slim/haiku/rag/monitor.py | 28 +-- haiku_rag_slim/haiku/rag/s3.py | 53 ++--- haiku_rag_slim/pyproject.toml | 2 +- pyproject.toml | 9 +- tests/test_s3_integration.py | 52 ++-- tests/test_s3_monitor.py | 192 +++++++-------- tests/test_s3_source.py | 236 ++++++------------- 8 files changed, 247 insertions(+), 385 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/client/documents.py b/haiku_rag_slim/haiku/rag/client/documents.py index a0b0003a..100bc51a 100644 --- a/haiku_rag_slim/haiku/rag/client/documents.py +++ b/haiku_rag_slim/haiku/rag/client/documents.py @@ -463,15 +463,17 @@ async def _create_or_update_document_from_s3( """Create or update a document from an s3:// URL. Two-stage change detection: - - HeadObject ETag matches stored metadata["etag"] → skip GetObject and re-chunk. + - HEAD ETag matches stored metadata["etag"] → skip GET and re-chunk. - ETag differs but content MD5 matches → refresh etag only, no re-chunk. - Otherwise download, convert, chunk, embed. ``uri`` overrides the s3:// URL as the stored document identifier. """ + import obstore # type: ignore[import-not-found] + from haiku.rag.client.processing import get_extension_from_content_type_or_url from haiku.rag.embeddings import embed_chunks - from haiku.rag.s3 import make_s3_session + from haiku.rag.s3 import make_s3_store metadata = metadata or {} stored_uri = uri if uri is not None else url @@ -485,39 +487,39 @@ async def _create_or_update_document_from_s3( converter = get_converter(client._config) supported_extensions = converter.supported_extensions - session, client_kwargs = make_s3_session(storage_options) + store = make_s3_store(bucket, storage_options) - async with session.client("s3", **client_kwargs) as s3: - head = await s3.head_object(Bucket=bucket, Key=key) - etag = head["ETag"].strip('"') - content_type = (head.get("ContentType") or "").lower() - if not content_type: - content_type = "application/octet-stream" + head = await obstore.head_async(store, key) + etag = (head.get("e_tag") or "").strip('"') - existing_doc = await client.get_document_by_uri(stored_uri) - if existing_doc and existing_doc.metadata.get("etag") == etag: - updated = False - if title is not None and title != existing_doc.title: - existing_doc.title = title - updated = True + existing_doc = await client.get_document_by_uri(stored_uri) + if existing_doc and existing_doc.metadata.get("etag") == etag: + updated = False + if title is not None and title != existing_doc.title: + existing_doc.title = title + updated = True - merged_metadata = {**(existing_doc.metadata or {}), **metadata} - if merged_metadata != existing_doc.metadata: - existing_doc.metadata = merged_metadata - updated = True + merged_metadata = {**(existing_doc.metadata or {}), **metadata} + if merged_metadata != existing_doc.metadata: + existing_doc.metadata = merged_metadata + updated = True - if updated: - return await client.document_repository.update(existing_doc) - return existing_doc + if updated: + return await client.document_repository.update(existing_doc) + return existing_doc - file_extension = get_extension_from_content_type_or_url(url, content_type) - if file_extension not in supported_extensions: - raise ValueError( - f"Unsupported content type/extension: {content_type}/{file_extension}" - ) + content_type, _ = mimetypes.guess_type(key) + if not content_type: + content_type = "application/octet-stream" - get_resp = await s3.get_object(Bucket=bucket, Key=key) - body = await get_resp["Body"].read() + file_extension = get_extension_from_content_type_or_url(url, content_type) + if file_extension not in supported_extensions: + raise ValueError( + f"Unsupported content type/extension: {content_type}/{file_extension}" + ) + + get_resp = await obstore.get_async(store, key) + body = await get_resp.bytes_async() md5_hash = hashlib.md5(body, usedforsecurity=False).hexdigest() diff --git a/haiku_rag_slim/haiku/rag/monitor.py b/haiku_rag_slim/haiku/rag/monitor.py index 7cabdfd4..40e18a5a 100644 --- a/haiku_rag_slim/haiku/rag/monitor.py +++ b/haiku_rag_slim/haiku/rag/monitor.py @@ -234,7 +234,7 @@ class S3Watcher: entry: S3MonitorEntry, supported_extensions: list[str], ) -> None: - from haiku.rag.s3 import make_s3_session + from haiku.rag.s3 import make_s3_store parsed = urlparse(entry.uri) if not parsed.netloc: @@ -245,7 +245,7 @@ class S3Watcher: self.bucket = parsed.netloc self.prefix = parsed.path.lstrip("/") self.uri_prefix = f"s3://{self.bucket}/{self.prefix}" - self._make_s3_session = make_s3_session + self._make_s3_store = make_s3_store self.filter = FileFilter( ignore_patterns=entry.ignore_patterns or None, include_patterns=entry.include_patterns or None, @@ -265,20 +265,18 @@ class S3Watcher: logger.error(f"S3 watcher refresh failed for {self.entry.uri}: {e}") async def refresh(self) -> None: - uris_seen: dict[str, str] = {} - session, client_kwargs = self._make_s3_session(self.entry.storage_options) + import obstore # type: ignore[import-not-found] - async with session.client("s3", **client_kwargs) as s3: - paginator = s3.get_paginator("list_objects_v2") - async for page in paginator.paginate( - Bucket=self.bucket, Prefix=self.prefix - ): - for obj in page.get("Contents", []): - key = obj["Key"] - if not self.filter.include_file(key): - continue - uri = f"s3://{self.bucket}/{key}" - uris_seen[uri] = obj["ETag"].strip('"') + uris_seen: dict[str, str] = {} + store = self._make_s3_store(self.bucket, self.entry.storage_options) + + async for batch in obstore.list(store, prefix=self.prefix or None): + for obj in batch: + key = obj["path"] + if not self.filter.include_file(key): + continue + uri = f"s3://{self.bucket}/{key}" + uris_seen[uri] = (obj.get("e_tag") or "").strip('"') existing_etags = await self._existing_etags_under_prefix() diff --git a/haiku_rag_slim/haiku/rag/s3.py b/haiku_rag_slim/haiku/rag/s3.py index 9c9d489f..6629a7e6 100644 --- a/haiku_rag_slim/haiku/rag/s3.py +++ b/haiku_rag_slim/haiku/rag/s3.py @@ -1,42 +1,39 @@ from typing import Any -def make_s3_session( - storage_options: dict[str, str] | None, -) -> tuple[Any, dict[str, Any]]: - """Build an aioboto3 Session and S3 client kwargs from LanceDB-style options. +def make_s3_store(bucket: str, storage_options: dict[str, str] | None) -> Any: + """Build an obstore `S3Store` from LanceDB-style storage_options. - Accepts the same dict shape as `LanceDBConfig.storage_options` so users can - copy-paste their LanceDB credentials. Recognized keys: aws_access_key_id, - aws_secret_access_key, aws_session_token, region (or region_name), endpoint - (or endpoint_url), allow_http. Empty/missing keys fall back to the boto3 - default credential chain (environment variables, IAM role, AWS profile). + Accepts the same dict shape as `LanceDBConfig.storage_options` (the same + Rust `object_store` crate is used by both LanceDB and obstore, so the keys + line up). Recognized keys include aws_access_key_id, aws_secret_access_key, + aws_session_token, region (or aws_region), endpoint (or aws_endpoint), + allow_http. Empty/missing keys fall back to the AWS default credential + chain (environment variables, IAM role, AWS profile). """ try: - import aioboto3 # type: ignore[import-not-found] # ty: ignore[unresolved-import] + from obstore.store import ( + S3Store, # type: ignore[import-not-found] + ) except ImportError as e: raise ImportError( - "aioboto3 is required for s3:// sources. " + "obstore is required for s3:// sources. " "Install with: pip install haiku.rag-slim[s3]" ) from e - storage_options = storage_options or {} - session_kwargs: dict[str, Any] = {} - client_kwargs: dict[str, Any] = {} + options = dict(storage_options or {}) + allow_http = str(options.pop("allow_http", "")).lower() == "true" - for key in ("aws_access_key_id", "aws_secret_access_key", "aws_session_token"): - if key in storage_options: - session_kwargs[key] = storage_options[key] + # Custom endpoints (SeaweedFS, MinIO, etc.) need path-style requests; AWS + # accepts virtual-hosted-style by default. + has_custom_endpoint = bool(options.get("endpoint") or options.get("aws_endpoint")) - region = storage_options.get("region") or storage_options.get("region_name") - if region: - session_kwargs["region_name"] = region + kwargs: dict[str, Any] = {} + if options: + kwargs["config"] = options + if allow_http: + kwargs["client_options"] = {"allow_http": True} + if has_custom_endpoint: + kwargs["virtual_hosted_style_request"] = False - endpoint = storage_options.get("endpoint") or storage_options.get("endpoint_url") - if endpoint: - client_kwargs["endpoint_url"] = endpoint - - if str(storage_options.get("allow_http", "")).lower() == "true": - client_kwargs["use_ssl"] = False - - return aioboto3.Session(**session_kwargs), client_kwargs + return S3Store(bucket, **kwargs) diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index a8d3789d..83b83af9 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ # Document processing docling = ["docling>=2.91.0", "opencv-python-headless>=4.13.0.92"] # S3 / object-storage monitoring -s3 = ["aioboto3>=13"] +s3 = ["obstore>=0.9,<0.10"] # Embedding providers voyageai = ["pydantic-ai-slim[voyageai]"] # Rerankers diff --git a/pyproject.toml b/pyproject.toml index 2539c311..26755586 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,14 +58,6 @@ exclude = [ [tool.hatch.build.targets.wheel] only-include = ["/README.md"] -[tool.uv] -conflicts = [ - [ - { package = "haiku.rag-slim", extra = "s3" }, - { package = "pydantic-ai-slim", extra = "bedrock" }, - ], -] - [tool.uv.workspace] members = ["haiku_rag_slim", "evaluations"] @@ -77,6 +69,7 @@ members = ["haiku_rag_slim", "evaluations"] dev = [ "haiku.rag-evals", "datasets>=4.8.4", + "obstore>=0.9,<0.10", "mkdocs>=1.6.1", "mkdocs-material>=9.7.6", "pre-commit>=4.5.1", diff --git a/tests/test_s3_integration.py b/tests/test_s3_integration.py index 9dfe90be..156fe872 100644 --- a/tests/test_s3_integration.py +++ b/tests/test_s3_integration.py @@ -14,7 +14,7 @@ from haiku.rag.client import HaikuRAG from haiku.rag.config.models import AppConfig, LanceDBConfig, S3MonitorEntry from haiku.rag.store.engine import Store -HAS_AIOBOTO3 = importlib.util.find_spec("aioboto3") is not None +HAS_OBSTORE = importlib.util.find_spec("obstore") is not None S3_ENDPOINT = "http://localhost:8333" S3_BUCKET = "test-bucket" @@ -161,42 +161,28 @@ async def test_app_info_empty_db(tmp_path, capsys): # local — these tests verify the watcher path, not LanceDB-on-S3. -_aioboto3_required = pytest.mark.skipif( - not HAS_AIOBOTO3, - reason="aioboto3 not installed (uv sync --extra s3)", +_obstore_required = pytest.mark.skipif( + not HAS_OBSTORE, + reason="obstore not installed (uv sync --extra s3)", ) -async def _put_object(prefix: str, key: str, body: bytes) -> None: - import aioboto3 # type: ignore[import-not-found] # ty: ignore[unresolved-import] +def _watcher_store(): + from haiku.rag.s3 import make_s3_store - session = aioboto3.Session( - aws_access_key_id=S3_STORAGE_OPTIONS["aws_access_key_id"], - aws_secret_access_key=S3_STORAGE_OPTIONS["aws_secret_access_key"], - region_name=S3_STORAGE_OPTIONS["region"], - ) - async with session.client( - "s3", endpoint_url=S3_STORAGE_OPTIONS["endpoint"], use_ssl=False - ) as s3: - try: - await s3.create_bucket(Bucket=S3_BUCKET) - except Exception: - pass # bucket already exists - await s3.put_object(Bucket=S3_BUCKET, Key=f"{prefix}/{key}", Body=body) + return make_s3_store(S3_BUCKET, S3_STORAGE_OPTIONS) + + +async def _put_object(prefix: str, key: str, body: bytes) -> None: + import obstore + + await obstore.put_async(_watcher_store(), f"{prefix}/{key}", body) async def _delete_object(prefix: str, key: str) -> None: - import aioboto3 # type: ignore[import-not-found] # ty: ignore[unresolved-import] + import obstore - session = aioboto3.Session( - aws_access_key_id=S3_STORAGE_OPTIONS["aws_access_key_id"], - aws_secret_access_key=S3_STORAGE_OPTIONS["aws_secret_access_key"], - region_name=S3_STORAGE_OPTIONS["region"], - ) - async with session.client( - "s3", endpoint_url=S3_STORAGE_OPTIONS["endpoint"], use_ssl=False - ) as s3: - await s3.delete_object(Bucket=S3_BUCKET, Key=f"{prefix}/{key}") + await obstore.delete_async(_watcher_store(), f"{prefix}/{key}") def _watcher_entry(prefix: str, **overrides) -> S3MonitorEntry: @@ -210,7 +196,7 @@ def _watcher_entry(prefix: str, **overrides) -> S3MonitorEntry: ) -@_aioboto3_required +@_obstore_required @pytest.mark.asyncio async def test_s3_watcher_initial_sweep(tmp_path): from haiku.rag.monitor import S3Watcher @@ -236,7 +222,7 @@ async def test_s3_watcher_initial_sweep(tmp_path): ] -@_aioboto3_required +@_obstore_required @pytest.mark.asyncio async def test_s3_watcher_detects_new_object(tmp_path): from haiku.rag.monitor import S3Watcher @@ -258,7 +244,7 @@ async def test_s3_watcher_detects_new_object(tmp_path): assert await rag.count_documents() == 2 -@_aioboto3_required +@_obstore_required @pytest.mark.asyncio async def test_s3_watcher_detects_modified_object(tmp_path): from haiku.rag.monitor import S3Watcher @@ -290,7 +276,7 @@ async def test_s3_watcher_detects_modified_object(tmp_path): assert "new content body" in second.content -@_aioboto3_required +@_obstore_required @pytest.mark.asyncio async def test_s3_watcher_orphan_deletion(tmp_path): from haiku.rag.monitor import S3Watcher diff --git a/tests/test_s3_monitor.py b/tests/test_s3_monitor.py index 0f946b1f..b125c592 100644 --- a/tests/test_s3_monitor.py +++ b/tests/test_s3_monitor.py @@ -1,5 +1,4 @@ import asyncio -import sys from unittest.mock import AsyncMock, MagicMock import pytest @@ -10,59 +9,53 @@ from haiku.rag.store.models.document import Document @pytest.fixture -def fake_aioboto3(monkeypatch): - fake = MagicMock() - monkeypatch.setitem(sys.modules, "aioboto3", fake) - return fake +def s3_listing(monkeypatch): + """Patch `obstore.list_obs` with an async-iterator returning controllable batches. + + Returns `(set_batches, list_mock)`. `set_batches([[meta, ...], ...])` + seeds the next call's pages. + """ + import obstore + + batches: list[list[MagicMock]] = [] + + def list_obs(_store, *_, **__): + async def _iter(): + for batch in batches: + yield batch + + return _iter() + + list_mock = MagicMock(side_effect=list_obs) + monkeypatch.setattr(obstore, "list", list_mock) + + def set_batches(new_batches): + batches.clear() + batches.extend(new_batches) + + return set_batches, list_mock -@pytest.fixture -def s3_paginator(): - """Return (paginator_mock, set_pages) — `set_pages([page, ...])` rewires the iterator.""" - pages: list[dict] = [] - - async def _paginate(**_kwargs): - for page in pages: - yield page - - paginator = MagicMock() - paginator.paginate.side_effect = lambda **kw: _paginate(**kw) - - def set_pages(new_pages): - pages.clear() - pages.extend(new_pages) - - return paginator, set_pages - - -@pytest.fixture -def fake_s3_client(fake_aioboto3, s3_paginator): - paginator, set_pages = s3_paginator - s3_client = MagicMock() - s3_client.get_paginator.return_value = paginator - - client_ctx = AsyncMock() - client_ctx.__aenter__.return_value = s3_client - client_ctx.__aexit__.return_value = None - - session = MagicMock() - session.client.return_value = client_ctx - fake_aioboto3.Session.return_value = session - - return s3_client, set_pages - - -def _entry(**kwargs): - base = { - "uri": "s3://my-bucket/incoming/", - "poll_interval": 60, - "delete_orphans": False, - "ignore_patterns": [], - "include_patterns": [], - "storage_options": {}, +def _meta(path: str, etag: str) -> dict: + # Real obstore ObjectMeta is a TypedDict; raw S3 ETags include quotes. + return { + "path": path, + "e_tag": f'"{etag}"', + "size": 0, + "last_modified": None, } - base.update(kwargs) - return S3MonitorEntry(**base) + + +def _entry(**kwargs) -> S3MonitorEntry: + return S3MonitorEntry( + uri=kwargs.pop("uri", "s3://my-bucket/incoming/"), + poll_interval=kwargs.pop("poll_interval", 60), + delete_orphans=kwargs.pop("delete_orphans", False), + ignore_patterns=kwargs.pop("ignore_patterns", []), + include_patterns=kwargs.pop("include_patterns", []), + storage_options=kwargs.pop("storage_options", {}), + **kwargs, + ) def _doc(uri: str, etag: str, doc_id: str | None = None) -> Document: @@ -75,18 +68,9 @@ def _doc(uri: str, etag: str, doc_id: str | None = None) -> Document: @pytest.mark.asyncio -async def test_s3_watcher_refresh_upserts_new_objects(fake_s3_client): - s3, set_pages = fake_s3_client - set_pages( - [ - { - "Contents": [ - {"Key": "incoming/a.txt", "ETag": '"abc"'}, - {"Key": "incoming/b.txt", "ETag": '"def"'}, - ] - } - ] - ) +async def test_s3_watcher_refresh_upserts_new_objects(s3_listing): + set_batches, _ = s3_listing + set_batches([[_meta("incoming/a.txt", "abc"), _meta("incoming/b.txt", "def")]]) from haiku.rag.monitor import S3Watcher @@ -110,9 +94,9 @@ async def test_s3_watcher_refresh_upserts_new_objects(fake_s3_client): @pytest.mark.asyncio -async def test_s3_watcher_skips_unchanged_etag(fake_s3_client): - s3, set_pages = fake_s3_client - set_pages([{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}]) +async def test_s3_watcher_skips_unchanged_etag(s3_listing): + set_batches, _ = s3_listing + set_batches([[_meta("incoming/a.txt", "abc")]]) from haiku.rag.monitor import S3Watcher @@ -126,9 +110,9 @@ async def test_s3_watcher_skips_unchanged_etag(fake_s3_client): @pytest.mark.asyncio -async def test_s3_watcher_upserts_when_etag_differs(fake_s3_client): - s3, set_pages = fake_s3_client - set_pages([{"Contents": [{"Key": "incoming/a.txt", "ETag": '"new"'}]}]) +async def test_s3_watcher_upserts_when_etag_differs(s3_listing): + set_batches, _ = s3_listing + set_batches([[_meta("incoming/a.txt", "new")]]) from haiku.rag.monitor import S3Watcher @@ -147,9 +131,9 @@ async def test_s3_watcher_upserts_when_etag_differs(fake_s3_client): @pytest.mark.asyncio -async def test_s3_watcher_strips_etag_quotes(fake_s3_client): - s3, set_pages = fake_s3_client - set_pages([{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}]) +async def test_s3_watcher_strips_etag_quotes(s3_listing): + set_batches, _ = s3_listing + set_batches([[_meta("incoming/a.txt", "abc")]]) from haiku.rag.monitor import S3Watcher @@ -165,9 +149,9 @@ async def test_s3_watcher_strips_etag_quotes(fake_s3_client): @pytest.mark.asyncio -async def test_s3_watcher_deletes_orphans_when_enabled(fake_s3_client): - s3, set_pages = fake_s3_client - set_pages([{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}]) +async def test_s3_watcher_deletes_orphans_when_enabled(s3_listing): + set_batches, _ = s3_listing + set_batches([[_meta("incoming/a.txt", "abc")]]) from haiku.rag.monitor import S3Watcher @@ -189,9 +173,9 @@ async def test_s3_watcher_deletes_orphans_when_enabled(fake_s3_client): @pytest.mark.asyncio -async def test_s3_watcher_does_not_delete_orphans_when_disabled(fake_s3_client): - s3, set_pages = fake_s3_client - set_pages([{"Contents": []}]) +async def test_s3_watcher_does_not_delete_orphans_when_disabled(s3_listing): + set_batches, _ = s3_listing + set_batches([[]]) from haiku.rag.monitor import S3Watcher @@ -211,10 +195,10 @@ async def test_s3_watcher_does_not_delete_orphans_when_disabled(fake_s3_client): @pytest.mark.asyncio -async def test_s3_watcher_orphan_scope_is_per_entry(fake_s3_client): +async def test_s3_watcher_orphan_scope_is_per_entry(s3_listing): """A doc under a different bucket prefix must not be touched.""" - s3, set_pages = fake_s3_client - set_pages([{"Contents": []}]) + set_batches, _ = s3_listing + set_batches([[]]) from haiku.rag.monitor import S3Watcher @@ -234,17 +218,15 @@ async def test_s3_watcher_orphan_scope_is_per_entry(fake_s3_client): @pytest.mark.asyncio -async def test_s3_watcher_applies_include_and_ignore_patterns(fake_s3_client): - s3, set_pages = fake_s3_client - set_pages( +async def test_s3_watcher_applies_include_and_ignore_patterns(s3_listing): + set_batches, _ = s3_listing + set_batches( [ - { - "Contents": [ - {"Key": "incoming/keep.md", "ETag": '"1"'}, - {"Key": "incoming/draft.md", "ETag": '"2"'}, - {"Key": "incoming/skip.txt", "ETag": '"3"'}, - ] - } + [ + _meta("incoming/keep.md", "1"), + _meta("incoming/draft.md", "2"), + _meta("incoming/skip.txt", "3"), + ] ] ) @@ -271,24 +253,27 @@ async def test_s3_watcher_applies_include_and_ignore_patterns(fake_s3_client): @pytest.mark.asyncio -async def test_s3_watcher_observe_survives_transient_list_failure(fake_s3_client): +async def test_s3_watcher_observe_survives_transient_list_failure(s3_listing): """First refresh succeeds; second refresh raises; loop survives and recovers.""" - s3, set_pages = fake_s3_client + set_batches, list_mock = s3_listing - pages_initial = [{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}] - pages_after = [{"Contents": [{"Key": "incoming/a.txt", "ETag": '"abc"'}]}] + pages_initial = [[_meta("incoming/a.txt", "abc")]] + pages_after = [[_meta("incoming/a.txt", "abc")]] - set_pages(pages_initial) paginate_calls = {"n": 0} - async def paginate_side_effect(**_kwargs): + def list_obs_side_effect(_store, *_, **__): paginate_calls["n"] += 1 if paginate_calls["n"] == 2: raise RuntimeError("transient list failure") - for page in pages_after if paginate_calls["n"] > 1 else pages_initial: - yield page - s3.get_paginator.return_value.paginate.side_effect = paginate_side_effect + async def _iter(): + for batch in pages_after if paginate_calls["n"] > 1 else pages_initial: + yield batch + + return _iter() + + list_mock.side_effect = list_obs_side_effect from haiku.rag.monitor import S3Watcher @@ -305,7 +290,6 @@ async def test_s3_watcher_observe_survives_transient_list_failure(fake_s3_client ) task = asyncio.create_task(watcher.observe()) - # Let three iterations run: initial refresh, transient failure, recovery. for _ in range(20): await asyncio.sleep(0) if paginate_calls["n"] >= 3: @@ -334,15 +318,13 @@ async def test_s3_watcher_invalid_uri_rejected(): @pytest.mark.asyncio -async def test_serve_starts_one_s3_task_per_entry(monkeypatch, fake_s3_client): +async def test_serve_starts_one_s3_task_per_entry(monkeypatch, s3_listing): """`serve` wires one S3Watcher task per MonitorConfig.s3 entry.""" from haiku.rag import app as app_module - captured_tasks: list = [] original_create_task = asyncio.create_task def tracking_create_task(coro, *args, **kwargs): - captured_tasks.append(coro) return original_create_task(coro, *args, **kwargs) monkeypatch.setattr(app_module.asyncio, "create_task", tracking_create_task) @@ -370,7 +352,6 @@ async def test_serve_starts_one_s3_task_per_entry(monkeypatch, fake_s3_client): monkeypatch.setattr(app_module.S3Watcher, "observe", fake_s3_observe) - # Provide a dummy supported_extensions to skip docling import. class _Conv: supported_extensions = [".txt"] @@ -387,6 +368,5 @@ async def test_serve_starts_one_s3_task_per_entry(monkeypatch, fake_s3_client): await app.serve(enable_monitor=True, enable_mcp=False) - # Both S3 entries should have triggered observe(), plus the FileWatcher. assert fw_observe_calls["n"] == 1 assert s3_observe_calls["n"] == 2 diff --git a/tests/test_s3_source.py b/tests/test_s3_source.py index c5f0ba7b..061accac 100644 --- a/tests/test_s3_source.py +++ b/tests/test_s3_source.py @@ -7,116 +7,70 @@ from haiku.rag.client import HaikuRAG @pytest.fixture -def fake_aioboto3(monkeypatch): - """Install a fake aioboto3 module in sys.modules. +def fake_obstore_io(monkeypatch): + """Patch `obstore.head_async` and `obstore.get_async` with controllable mocks. - Returns the module itself; callers configure `Session` to control behavior. + Returns a tuple `(head_async, get_async)`. Tests assign `.return_value` to + each to seed responses. Real `obstore.store.S3Store` is left intact — + constructing the store still exercises the storage_options path. """ - fake = MagicMock() - monkeypatch.setitem(sys.modules, "aioboto3", fake) - return fake + import obstore + + head_async = AsyncMock() + get_async = AsyncMock() + monkeypatch.setattr(obstore, "head_async", head_async) + monkeypatch.setattr(obstore, "get_async", get_async) + return head_async, get_async -@pytest.fixture -def fake_s3_client(fake_aioboto3): - """Configure fake_aioboto3.Session to return a controllable S3 client. - - Returns a MagicMock representing the S3 client (head_object, get_object). - Tests override its return values per-case. - """ - s3_client = MagicMock() - s3_client.head_object = AsyncMock() - s3_client.get_object = AsyncMock() - - client_ctx = AsyncMock() - client_ctx.__aenter__.return_value = s3_client - client_ctx.__aexit__.return_value = None - - session = MagicMock() - session.client.return_value = client_ctx - - fake_aioboto3.Session.return_value = session - return s3_client +def _meta(etag: str) -> dict: + return {"e_tag": etag, "path": "ignored", "size": 0, "last_modified": None} -def _streaming_body(data: bytes) -> AsyncMock: - body = AsyncMock() - body.read.return_value = data - return body +def _get_result(data: bytes) -> MagicMock: + result = MagicMock() + result.bytes_async = AsyncMock(return_value=data) + return result -def test_make_s3_session_translates_lancedb_keys(fake_aioboto3): - from haiku.rag.s3 import make_s3_session +def test_make_s3_store_accepts_lancedb_keys(): + from haiku.rag.s3 import make_s3_store - storage_options = { - "endpoint": "http://seaweed:8333", - "region": "us-east-1", - "aws_access_key_id": "AKIA", - "aws_secret_access_key": "secret", - "allow_http": "true", - } - - session, client_kwargs = make_s3_session(storage_options) - - fake_aioboto3.Session.assert_called_once_with( - aws_access_key_id="AKIA", - aws_secret_access_key="secret", - region_name="us-east-1", + store = make_s3_store( + "my-bucket", + { + "endpoint": "http://seaweed:8333", + "region": "us-east-1", + "aws_access_key_id": "AKIA", + "aws_secret_access_key": "secret", + "allow_http": "true", + }, ) - assert client_kwargs == { - "endpoint_url": "http://seaweed:8333", - "use_ssl": False, - } - assert session is fake_aioboto3.Session.return_value + assert store is not None # construction must not raise -def test_make_s3_session_accepts_native_aliases(fake_aioboto3): - from haiku.rag.s3 import make_s3_session +def test_make_s3_store_no_options_uses_default_chain(): + from haiku.rag.s3 import make_s3_store - _, client_kwargs = make_s3_session( - {"region_name": "eu-west-1", "endpoint_url": "https://s3"} - ) - - fake_aioboto3.Session.assert_called_once_with(region_name="eu-west-1") - assert client_kwargs == {"endpoint_url": "https://s3"} + store = make_s3_store("my-bucket", None) + assert store is not None -def test_make_s3_session_no_options_uses_default_chain(fake_aioboto3): - from haiku.rag.s3 import make_s3_session +def test_make_s3_store_missing_obstore_raises_actionable_error(monkeypatch): + monkeypatch.setitem(sys.modules, "obstore.store", None) - session, client_kwargs = make_s3_session(None) - - fake_aioboto3.Session.assert_called_once_with() - assert client_kwargs == {} - - -def test_make_s3_session_allow_http_only_when_truthy(fake_aioboto3): - from haiku.rag.s3 import make_s3_session - - _, ck = make_s3_session({"allow_http": "false"}) - assert "use_ssl" not in ck - - _, ck = make_s3_session({"allow_http": "True"}) - assert ck["use_ssl"] is False - - -def test_make_s3_session_missing_aioboto3_raises_actionable_error(monkeypatch): - monkeypatch.setitem(sys.modules, "aioboto3", None) - - from haiku.rag.s3 import make_s3_session + from haiku.rag.s3 import make_s3_store with pytest.raises(ImportError, match=r"haiku\.rag-slim\[s3\]"): - make_s3_session({}) + make_s3_store("my-bucket", {}) @pytest.mark.asyncio -async def test_create_document_from_s3_new(fake_s3_client, temp_db_path): +async def test_create_document_from_s3_new(fake_obstore_io, temp_db_path): + head_async, get_async = fake_obstore_io text = b"S3 hosted content" - fake_s3_client.head_object.return_value = { - "ETag": '"abc123"', - "ContentType": "text/plain", - } - fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + head_async.return_value = _meta('"abc123"') + get_async.return_value = _get_result(text) async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document_from_source("s3://my-bucket/folder/file.txt") @@ -125,92 +79,76 @@ async def test_create_document_from_s3_new(fake_s3_client, temp_db_path): assert doc.metadata["etag"] == "abc123" # quotes stripped assert doc.metadata["md5"] # real content MD5 assert doc.metadata["md5"] != "abc123" - fake_s3_client.head_object.assert_awaited_once() - fake_s3_client.get_object.assert_awaited_once() + head_async.assert_awaited_once() + get_async.assert_awaited_once() @pytest.mark.asyncio async def test_create_document_from_s3_skips_when_etag_unchanged( - fake_s3_client, temp_db_path + fake_obstore_io, temp_db_path ): + head_async, get_async = fake_obstore_io text = b"S3 hosted content" - fake_s3_client.head_object.return_value = { - "ETag": '"abc123"', - "ContentType": "text/plain", - } - fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + head_async.return_value = _meta('"abc123"') + get_async.return_value = _get_result(text) async with HaikuRAG(temp_db_path, create=True) as client: first = await client.create_document_from_source("s3://my-bucket/file.txt") - # Second call with the same ETag must not GetObject. - fake_s3_client.get_object.reset_mock() - # Re-arm body so a stray call would still produce something readable. - fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + # Second call with the same ETag must not GET. + get_async.reset_mock() + get_async.return_value = _get_result(text) # re-arm just in case second = await client.create_document_from_source("s3://my-bucket/file.txt") assert second.id == first.id - assert fake_s3_client.head_object.await_count == 2 - fake_s3_client.get_object.assert_not_awaited() + assert head_async.await_count == 2 + get_async.assert_not_awaited() @pytest.mark.asyncio async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk( - fake_s3_client, temp_db_path + fake_obstore_io, temp_db_path ): """Multipart re-upload of same content: etag changes, MD5 doesn't. - Expected: GetObject runs to verify, but no re-chunk; only metadata.etag updates. + Expected: GET runs to verify, but no re-chunk; only metadata.etag updates. """ + head_async, get_async = fake_obstore_io text = b"S3 hosted content" - fake_s3_client.head_object.return_value = { - "ETag": '"abc123"', - "ContentType": "text/plain", - } - fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + head_async.return_value = _meta('"abc123"') + get_async.return_value = _get_result(text) async with HaikuRAG(temp_db_path, create=True) as client: first = await client.create_document_from_source("s3://my-bucket/file.txt") original_md5 = first.metadata["md5"] original_updated_at = first.updated_at - # Same bytes, different ETag (simulating a multipart re-upload). - fake_s3_client.head_object.return_value = { - "ETag": '"def456-2"', # multipart-style ETag - "ContentType": "text/plain", - } - fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + # Same bytes, different ETag (multipart re-upload). + head_async.return_value = _meta('"def456-2"') + get_async.return_value = _get_result(text) second = await client.create_document_from_source("s3://my-bucket/file.txt") assert second.id == first.id - assert second.metadata["md5"] == original_md5 # MD5 unchanged - assert second.metadata["etag"] == "def456-2" # ETag refreshed + assert second.metadata["md5"] == original_md5 + assert second.metadata["etag"] == "def456-2" assert second.updated_at >= original_updated_at - # GetObject ran once (initial create) plus once more for the etag-changed compare. - assert fake_s3_client.get_object.await_count == 2 + assert get_async.await_count == 2 # initial create + etag-changed compare @pytest.mark.asyncio async def test_create_document_from_s3_etag_changed_md5_changed_rechunks( - fake_s3_client, temp_db_path + fake_obstore_io, temp_db_path ): - fake_s3_client.head_object.return_value = { - "ETag": '"abc123"', - "ContentType": "text/plain", - } - fake_s3_client.get_object.return_value = {"Body": _streaming_body(b"original text")} + head_async, get_async = fake_obstore_io + head_async.return_value = _meta('"abc123"') + get_async.return_value = _get_result(b"original text") async with HaikuRAG(temp_db_path, create=True) as client: first = await client.create_document_from_source("s3://my-bucket/file.txt") - fake_s3_client.head_object.return_value = { - "ETag": '"new999"', - "ContentType": "text/plain", - } - fake_s3_client.get_object.return_value = { - "Body": _streaming_body(b"different text now") - } + head_async.return_value = _meta('"new999"') + get_async.return_value = _get_result(b"different text now") second = await client.create_document_from_source("s3://my-bucket/file.txt") @@ -222,40 +160,8 @@ async def test_create_document_from_s3_etag_changed_md5_changed_rechunks( @pytest.mark.asyncio async def test_create_document_from_s3_rejects_invalid_uri( - fake_s3_client, temp_db_path + fake_obstore_io, temp_db_path ): async with HaikuRAG(temp_db_path, create=True) as client: with pytest.raises(ValueError, match="Invalid S3 URI"): await client.create_document_from_source("s3://only-bucket-no-key") - - -@pytest.mark.asyncio -async def test_create_document_from_s3_passes_storage_options( - fake_aioboto3, fake_s3_client, temp_db_path -): - fake_s3_client.head_object.return_value = { - "ETag": '"abc"', - "ContentType": "text/plain", - } - fake_s3_client.get_object.return_value = {"Body": _streaming_body(b"hello")} - - async with HaikuRAG(temp_db_path, create=True) as client: - await client.create_document_from_source( - "s3://bucket/key.txt", - storage_options={ - "endpoint": "http://seaweed:8333", - "region": "us-east-1", - "allow_http": "true", - "aws_access_key_id": "AKIA", - "aws_secret_access_key": "secret", - }, - ) - - fake_aioboto3.Session.assert_called_with( - aws_access_key_id="AKIA", - aws_secret_access_key="secret", - region_name="us-east-1", - ) - fake_aioboto3.Session.return_value.client.assert_called_with( - "s3", endpoint_url="http://seaweed:8333", use_ssl=False - ) From 6f6ebf27c867fa5354bf1d8d5204d96fcbe55127 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 29 Apr 2026 14:47:59 +0300 Subject: [PATCH 5/7] Document s3 storage & monitoring --- CHANGELOG.md | 12 ++++++++ docs/cli.md | 11 +++++++ docs/configuration/processing.md | 26 +++++++++++++++++ docs/configuration/storage.md | 11 +++++++ docs/server.md | 50 ++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 140f8a5a..dbb1a98d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # Changelog ## [Unreleased] +### Added + +- **`s3://` is a first-class document source.** `create_document_from_source`, the CLI `haiku-rag add-src`, and the MCP `add_document_from_url` tool all dispatch on the `s3` URL scheme. Two-stage change detection keeps `metadata["md5"]` semantically uniform across all sources: HEAD ETag matching the stored `metadata["etag"]` short-circuits without GET; if ETag differs but bytes hash to the same MD5 (multipart re-upload, server-side `CopyObject`, SSE mode change), only the etag refreshes — no re-chunk or re-embed. Closes #357. +- **S3 / object-storage monitoring.** `monitor.s3: list[S3MonitorEntry]` adds a polling watcher per bucket prefix alongside the existing local-directory watcher. Each entry has its own `poll_interval`, `include_patterns`, `ignore_patterns`, `delete_orphans`, and `storage_options`. The same `serve --monitor` flag enables both. Orphan deletion is per-entry (scoped via `uri LIKE 's3://bucket/prefix/%'`); other buckets and prefixes are never touched. +- **`[s3]` optional extra** (`obstore>=0.9`). Required for `s3://` sources and the S3 watcher. Uses obstore — the Python binding to the same Rust `object_store` crate that LanceDB uses internally — so `monitor.s3[*].storage_options` accepts the same dict shape as `lancedb.storage_options`. Empty/missing options fall back to the AWS default credential chain. +- **`scripts/run-integration-tests.sh`** — wraps `docker compose up --wait`, `pytest -m integration`, and tear-down so the SeaweedFS-backed integration suite is a one-liner. + +### Documentation + +- New "S3 / Object Storage Monitoring" section in `docs/server.md` and `docs/configuration/processing.md` covering the `[s3]` extra, polling cadence, ETag semantics, credentials, and CLI usage. +- New "Deployment Pattern: One Writer, Many Readers" subsection in `docs/configuration/storage.md` documenting the recommended IAM split (one ingestion process + N read-only consumers). + ## [0.45.0] - 2026-05-08 ### Added diff --git a/docs/cli.md b/docs/cli.md index 668dd451..b4c9848f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -75,6 +75,17 @@ From directory (recursively adds all supported files): haiku-rag add-src /path/to/documents/ ``` +From an S3 bucket (requires the `[s3]` extra — see [Server Mode → S3 / Object Storage Monitoring](server.md#s3-object-storage-monitoring)): +```bash +# AWS S3 with credentials in the default chain (env vars, IAM role, AWS profile) +haiku-rag add-src s3://my-bucket/path/to/document.pdf + +# S3-compatible endpoint (SeaweedFS, MinIO, Cloudflare R2, etc.) +AWS_ACCESS_KEY_ID=key AWS_SECRET_ACCESS_KEY=secret AWS_REGION=us-east-1 \ + AWS_ENDPOINT_URL=http://localhost:8333 \ + haiku-rag add-src s3://my-bucket/path/to/document.pdf +``` + !!! note When adding a directory, the same content filters configured for [file monitoring](configuration/processing.md#filtering-monitored-files) are applied. This means `ignore_patterns` and `include_patterns` from your configuration will be used to filter which files are added. diff --git a/docs/configuration/processing.md b/docs/configuration/processing.md index c96cf48a..e48279c5 100644 --- a/docs/configuration/processing.md +++ b/docs/configuration/processing.md @@ -322,3 +322,29 @@ Patterns follow [gitignore syntax](https://git-scm.com/docs/gitignore#_pattern_f - `**` matches zero or more directories - `?` matches any single character - `[abc]` matches any character in the set + +### S3 / Object Storage Sources + +In addition to local directories, the watcher can poll S3-compatible buckets (AWS S3, SeaweedFS, MinIO, Cloudflare R2, etc.). Install the `[s3]` extra and configure one or more entries under `monitor.s3`: + +```yaml +monitor: + s3: + - uri: s3://my-bucket/incoming/ + poll_interval: 300 # seconds between sweeps; default 300 + include_patterns: ["*.pdf", "*.md"] + ignore_patterns: ["draft*"] + delete_orphans: true + storage_options: + endpoint: http://seaweed:8333 + aws_access_key_id: ${AWS_KEY} + aws_secret_access_key: ${AWS_SECRET} + region: us-east-1 + allow_http: "true" +``` + +Each entry is independent — own poll interval, own include/ignore patterns, own `delete_orphans` setting, own credentials. Omit `storage_options` to fall back to the AWS default credential chain (env vars, IAM role, AWS profile). + +The dict shape matches `lancedb.storage_options` — the same Rust `object_store` library is used by both, so credentials configured for the LanceDB backend can be copy-pasted here. + +See [Server Mode → S3 / Object Storage Monitoring](../server.md#s3-object-storage-monitoring) for behaviour details (ETag-based change detection, orphan-deletion scope, CLI `add-src s3://…`). diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index ebb233bb..2a576417 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -74,6 +74,17 @@ The `storage_options` keys are case-insensitive and passed directly to the under **Note:** Table optimization is automatically handled by LanceDB Cloud (`db://` URIs) and is disabled for better performance. For object storage backends (S3, Azure, GCS), optimization and vector indexing are still performed normally. +### Deployment Pattern: One Writer, Many Readers + +LanceDB on S3 supports **exactly one writer + N readers** per database URI. Multiple writers against the same URI can race on the manifest commit and corrupt state — this is a LanceDB property, not something `haiku.rag` enforces. + +The recommended layout for production is "different buckets, same account, separate IAM roles per process": + +- **Ingestion process** — IAM role with `s3:Get/List` on the documents bucket and `s3:Get/Put/Delete` on the LanceDB bucket. Runs `haiku-rag serve --monitor` (with `monitor.s3` entries pointing at the documents bucket). Exactly one such process per LanceDB URI. +- **Consumer processes** (1..N) — IAM role with `s3:Get/List` on the LanceDB bucket only. Run `haiku-rag serve --read-only --mcp`, the chat TUI, etc. They never see the documents bucket. + +Each process picks up its own credentials from the AWS default chain (env vars, IAM instance role, AWS profile), so no credentials are hard-coded in the configuration files. + ## Database Creation Databases must be explicitly created before use: diff --git a/docs/server.md b/docs/server.md index 8917f4e6..bda27147 100644 --- a/docs/server.md +++ b/docs/server.md @@ -120,3 +120,53 @@ The file monitor processes documents using [Docling](https://github.com/DS4SD/do - RST (`.rst`) URLs are also supported - the content is fetched and converted to markdown. + +## S3 / Object Storage Monitoring + +The server can also poll S3-compatible object storage (AWS S3, SeaweedFS, MinIO, Cloudflare R2, etc.) for new, modified, and deleted objects, treating each one as a document source. + +Install the optional `[s3]` extra: + +```bash +pip install haiku.rag-slim[s3] +# or, for the full package: +pip install haiku.rag[s3] +``` + +Configure one or more S3 sources under `monitor.s3` in `haiku.rag.yaml`: + +```yaml +monitor: + s3: + - uri: s3://my-bucket/incoming/ + poll_interval: 300 # seconds between sweeps; default 300 + include_patterns: ["*.pdf", "*.md"] + ignore_patterns: ["draft*"] + delete_orphans: true + storage_options: + endpoint: http://seaweed:8333 + aws_access_key_id: ${AWS_KEY} + aws_secret_access_key: ${AWS_SECRET} + region: us-east-1 + allow_http: "true" +``` + +Then start the server with `--monitor` — the same flag enables both local-directory and S3 watchers: + +```bash +haiku-rag serve --monitor +``` + +Each entry in `monitor.s3` runs as its own polling task. On every sweep the watcher lists all objects under the configured prefix, compares each object's S3 ETag against the document's stored `metadata["etag"]`, and only re-fetches keys whose ETag has changed. When the bytes turn out to match the stored MD5 (e.g. the same file was re-uploaded with a different multipart chunk size), the watcher refreshes the etag and skips re-chunking. Otherwise the document is downloaded, chunked, and re-embedded. + +### Credentials + +`storage_options` follows the same convention as `lancedb.storage_options` — the dict is passed straight to obstore (the same Rust `object_store` library LanceDB uses internally), so any keys you've configured there work here too. When `storage_options` is omitted, the watcher falls back to the AWS default credential chain (environment variables, IAM instance role, AWS profile). + +### Orphan deletion scope + +`delete_orphans: true` is per-entry: a watcher only removes documents whose URI starts with that entry's `s3://bucket/prefix/`. Documents from other buckets, prefixes, or local-file sources are never touched. + +### One-off ingestion + +`s3://` URIs are also a first-class source for `haiku-rag add-src` and the MCP `add_document_from_url` tool — see [CLI → Add Documents](cli.md#add-documents). From 6631e339ee9c61fcf22f00c26e29239887a52a4a Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 29 Apr 2026 17:24:03 +0300 Subject: [PATCH 6/7] record VCR cassettes for s3 source tests --- ..._s3_etag_changed_md5_changed_rechunks.yaml | 82 +++++++++++++++++++ ...3_etag_changed_md5_same_skips_rechunk.yaml | 42 ++++++++++ .../test_create_document_from_s3_new.yaml | 42 ++++++++++ ...ent_from_s3_skips_when_etag_unchanged.yaml | 42 ++++++++++ tests/test_s3_source.py | 4 + 5 files changed, 212 insertions(+) create mode 100644 tests/cassettes/test_s3_source/test_create_document_from_s3_etag_changed_md5_changed_rechunks.yaml create mode 100644 tests/cassettes/test_s3_source/test_create_document_from_s3_etag_changed_md5_same_skips_rechunk.yaml create mode 100644 tests/cassettes/test_s3_source/test_create_document_from_s3_new.yaml create mode 100644 tests/cassettes/test_s3_source/test_create_document_from_s3_skips_when_etag_unchanged.yaml diff --git a/tests/cassettes/test_s3_source/test_create_document_from_s3_etag_changed_md5_changed_rechunks.yaml b/tests/cassettes/test_s3_source/test_create_document_from_s3_etag_changed_md5_changed_rechunks.yaml new file mode 100644 index 00000000..c26b4840 --- /dev/null +++ b/tests/cassettes/test_s3_source/test_create_document_from_s3_etag_changed_md5_changed_rechunks.yaml @@ -0,0 +1,82 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '83' + content-type: + - application/json + host: + - localhost:11434 + method: POST + parsed_body: + encoding_format: base64 + input: + - original text + model: qwen3-embedding:4b + uri: http://localhost:11434/v1/embeddings + response: + headers: + content-type: + - application/json + transfer-encoding: + - chunked + parsed_body: + data: + - embedding: gmkTtz+8PT0wHXE9Q+m7PF8r4LkDZE89vujfPD1OFb2weqY8kxupPICyFLwJLL48qxAluz/LHL22xMM8sgZNvUENNjygFNC7Tj/iu252V7vgKqK8uHqLvKiO7zw575872t8BPPcLx7zKT8C8lG7DvfrvhD1FRxS8tNFPPJ1087ydw7M8ZdhqvNRYvjvknQW8kRO1O0nTVLzmMRw83cd0PH+zvjxALB+9xt2WPM/A6DsSC2y8NAPWPOPZAjsjS7c8TI6+u7p0Z7s9SC08No/gOp6n77xFIfW8sV9WPKUp3bwDZg896M/8uwJ0Or3Qjm48DtznONajubuDyAC8ss/xvBnclDk/l9q8t/fqPCpGM71N4Sw8VSMHvIXGuzzulyE8/DCmvCQjnbuEQc+8RTD0vLb9y7umxTg8cncQPAmUzTzNid47Wndnu6JzTDwFeCM9g3YGPYAy9Ty3g5o6GsKTO9NCGzzI7xO8ZJpTPHswEz3RfZ68JKnXPKBCDbtll2C7ESGMvGICULyKLLa8ipKMO8LmATw74iY7I4C/O9k+oLvTtsu8bksEvb+XEbzW6BC83MHmOj8B9rupLUG8O9DluysC0jxJaGG8vdi6vPUMhbzBXEA8Cn6QPCNwTztWiLQ8JR1EuksjCT37+Qm5BE2uui7z3Dt83Da8/pU8vEBlcruZDVY7jv0qPHttAz0oHnI7xIboPCwix7z/cNG8I20IO4flirzI7lw8zi1AvFb0XbtgAiw8pcM7ugHTcjyJIzO9SIWrvB1qbryzBEa7rl7nO3Pq5jw6WI48whixPNH9ErtHbBU8ZcE0PUhZJTybu9g8nzDwO0F87jtDxam6AdMgPK3EO7zk1+E7HEYMvYbU77vikZs8Im7fOxSVC70O44q70k2+vF5WMbzm9Ii6hGGhu1X0qbpmJQu8BYj8vFYPirxlyQ+98C4uvSMiM7tn8hU9tfMpvAKQWzwa5Vc5TZULvN06iDwh3wM8dwYRPNDBuTsXDrY8ZQIcOy2RvrutMxS8xE2du3fPHLzeuL+7B7sKvJ2xCb19DeS7NnJTPA1ZYjwiabG7b6/Ou2Hn4zyIMia847Tcu3vLWDwi/po7RGgdO8qmhjsfl4C8U+WVPEoYODzpSGe80tklvPhQ7TpQluQ5eBKhvHW9QTvBX9w6rBDJPNLtkzw6giY8sQLgvHaThztHAk291S6wPKoWuDlkMxW8H1zmOxBYhzwu0qW6LxwcPBNW+7tL6pa7ynNsPB4ITLxWKiM9QvGJvK+dLrxL6Ns7+ipMvKtHEryfUDK8fQz+OoS+JLtSnAG8plkzOyWsK7yznpC8gsWJvEWZGrw4ZJK71vOAPKnOyLyxRqG8iMYSuzk7gruemTu9eUTrO1u08zqfApU2I23sPBFFhbyLaku8U4wrupDfVjvDFo46/v2Au15nertUruC8k5uDPU6asLyXOWa6tAVoPBQgQDt38yi8K/BrOiKihzxfLAM83qnXPADYhbwLy2u8fBYpvKFtITxKeMs8rLImPA/t+Tx0uqA7VIMCvazSKDxtv1e77xlJvTYiBz3LMr+75K4NvV82rbyntkU87ecLPC2CCzyzxGO8a3MvvI29krz/69Y7lSMYvfc2+jywoVQ9mUlePCP9Xjwbk1o9RjgZvUNKo7vDzs27oCQoPEM6wLxANa68v+BVvd7BzLtsE5K7lsuOvH//urvr/1w8AdI/vf0KQr2SoQq8AMo1u3JAJjx13Kk8hpw/vFd5ujiRfIc89NxYvRQwJD1Yoim6c0dWvINoCz2C+M+8ddWtO+JZQT3XW+o8cggkPDQ9LDtSZIs5G3OVOuIxLrs3bxO9EanlOkyokzy3a5Q7Uo5JvIAPlby4AJc8VY6CvG/vDTshhc67YlyRPHKN0zz12qy8qStLPMw3kDzBPfW7+hyMvDH/DbueMHQ8jBKGPC/kGzxJiHY7OJ8fvAaGhTwRvqU8YublvOOrwTvsAIw8DfcBPF1m6LrnUxK6F/gvPLLkpDzaWZ07roAGvAEwOTwQLQg8V4yFvPlsCL2CriQ9ElyyuhUIsDxBQ5a7TygbPeq207yxC9o6L1IhPBu1e7zUOw88nX+PvCT2Ezymkx09Wv1IPK1D6ztpYCk8uRZkvB0IujoNLke9tx6VvC3Qj7z56268moyevDENHrx7Iz09zzHHPHtmK7yLjMi6U3HcPECEPzxBji68y3CYuseSTTxMIWA879qkvKbju7xUvw68YbCbvDvuXLxkGoi8FKQqvJCVRzyLcVw7e81TvLZCdzzhZwi9e264vJeEcjuASSM8EteOO/DpWj05fBG8xaFMPcBOjjv2Lxa8ao+QvCrfB71uxP67lWINPII3czsyT9c8S02kvOVTYjxC91g7tP3IPGLYBj3U76+7SX2VvDn2IDzZb9O773IWO0W7c7zE3tM7dTLdOSZ6+zvvYq+8g3ASus1txr3lRLE86xOxPIVZSrtchE88Tcr4vIWeSbyL8328wa9KO9ZfMTvqIe28dOaFvOH6B72R7Co8StFfO5uhzTwCCAA8C8HSO6QlEzuSgtw7S/eYPUWUQzzAScw8RllkvPoXyjy0S6s7ZK/SPF261zyoEQw99RFHPEVC1DtsRcc7gFXAO2hTxrygp0y8k67LPIS0zTty4LA45ujFu/IkcjuNHVi8mOoZu/UBhzyVxBc8cPIjvM4mT7ymeEA8Kz7euzc8GTxK3/07/Ka0PPiHwLsOkoY8o5xTvbvoXj1rhaC7skoAvb+6HL3hpRm8ZhJPu7zNt7zlugc9+sPDPMr7dLsCahs8v2aoPG7ATTxWm4U8Ysn1vMhrBzyoYbe7h4quO85gSDz0FO48dqFbvauuLTzzwKy8Jt8TPYS/nTu2M+c7Zm0vu4GoW7tZ+Ac9RghDvV68Nzy9nvs7gfGLvEjCbzxZZvw74NZhPN+LqTwYgPy80ojFOjmyMLwV2Rw81CmWPACSSzxAET08xps9vIFi7rwc2GW8TgAZvaDXATxA2E88RHadPNLNdjxMHto7B8cFPfrQ27w/E9G70cbBvDHJ1zzW7Yo8B5a6PDZTPDy2CU+91Oywt0/VJr2Mhi+9TCecvEifhbysWdo62iLzO6PbabwfZmU8UOihPKFsp7ttO2q8EoUBPfTvujzLZqK8Q6k1vIqlgTzZS1w7QBOLu32nnLyBivi7edRPu7/RC73TaWO8QWZDPUv8i7wQvDq6iG6TvPpCK7wVnf067i9wOxnI1Dx4d/O7I5fwu6MjQ7xnbJ868jT/uzmT2Tzxji08MrvrvBgff7uMBAM8fU+5vOfO1zuatoG8B5srvPRFubzI9pw85P4iu+wEWb3VDBA8uziqO20ctLsUyfs7U4aYO6Yre7xZ4vo8hD0UvPvDGbw40Su7jwSxvdiZjTxwDaI8ZYApvaTVtTvmSrw86Mo5PKFP7DquZaa7bdxvuw1/yrti/7i8f5CwOQxArzxFV7O6ZZ8NPRcOjbvmsAe76i/Su4RfK7xoyCQ8U0u4u1AOIT3aM/Q7dMkIvZbbDrxkFMG8l/TAuqD+0Tx3FCI7NgUzvTzymTxnE4I9mvzsOzrx0ryYJD49zivyO+K3aLs60Q08gCMRvU8hgjtvQ128M84PPStCxLwk3wO70NQDPfZAmj3Mn3O8vXChPPAXND0R8KI55H+7vLl2sjgNfe48JUbqOlOHpbs3psw8OzxGPShstbwhcbE7zffsu1IG1zzKrfO8sa7EO5VG9TrDDIK8RVzUO1nXk7xzZMy8Vi/cOtNGnzmS7y87pNq6PJuvsTx5HUo83TRPvOs2uryQRiq9xq8cu0C/rToQ8Wi8+U5VvANUPzzc6Ey8irh2vJczi7z7lyC9cU16vM2tDbwlrlm7XKO6PNwEBL1G+Ym71fqFPFs4hjyDElc89paZvA890zy3kRu8tnSxvG8GZLpbI2q8TM1wPAVoyDzMnns8SCa5PIL1uzvQ8vM6cZfyvCAkqjzR3tu7uOtOvFSF0LuFcRi9gALsu1tOBj1wprw86W2NvFF1Ab37ues8b3iOPFasvTy4Row8VOIwvH07ETxCtRW8ut4+O/5GorzoPse8hO7jPHIYHzxp0h+8h5DMvOFnLjxlAd48zCUYPaBhprz72447Wd+5PPg77jvW30m9JvAZvTMomrvIVcC7ns+WPPxtJL0DzVo8TvGhu3vZZLzkvhm7HDe+vMgM8DtKxSy8LkkkvXch7zw8JK08wh6wvEwhTLzPtYg8oXXuvNaJhbumZ/U8jPrQvC8PpLpy5La8RjQmOgVwSzygJjY997d2PG0u7Dsiw/e6/EmJvNxyBj2KC4y8jm0PPcA9gLwX/6U7P7oAu3pj3juTcgi90sOUPCFGyzhcI4U8AOGgPJotDT15CsQ72l5SvOI5Cbz+vha7ywW4OzinDT2Rm+m5u8K0O/9TzTuXxFO8bBjKuxZUoDyWlhC8sjKPO69D9bsTBBI9M49XvTN4uryT8XC7xhNqu7+77juWsuw8KNCeOXLw8LxPwjA7uRQ6PGMxErt5Ev07jg5GPM99dzzMoA49Ml+VPLZQuTsSoGC8wIoUPV73Az1ESqc7kuGYuqRGeTlzOqA8pagnvcrjeTxRzxS8vB9xvHNQTDwpIzs7nT/7u5LLmDyQLQy92nhzPIBygDwYSfI8iBLkOgGA6Dyd9FS82C0Lu6rCtbyr6Dq9AR0dPAN8SzwRbpS7Eu8PPea68LtmF6S88QmpuzBURDsPmgM8UxGOO2EUkryrPHK8syHRuhed3TtuMH073N4nPI+H4LxpGui6dlI9vI6zvzy+ghu9WwwJPNDt7bpGbnK8kJbEPESmlDwbMIA8oVeBu8WQ4LuNpAA8iguGvGeiTLyn2gG9szGLPF04QjvQOQO9YAxMvJGuCrusfzM8D3zZu/7Xp7oxYBk9ArSGPMFlSr3WZEU8BnRAPcQEab1itLi8Eld7vNivvLwFYKC81oFIvO+KmTx4eQy7UMTUvKc5oztRCKS6SzByueJEirq4fJC6iZs1PAWcEzypjZo7T150vL4dbD2dtgy701cBvDnlFj1VnV47RuCYPIUqxjxybFe8UCE7vD1EUL3Exve7xP/Ru0I1vLwUqgo78NEZPPkLBT36i5S8Uec3vJ3m5jz9pZC83ae9PHJKvLu7Wto8cI2yuqrZfDyj0aw6rvkrvNawYbwMbWY80lPPvNBbarudWCi8XIIqvPmTODvkpjc9vpvRPOVP7DykYxo78geGurxjNbz3TCS8tGq9O3NyGLyrjmI8QjM0vFmCGDyk8pm8XAvZO7N3VDw7ZcY7uo9iPNYc3buyrf08r0KvO+Drf7yfSkI7lteaO6+hsLxjLdm6vvyLPPcEb7xy4gs9wATzu+y3dzznf+i6lroGPQCx/7vUebc5J0EQvYspGTx85y+87EnFvBPx5Ty7bmg8uzcbPO7lPb2J8AU8y8o2vIcJq7wS/gS7Oc6CPFJmaDybgem8PK9hPTEYsrpCGum8bxO9PNj4sTxlk4077izbu1ogPjsct1088nTfOxnY7bzvOmW8SqHLvIvJ+7uhhwK9WqYLPMJ5cbxUBbi8hXZ6u0Bj+Dt1vrE8DxeJvM6Gb7tpk7C6/XP2OnkLgzz3QDA7+HPHukwHqbz4Hjo8RT6+PGkK/DxG43Q8v65KPPos9LsZWSI8/eo/vJnyGrxKhNm64AsgPNevzTsmFBA8CQitvHmaBj3/34a8MLUOPX8inbsV1Y68Atc6PPFH5LqD7ae7da7fvO70HL38PZG8jfchvTod8jv4tJy8JRR7PDqb9Ly//pW8s4XPPPo5R7wR1rS8/buSPPFjBzymU4G7Tn8nPHDlb7y5RR49W4gfvAX1hbyshAA8afjMPL2rEjzbJgg9yScSvMb0YjwjmKe6bPICvMsbyrvYEXy8LiwOPRsZNDz87RE92WJsvBjxDDwbwiC87C8fPJ0OiDzgKuE7eG66vFZImDwpxBi6ZtMXvYKBibyoZw681jmJOt4WRrs35iS8XFOmPPrdyjy7sRI7uCPWvLdKprzRPLi8EfIAPePlNDwyV5Y8DklmPQzgDj3q2nW8g2GXu9tTKT3tFpE84mTFvOCDrzt30zO8cczHug8zTztOgDo9t551vEE2XD0CvDu8cT+mO/6+67unlQo8Y1K0u9KIPTzcQh28/rWXvK8I6jwBWyc9qLWqu7XYGrpRf+E8CSoSPPZxgryQrlA82gF2vOwyFzzMGA68hyiDPCMRybyPI7Q6pCCOvChDp7zxeQ27z2s7vAfcTjuzw6Y83UnKPH3Gajw0NVk7mB9cvFOpwDzk6iq8Qq9wPCv4Rb077ma8ufoCvLUPLDr933W7xM2MukscHzud4co8nIoWvCVbkjyhnl+8BXkvurlv0byMY7K8Jj8dvdoGx7zG1eW8FnhTPKZ39byLZL86dBCUvBlTaTv3HFY7zBsEvLrhkTwl2727fdooPW8wgzyN7TA8n8nCvGjZ0Tyto2G8pdnJu/PJTzyRstG8k9YLPUraabwwJoi7Z7IEPKvTPjxZ3IS8iNyeu1haszpai7W8N0ESO22MUjxiYwK8xM20OtcL3DxO1fI7daDfPGwdxrw9arc8pRUavCjlIjtGNZE8uC+/PEXHWzqYy0I7r7+UPIMNojwcdE89fz2qPChbt7wRrm48zscnPJoIs7xFOcq71IwOuUrnrbyalFM72KvOvAjKGzzPO508MdxUvIWh+ju+Whc9KbaPujbrJ7w+wk+83AzfvPw9VDxl8aS89OsCPRu1nTywujq78EZYuwWprDz/aLc8gnFAvOId+zs4G5+8Fo44POBMgjx7uJ68rYakvCQQj7yzz4E7Oi82PFM9nTzmKl48/0esu60KqLzmxe+8k3ltPNRNjzw8+mO82wURO+MEazt1QxW8Qgzeuk8Pj7tLZpO8xeaVPKl1nry2jv+7iEKjvIJfJDzDNLc8o36vOxeSGzzBUKK7UeuivIue+DxVlQk74qMUvZ+Erry5f/E8DYqfvFSCsbsYcWA876YrPQJF/rtY9Uq8PVq+vKRFFzyfBpw7cY+IvBIqwDu8+WA8qX46vNaKTjz1QfY7LNdcu+hlxzu5iHC8aL7wu4baDL06buk8XsM7vHrrVLyRmG682S6QvO4izDxwZok8PW9APYXQgb0+2i+83P1evJcuaLzgtcC5kRR1PBn+vjxRejq9tbTPPCv5kTzLUiu9jyCau0Ptg7wCKy08Ec7MPPlLmbzR27E7pnN1PNoDILwaPme7zAyTOxljFbu0AvO7GeDdPOkWtrwpLty8Mws5PN773TzYoB88QdCcunLqnTyHbtK7F4L9PKrRCr0flvK7lUsFO1JISjwu/MK6MP4nPJngKLyk5/a8tTU5OuHSrbwSzcy7zhl8vMKF0DwKLy47gJkmvczHBzyAiY88owNTPM1hOzwWVAK9utapvI/B3TxEKr48rA4dvVeG2jk4yNu6oXFCvWZ84DsImRy8vhOMPFWqqrxx3b08eKcWO/W+8bwRH4Q8bfKZOpdvSrzWOIK9xxWTOkT8CD1yQYO8p3EVvV3ckLxpRVi81Yl5vO2wwbxoBdg6JfkPPBcKXzy6Fog8gqBWPMnmZ7xM10I8tRjdvGDAID2wKaQ8nJ0oPO/dSzwfHKQ7IlPRvFQnhLxDTLu8gkmvO9cOGbxB3Oo6mlKTPMbJ9zyhA+W7tiDzO2eFCbylG2o79vGZuh+/47rWWrY87LeNPKHWrLxhuwO8uDCoPFVLzbwDuaU5Z2XMOQZiaLy7dKU8r6KtvENGwTyQB1a8Mi8kPAIwjzsPQLE8favrPJzBrryFPo07NjYqPfzh1ryt5PO8JfHEO9X1crsvmpK8ry8YvbURDbxHACE8Bj7iOhUulzsvz4085BeCPJegwryA8WE80DI1O7FiVTwQgY88xAZMPFtPEru33M68AnFUvNUPFL01PJu8ox7dO21DYLx1svu7ixSmvFDn+ztdtbG8crEJPWSf2LwtUgm9xr2TvDOX7zv9i++8aBZoPOvlHT2wwC+86j9APC1qxLuvhfK8DcePPLRyEL3dIyA9n1pKOyJ+R7zRO327Wq8GvIsLtTxujHi8ZELLPC/0N7y8hOM6LJp4vBbWnTyHprM8UotlPPq9Zzxc9n68go3iO99ut7weZN88lErGOiQ9Bbxdp4Q68iryO3KbCbt6pVU8tGrRuoB/kbxKnBI9mJtiPB1qN7zIuyc73JdsvDLf1DvgpwE8xvlavMilET2YcA88AweTPA0whbq8bYU7zvGlO+daBz2b8Ey8teNJvBiXnTvWdfg745MgvBc5tTwFhA09d+7mPAnzpbyH78E8q4+CuiAnFD1VMrq8tQe/usfsgzzg3OM8/AWYPE0KhTxSLqS7bjSzvGbGWrw03js8mbiVPJf/EjxX0u+6ChJaOqfqbzodsOm70xRevBrUiTyzCPI83wEFO+rmZDuv8Ju8Bp+xu795Urz9Ko88omOQPEkmMrzkDt+7iMTRvPWpGTy1+wQ9MB74uy5vOrqlgUM8TVwWu8TkID2U8x89koknPFeYlDwkLPG82R/TO/7drDw/kEK9isuWPF79LDzwsgq9yyXJvK2dGjuB1uY8rPGPPIRi47vCXpq7bTnWu5QcAzxqplu7+3HdPJWXIjy6GyE9yrAiPF4AArypENG8yw7CvOl6yDyWAKI84fazvOveorv1eya6xFx6vN0uiTww+dG7c6uhvNzplrxYKES9vCxBPGgBzrxDgs48mgiHPFHGaLxIDFA86TSkvAWXDT0bSEO8OnHcvJ09l7wTlmc81MulvF/SKLwGBOw8SO7nPD+7urvhooQ8YoyHuk6RwDvqE2i8W8GbOxrme7xgh4Q69uoIPM3eQrz7NbM7fXEaPU5oSLy/tPG8We6Puw65rDmULnw8achQvHnCgDo1ElI8DLG5vAzw4Tw6TZM7XPp2PDaYr7wdy0+9HWQePSJk97z3ABC7Jp1Auzi7Jb0LZRS8KAEVPClUWLwd+2y8GQgEOfHzDbxPJdi8mbWJPMRLFruwdx25ATpQu7K/Az29pBI8V5R5PF4nujzMhtG6ErjSvCBY6TvA6T28q02zvOVauTs4T+W7DxyoPAR/MzxlAzW9gIrhvEplJboTE3g8x6WqPL8aCrxnHZ68tfbmvJUUnzxECms8I+CvukZqC7vCRvM8vl89PN/lmDs+IXY89gjNvI8JETwy0i+7fWYfPHL9jDxLSlI87G0DPH3f3jx1hYO7DK71O8tuqzzQOt68a4FQPP/OCbyqDta8R2Wqu8A6ODtTn7W8GEVYvOx/+Dw+D5e8eZIFPLoRYLvBypQ8cUyJO3tNQTzNBhA9JYmGPEg3QbxFZHm8JByBvItSH7rlrdg7U+dKPUC0gzzRdli80nMavGAuuDy76qU8hJaFOpCkp7yMthG9hBxsvOIVgbwdG9Y7N3f+O9TFB7yoQom8aOT7OiMESz1MyJ88Pg8GPWOiizza0l68oI9OPHMh6DyWz3e8bkOAvClK/jrw6oW7Ih6ivLHSsjt9uaY8TPRkPMZxrzwVIwg8kkMQPACEjzwFuxK9+ZAfPNzMJryn1J+8TdbWPE/EHrqG4L+7ByizPA6SGbz0t/o8kys5vApdGb0Z6S079k6gvG0ZtzyOMXC8ZKo4uxBYJDpO1FG8epmdvLrNtTxAP1k8PeUUvVJdBbsyoAs7fE3JOjDWtLtm2mo895SsvOlZjjxfsTs4aZitOxCStbzMKhO8xNfgvDqzXTv4FK28+FDLu9UyADwBgDA8uOSkvJ7yTjyJ6AG8MpzEvCzmGj3Dcqg7giMuO0MTwbwTBhc9vAEKO2Rr3jzJw+S7oEaavOMTvzx0dRS8qwgKPTBw/TusOvo8O96WvNQhmjzh/Io7GBmKvJP/1LsfbQa8CBXWO70yLTv40bO7ETjYPJbVhzxeho86Kf8RPN7Xm7sq9ti88HCOO93uUzuATa678mMvu/aYZzyHUJ68gu/uO0uY8zgij0W8CvAevU2mYzzbnZu8ldifPFG0mjz+rm87Ue1UvLqEEryhCte851GFOyagSLtwKag81zvwPGqHxrxdC3e8PSLzPPW0+bpgKnm8UXyGvDADO7yt0ik7dXCPuwyysLw50Zg83LpbPNfZdzzxiAW7UBkBPOG7HbxzB2s5wEIBvao/Tbo86hi5BJ5KPBez+bvXv5i8pn9YvXDvQLswq7S87lRRvD5pDTzjJIA8IdjeO2ETTTu7q5k8bRRsPPbahTy2XMG89GKyO8aW5ju7owc9ri2Cu7sAyDyFNSK8H1wyvFQlprzQBu27bFbYOvFM8bxcwWS7FQqjO1iTdruOL1K8NXQpPUCZgDsaCjI746UxO1pnaTzSqsM7vJ7fPPAO17y5Ooe8WWQGPdLkubylESK7R/K8PAkLKTwo+sy8SQ09vKyh6bwYpsq8x8a6OkXj5Dps2oe6+rPwu6c037yL3pu8X4X2O6HkNDwaBhy8N3NmvRcotLzK2JU7QTOxuzzrVT1XF+27NT26PNsllrw3+Aa8CZdIPC2SRDsTj9i8c8WJPIz7ADw9B8a8XFBCvJSCG7wpCG29eJbIPK2fLzxbTZa7UQuwPAZjCj2Lpyw7u3BkOyPKjLwi4l+7cXQHPQmyRbwvJji8Rbm+vMFHdjy68TM9/wMYuzHcfDxO8i09nq1YuRfTjbyM3YI61CE7u0uIljwpcI87Pf0auY3QI72OsZe8DwHBu6d+K73zmt+8ce8ZvMiDXb2HYH88GkQDvUEVRb1eQpI7K9NbPBHHejwReau6ODYNOhMvhDyo1qE7Zqa6OoMjHryShQK8Uervu1KxC7sSF3u8a0mEvBQ7Aby2X548h6QVPO1FyLz3RJ88hC6gPF+jlLx3JAW9X85rPOl6VDwO0hm8dfFYO/L6szyodx67VwSOPIyy4DsarEO8u+ksPIaHLjyZTTo8Xm4VPaO5E7wvaQc8l+UJvZpgdzwI4UW87c+2vM4WHD3oe4i6L319vLnQYDiw50O8nnmOPPP6Bj18GAS9UxhTPALEWbzIfIW81ztSvKqVIjx8Dgw7t1S+vO1GHDw/TmW7n7d/PIf57rnH4xQ7CTSJvJjqEzuMHJK8Jm9lvMMVYbuYxoS8lx75OxAh4juFzgm9/XYSvCH7jDyg3n+8zdLau2JaSrxAvRO84IOsPAFblzya05Q8icHLOyYshT0ilnS8+beHPPu7VDyQ/Si9oHmJOf4HyDytTuy87QxIPVrNnTxt4hC86V0APcVzcbsW6fQ7t7qTOzAOubytDJK8afJSPHoekTyyowy8y+vHvGvsHLyNG6I7QuQZOwp8/zwlvwM8gvfwvOLqlrypFHg8I/mYOuoRQDwxYRm8DDpgu485r7zn49W8uDlzO8FlQLv7GQk7H8GGvN4EGz0ZePc7zzosPPQrqLxpT4o8w11JPEvXtbv6x1Y7dvpYuxJybzzlUdk8PeL/O1tXeTy22Os8WLlOPFZa3zweu428tHagPBy7/7zFY0G7UBZxPP2pgrwclcM8L0B9PAf7/rwYFPW8YLpXPLY6HLxDiYk87xslvbNjd7t9RrA5MD5WvOUtOr04aMm8jZBzvEDgMLtr2lc8rV2GPKLqlLtvYsm8u3gpvCw5HLw/+108tJiCvJeKBbzywUu7PmifPEy7+LyoCZw8CoU5PFOlFTwCbeW8wR8Zve/VDT2TFvc8xartuqwO2LxDWUE9H+eevFcz6zpqXXU8nqP1u5JtjryuCCe8t51lvA7s7rwd0Ji8LmVjOueXkzx1bTy8cibvu1AmNT2q8Ru9sSa4us70lrxXbu+8owXmOWxqeLxZIzM8LmiBPFZiOTxYVSa8WELFvHQ9IrynBIE78d6kPDjKj7zEEfE6mda5PKpsbjwBhPQ82YagO0wahzzUOKg72P5ePPoX6rw8eLy7gzgJuw7SA71zLbO7B2GTOqboA71/1kk87vO2uxtnbzzZqwO8vQxfPFf3A7xrMbm7Z9iCvBkz/TzHOKu7iTBZvFKfJDvq3jS8VP/BvBAgSbzY49M8xEgavBQS2zvHvmo8gEnLu2cYHzr+QQs8ErXUvIcaBbwNttg64CTPPN/7sDsRHGy6mbI3OvxPkDxZg8i7fYx2vGg1gr1hJnU8x/WFOygF1bxLh5g8MiGaPBt+2jwrrbW8wzmjPPqvkLxBzXG8Ld4Zu1S9eryf33m8FxydPFjVbryQGrK8kUROu1hnpjuG1pi6K/Q1vH8XBDx03o28/Hm4uz8J4DiB0xi8F/UAvYHwSDtYmr28/o4svehY8DzLp505hYKCPISFfjxbWT28JGMJPF7fID0LKYa8MmBUvMyh3Lw8jY88IkjvPJvMdzzECZO7QLwNu7dkpbw/G8S7fQ+jOk0TPDz3fho6fHE/vJNah7xY7To8G2IFPQup7bl/GkQ8lbZ/PPHqHTx52QI8fYMSvHzcPDz3r/U6wSmrvH4ZXDzfk5O8Gve5vG85IDwdYci8VSwnvNUyIbyULsM7dr3wu+c5i7ulpiA8uxbgu9QeHD1SOZW8uNz7vFD8/bulTuK8ca6pO4jYnbyDE0O8j24PPZUfATup1pI760IFvEyJwzw5ucQ4IeOqu1A5fzxk9Qi8LhR4PCdoSDxJgyG88EVOvMlNSbu3y0K8/yMIPZFjLLzRPOy8ZfhVu25eu7vMnHy73PDwPKHRCbtQMlA6+YTnO8wepDpjBMU8h+Nbu3dutDwiBCQ9AUCRPJg7xDy20ak7NfIPvRei8Lsl3J253gphvKBR8DtvweQ8f78lPO3glLt6AwU9ZFCqPGycDDykcjo87+Nru9+DZDxAZcA7RSDNPNaws7y62ry7dSUEO0w0Nbz3YRk8g+iLvHuLiDwxEVw8wpyVO5ipmbv51gI7uHe8u7G5obw3p6C8ZPjTvEgVJD3xnE88FaLGvK1H1bzc/eU7GDOrPCmu+Tvcra+769nuO+NTqbwzAJ68f28Svc/MozzA4TC8hUnpOwPtBbwnvT89bvMpvchARTySvFa7ZSnEPEd0Ar3IHGO80NnZPD3TIDxYG3q8lkMYvCZXkjy51qm334CDPBATsLwoJws8y6MKvL+qcjyGOaU7yqyOPI5z4jsFE2U8JqHCO4VarLy97a08CiG2vCn7xTxN6e08cEbfO9Cgp7zGVtY7ima3PPwp+jyL/CI9zxZMPBsbkLw+ezu80x02PAwHjroZNIa6LTcEPQf0Qbx6d8u8VRoRPBDlVzxpGg691JgfvJ2burpbVaM8q7EEPdXeqrxo4BG8v4HRujTqOby3asw7dRWNu7+JPbzvVKO8zawPPfw5YzyW2xw77wC2PMCnNTxL7N27PKnduxMHJTzvMAi8aFvDO6+4Yjwbyny7V1EEux2ZQjzDWbc74BaZO6jdizrTBNC75Xy1PGu+prxzLBy8QledPNp/ULzDmKG7qdlhvHKm+zsAxcK7BfzSvDBXqzweoU88cwwFPD4W8zvJiSA6215NvHPmWrv8FbI7sULTOwDwr7yk4xq86N2BvK5xFLpo8uw79dNyu3rI3DyIsAA86WR1u0IgfDzrfrK8l7IzOzf6K733bzm7QvzrPHWJcjzN+dm60IIovJYWtby3cic8BJMjvRBiILx9gIu6iCocvAB4cLxfihY8cq0avF3gibz2VyE8kPNnPA/xxbwY8Ok8UwU5vA== + index: 0 + object: embedding + model: qwen3-embedding:4b + object: list + usage: + prompt_tokens: 3 + total_tokens: 3 + status: + code: 200 + message: OK +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '88' + content-type: + - application/json + host: + - localhost:11434 + method: POST + parsed_body: + encoding_format: base64 + input: + - different text now + model: qwen3-embedding:4b + uri: http://localhost:11434/v1/embeddings + response: + headers: + content-type: + - application/json + transfer-encoding: + - chunked + parsed_body: + data: + - embedding: xnlWuX6KMT3JS1w9m3liOkwkh7piHSk9bRiBPfy/xLxdgoc8elo9O8gxfzwtYZ08ro4/O5Civbxxn7U7dPYhvRmc3Lx6TN470y8vPNT/pbtbFcK82ksNPE08ED1MOhy8ZksWPG7EHLxBquC8khqWvbibej1fBi08aeDVu6kOcL2A+AY94P3UuzpNCTwQ6vG7lZldvCxQR7ztQGk8TuAMvI6dgDwmRS28T0zGukXfhjvip048Qe6RPOWf1Tv+QDY97k/YvMWvdbzICsI7SaiAPItkn7yWSge9zPpVO9ZCnDw9F8w8N6ahOlkwebxg9Jq8tzHSuyISKbz//nG8z4LcvGDMfLvECMm8R3uGPHlolLzUs688cibeu1mFAzvq+1k9Y8JtvDY5jDxaC948GpEHvZtsgjrwpqs8Vn+bPE5djDwCiw48DlQOPNR8/Tibi2I8oxnbPMICizx5mHo863sZPJltRjtBydq7Lb85PMMvYzwW2yO8GDITPWpHLjyR9/c7p0L/u0MnpbwyDMe8A4woPH+DyzvlfwU8kh5qPJURx7yqqMy8afncvKxPmrw+eMA7HyY6u2cOQTx9Gku8vFSuuw/WFT07B/U6ZyipvMl1trwSAEe60zwVPeghYjz9FwQ8M24/vBq8Dz3fzJG8/dQgvFrcBTzfCYW8lGIJvJ1HD7qnAoU8y84xPCQlETwDsaE7ZGD1OgWSp7zJLWq7PvNwO9jj/7uLSgC8A8W1OsrZAzxXZ9O7rLtNu0B6WjxbBQI9vwfyvLltWDskr6q7o4HZPOOmvjzrZhY8KRrOOwT9XbzDaxU74lznPBsjXjz3ENU80snwu7Oa8zwGqea7sOZwPKJm4TtwX7E8WicWvJ9nWT2DcUA86fQtPO85rrzZ6Iq6CalOvBa3xrwvQzK6W3+3vPbVoryHYtC7PcSvvM3Ep7xZvh+9WuATvNLQabw+Cr67Dv9HvPsaqzpgR/s8pXZsvAEg0DzpKwU8KTc0vL3zPDwYj+E8DPcaPINr8jtMxu47Wobyu0AcgbtDEyO8LC5PvIo6nLyIm6O66dm4umkLUDxgC3c8ok9TuN68WT1uasu8Y6jgu1FGsLwtbkE8VQEVvHvqqTyZaNa8G5phPAkuLjwCpVq8CvybO29B5rpnJoM8X/TdvKxJKbqY4Yo8hmpxPVSXlzwRvRI8d96bvNDhGTz0/k29AwZaPDiphjtN3V68Ck24OyLRy7oucJU7oitdPLx1yztqtR87ISamO5lO7rswvsQ8fZjdutwbYrxv0QW7wRiZu4dfGrzW3Cq534VLPMtGlDvLCaq85jW9PPsCNryI/Iq8KQryvAAQLDtdsda74lOUPKsbv7wXv4y84weEukygw7sdwW+8F5K6OWqaqbxUTzy6n3TvPBbsQ7yj8nq8OCiVu+S7WLuVzo08fXolu7hACbwtMqy8COCTPRQ0QbyETUQ7PmuPPDbl3zvwcIi80NllO4T1JzxXUcY6kB5TPcMiwrucE/q7HwgovSqAFjyWas88FgS0uyKMNLxBYoM73KT1vBeWNTz4YAk8XTQPvQP/lTyvqXC85aiCPE8XUbsG9iM8VQUjO2/0vbwvpm28XGrPuua4eLxETTs7zRCbO0s1CzyuQmk9gT9xO2qM9jspFxu86SqAvL2u4bzt9w68KrMNPBphN7x6uUK71EWIvJZ4NDtAZza6oi21vCsNgrvNkpu7Q/cYvaIKV72Ea2m8fsQAu727trqinsY8nsWUPGCH4Du535c7h+m7vNaQtjxe+bK886khvPueszx4dF28Htg5vBT8Ez051Vo8Uc7vOxwikrt8RD28QZyFvMvKybzFFpO9F7NBvIGjzzzKlK02/fU0vFbmYbwl9s08sLxJuuNQwbzltq28FO8bPPgmAT1Ikdq8QvyaPKQwdD11oRC90ZIAvCSThDuK+tu7bkJmPLV78bsQi0O85cadvKzsHD3Zwwi8ZuncvHp9NDxxz9c7TN+YPJ9yxjx+Kqg87wLVu5cokbuB6Aw6JILvuyZnSDyxK407alG/vEe02byEy/I80YnkvEfahzxdDRO81AvyPABjwzsR2e07La0PvEFvCD1oV8E8lzvKOxBiIbrsr3k9dQ5IPEppgjzDegc87HC0vMVvjDqQT9S8/NUKvaRUH72j0Iq79ypmvMs5DryhUj49GJeEPJybPzvMicu7sQx/PE0iBDz4F+q8jjgYu7AH2Dv+jZY8852du4iFFL1fvje8JkO8OXVPirzO8cq7whVQPBwzkzwsTAI895ZCvGV0gjxyA9e8DjWbvPQ7NjwncvM8BRLyvDeuJD1K2QG8OdA3PZwnmbwBc6+8DLg4vJgDPb0uFSk8JhK0vHnwoDwL4hk92NuVvHVurTyGlIU86PrsPKEA9buKLZu8VoNkvGRJxTwiGwO8P7UmvCw417yppv07VIcJPKhBzLyrmay8Te/Au8pwmL18K/o8jJUKPaj197tzHeu7325SvBI2RLwz1ju7VfvVvKmdHD12jJG8wQRNu8gC7zucZLG7tJ4rPOZTkjwjdwa8Pl7svMbhnbyIkKM6BXASPY4YRLxmV5c76XLeO4J/8rsOaGI6FDg6PR3DMTyp/nq8+USJPIHnCTzsEUm8H9OCvAfY37xvkkS8Vw4fPA4WdjwLzc278nHou1mVZjp/Kmq8xDcvPbMexDwjZhc71+2nvH8XKj1ptd48DF81vUDPsbzH9Qk9ilAOPQ3e6ztImL08wacrvUP8CD32HZG8W2D5vPKv1rw6Px+7EPx1vEQ8tby3iqM8VlcaPQTAGbzhohm9vTHwPItzxzt+yd06wrnhvEYZ2zwDEhY4nvidPN4usTxnt/M7i02KvJP9xDvvUSq7EZYrPclqb7wuqoE8vH7VvDQ5BbyqgTY8xEmQvBriOLrPqfq8F8VfvENsCzyvBwe8P0eqPNCipzzeRTo8ZlC8Osshxryhga27QqwFPdgteTwbEYc8ebCDPFH5hby68Me7s6y3vNEOgjxZOYM82H+qPCpQXbt/dhy9As6bPPj3wrz3Tgm6F+DAvLJ80jwZlsA8WkPbPAOmcbxrOna7mVkfvSPGCr2xQMi8TSJHvI/4zbsDKRY8M4yWvI1dhrsRHzC8U3qPOghgLLynlfE7y5qnPCYo3zxMhuy8fnR5vIsS3jxyfSK8EV0JPXsDOL01m6W85MTCvJPeAL31gHk7FyrfPPz+X7wB0Z282v/nO9mhdbxCsok8PwiZPNM8szyyEaC8Q4a3PFmNUTqIoPY6mttJvENTFzxQBZk8XJnrOnMqLrwhPja8QyhXvANSKbzm7Gq8lh0dvA0FH7z1evo7LxgMvJj3cL3+A4e84yIwOyzxIDwODRk8yiWMOoBvpLqcCII9f8yFvBv837yoSaw7Y6ohvfHnxjxRwV48UG83vRBpdzz/syI9dXjhPOFnHjzihtE6kZyUvHFQNL0QPbG5Rb3LvArDED0bIMa6fZLMPHbezbu9qjE8UT+NO6bgy7ywP7g8U0bmvK7DAj39+gc8u4k3vF1LkLwkSYG7N4DVOyj23ryoB7Q77QuEvSJH8zqy8TY9FTuMuig7jLtzalc9+eVPvBbm7rpKSQI7LdnkvAL3Ujs3bF68MJG/OWgT5rwR3yw8VGeqPO+fAT3wx528352QPLSu9jy1Qo45Sg+Gu/JmAjyOYys9B7S5OXTn27siFHk8ebYFPQDM1TurK1M8WG0CPLhl7zs5kR+9Vgmlu/L/l7rP25K8hJkaPPTIVzyMeXm8LEqXPEYafLyaQog8YSnWuqSFMj1XOfY6vb8Avd4ZVr3zrIe8ztvUu7NAyztu01g7QFwnvMbnVby7RTe8nFy5u09rhTz1J+q8aJaEvJOl+zs5RPS7fRIQPUImKbxX/bM8y2nwPPouhTteNny7Fh4QvMfvlDxW71y86TEsPAfRCbp3C1K8lCGvvIdtnDsQWYs7rrVDPCq4bDwwnLO4ZWkivWy0yjy4Z468eb3QvKzXxjwOdgO8IeSSPLTk6jtrBRA8QTbUuzjjrLumOqs82qPUPFxdQzzlGLk8t7SmO2828zsj0Wu8oHYZPUwBHbzv2Go5Ojqtu2ktVLk6YFi8er34vLpenjzzlfg4rxtJPC3IibmxXgE8mAUgPVj+/bxeGLi8v/sLvYv5ejzKesG6mCgoPLlULL3ZIVA7eWkEvKKLgrvvGIE80oGDvI8AQrwF0zu87/mGvEZjkjwuztU8Fd/ku9Pbrbww6IQ8AoHwOzS2tTsy/BI9j8YpvSnFqLzQDss6iES+u/HzIzsw5/M8+VxbPOsRWzz0fo08gIEbPCu+Ubt6kYc5OlgPPSavTL2b2/+7GjAFPIG4qrxBfoK8/sIRPdUPFbx7HeQ7HtKEPJeepTzvIQ47dIKvvP+t4DsuC2I8Dc09u2cbPj2u6l68Y25fPEsEfjwCiRM8Wo3btxMWKzx37IS615AaO37Csrwd8+c8/Cc9vUSlNL2rWom8aEU2vNNTEDxKIkE9FSFhvAHsDL1KL6E8VcVmOkkV/TxCWOe4K/rvvF48L7wQnl49SdQpPD4kmzznrn26UtZEOz4Ooj0ClYg5Yu7xPNXFCLxrpjU8nK1CvTPUk7oJ0QC9jXa3vJ3lFrwxZ948HSu9OjNwtTwKaAS9XtcbPIPvBDzCboQ8996vuz65FD2BoKM8BU2+NxO797m29yO9zS48vDOthzxf2EM7VVURPXBZNrxLRYi8DQBeO7pyjjyPc708yJ0pO1R20TwVSii9UK7GPEE+/7m+M7Y8lHIFPLnBMbyXL0O8V26QvPmZAT14vQq9jMArOogz1DzPaQm7mX+wPILa5TzkLz+8dlkHOyxkljvUjTG8S65bvOzI0ry0F7q8ARdCPOXcqTyucSO9waHbuwb0obx86EW6MU3FvH/RN73Dmbs8JxlqPLFRC72WC5U7sI4GPeZI6rz+Vwu7rHKFO3+eT73DE7W86AhLvGPcvbuqaYC8cV6tvIYS7boqdcA69oOKPEpbozyc3c47g9NIuyesILxR4Rq8yDuJvN2QPD1WMBs8hpyQOlaDFD3csas7W18cPIbAgTww9pS6hU5EPNpQWL1e3Ri8e9O3O0huB70io7w7aYKUO5A5zbunahS8gPpUvMz4UDwZhvo7N4ynPHAcmzx0xmi6eUqIPFOFa7sPmwK77SXHvJCDLrylACw7MLvlvNCKyjyBKGu7lHOJPHBLz7pZRcc8FvMCPd0LFz33niS8HSD3u7pA0TuZYpK8072BPBCPwzucOmw8D4eKvL99TDs6JhC9YItpvBFj5Dy50Eg8RLUbvGhXzjtBho48zGkqvELtUjyFV308yLWyPNUa0LwEsMI8EggYPfztqrsed8Q8k5VJvS6MrTxFDIo7RrTrPD8RD7yTkxC8476bvDu7CbvnfuM7cAwRvaGEqDwiT7a6sYAwPP6OIb196Bu8zrAJvLUiML0N94Q8WCJouyzniTqwlKK8qpSDPF3g1Txtlw+80vyaPIWuijw7QRU8sb64vE2zUzyBlpo8bl8yO9sJwrwVHj08xcIQvIcvI7wfrwO9goT4uhKtpbwz+me8tYVlvPOxRLzVWwy8FeKdvHS0zrwwTsk7cPHyO6prwjpGe9Y7h2uyujGrcrzbfNk8bekKugBkELwSyQY8psPzPPPbArxaDy27R+XGO6SdzLy/YN+8gBeAvGEhkTxv4kk8jVafu73hkLk02Qi81W3oPKzMsjxReJm8QU4iPH22Bbw/oZK6brFkvJ7dqLxpejW8IBOKvU6OAj3b88+8si93PPZi+bt21yq8baI8PKuZP7s6vOu7g261PNt4hbtFMVq8MiOquhyjXrwITzs9hyfPvNm/Pbz4CBM8EOGCPMnXD7xA17Q8hHHqvKEqpzzcHZs771VovIJ+kLwSL488IBsHPcd5lzs5OqM8Bn0qvbm7iTyG3l28/iboOx1+yjvSe446DbJ5uoSfFbukFiu8iQK4vCNF5LyxURM8xDcuOqGNYjwx0oY8hx2oPJeq+DzSkyo1CyD7vBTjJzys5fG8yTUVPWcWATw0vYs89H6DPb/r2zwYFF68yO7Xu5ts7Tx6OJy6gVEavPxGzLmSUqi8hJsHvV3ypDxt2eY8wOnivHApyTwYAsm7RLy0PAEfFzxNs2U9s/4YPFHKzTsAQkA7U6k5uqgU8TyCTyU9c3XaOzDFTzvm0dg7hqi3u8K/X7zWC808fJZ0vAFcvbrtIRm8yTBoPICMm7w+Z9S6co6OvDIwG73YCA09EoHQu4ZiUzwAaqE7dL4yPfnaNzxMwy47u6tLvTZcqzwALdK8Dg8FPMvDcLtvZu47zKYHPCvj3DsscO47LCujPNhxUDwiiio9NjFoPIbm8DzjbAO8+WZ+PJbU+rwt18y8pntMvGOsnrwgaK+8dMmDvNnUnbzV0o86ltlsvPmZGTmNZgU8jjwQO9dgxTxQWRG8w33QPFkoPzx1EmO89PJDvNuBujwDj746A49zvHgnGbzO42u8xLffPEelyzz881+74giNPBR0qTwttpm8fIoEvAlYlruA/g48pw2hPJ5epDyTzOq7VnTlOzu9sTzOzlQ7RQwgPbMHTbwTdQQ9kULevMmsALzd4bE7C9DjO9FWjzwsEgK7rT6yO3tyyLu6zfk8jtFVPO1zAL3Xe/S7I5nLO+kThLx744+7KEUnvEGqxbs4gqu7s8JyvCW2GjwEscw8yWxrvNP9rTw84GE9u7gXPGiwMzzd+sK87fh9vNWgcjyUSN28pb0LPfYxqDzq/a08+3F1u4rB0zwMbmI7mpeMvLn0djzMZKm8Ua14O77zhTxLi928ugodvNDU+Lt5GqO8xzQRPIf6sjyp3qg8EmiYO1Murryaot+83V4BO0Yw4LoJ0aG8lTElPPeOojxsoCO9wMd1PJCnlzw6Sg29prJePJefUTyynJG8fc/XOqI45ju5lRA88cidPCc/njuaEoS7i6U+vTxLBT2A9JA7Ij/SvKK6j7z84HE8w4yoOmfZb7tE9gs80QX6PLIUgzvxLRi9sSbZvJfVRbz3EIm8s1VKu4679Lu7pm+8hsMJO1attjwxYGw85EGZPB48h7yeGDY8sZM+vGZaybzRGL48omXCOMDKEzxXlb+7f1UJvaokzDwKL1c84GwrPaVYB70vSCK8KelavYKUD70mCVo6aOYHvKVgPjwRdu68RyLOOwFpvzzEBhe9Gp2PO+DC+7xrm7A65RbwuwAao7tBHYE8QmBxvPCxKryl0Cs7joIlOxyPPjyik4W82dK5PBKTBbylCK+7rsktPIF3lT05gVq8oxI7vLDsdDwcTW67HoQJPci4Grw+UYy88nGtPALAxzwtXi87M/UzO1BgdTwWX0y8U/QBvOBulrzoQ5U8Ruv5vJztODzMPhk6FscYveVsbzws9YU8xpsAPHUSy7qDtiO8sBCHvMX90zx8kQg9mpDOvOm/cTy+lPy7GPpBvDo2Bj0DeHQ77qOgO7AR/rw1hUM8yJgpu16bzrxGV4Q8jf8lvLs46Lzm8oG8AI+ivJCxCT3crAq9SOsNvQ5SujsIFGe8ktYVvOSblruOn2G8OCD0OjvYMD0mj4q8nm4APFGCfrzN3PY8qKWbvDyvFj0wbQ67InxEPAWNSLuaGiU8ILkVveyFyrztmkC8/SnQO2PjBzzyroo7zmS+PHfItTyZK826ulFPOzPphbyZ2vg60bLcO51cyzxY1YE8IuzmPGkcpruiBqi7MMRHu0KjQb2ce8A8pylEO46+yLw3Fhc7WvGnPKlHz7sPr3W8QCjuu78RbTwaNXK7Wq6JPM8WTrwt3rc8o9AXPUfNqLwvZFi8YUJ7vIrOVLxnMSu9S1vCvKfapzvZ+Jc8zFVku0ArgbvT5KI8eXnJPG4fT7zWP0+8jeTZu6wewTzk+ZS5sJzSO+kjz7ga8V+9lHQ1PHR+tLw950m8xZqlOqoJqLsk2wC7e7MOvOfSszyQaqa8XTBIPX6dnbx627K8Rv0VvEKxRrzDBae8/yS8PEX9ojwjasK7M0BDO3Q4ebpGyuW8ftpju3lBqrwEnnO8yQomu+aScLzOMdO879DLu0aqQTw6TIe8htnNuu38cTkRK8M8fBRIO+axtDx86LA8PZ3bupuqizyFKTY8eBE1vGnBWbzXX1A8M+cqPDRiZztk3bQ8EYymPJp87ryxYOg8/Us9u2/Ufzsisk08QQ73O4Mwqzkeah48dpd9vPsSHDzzO6Q8aBpbu2MH/zx3eY88tuV0vFvhqjuEih08Mi2CvC4+kzudiYy8WroRvXlQBzwtx6c8V7enu/8lJT0gn/o81LGuPMSw3rsTAGk7vLZdvFjgMT2Gdcq8/+2wu4aggbvEHTy8MwuCPHKQHzwZK5a8UMwkvQg/xDusd/4706rGPI0s9DxbDqc7XFUGvQAtKjqs7z87YX6LvOeEnDy//Ek8ERLivOLOXjydNQW9yK2bOlUiqrzgau08/7OVPP3lPLrLizu7N73QuykJXjsL8Ig8EWSsO5bSnrysHzq8uXNtPOiLbDtlfVE87vNQPH/lkrsLWCi9doz2u1TlrjyNzk+9T2wLPAfvRTw0/d+8/qL8vOz92rzihDk97oYjvM2L3Ly1W1K8TKkCvNYOfDxcL0S83+z1PPD29zpruuc8rUnZPDwWLLwZCUW8E4/qvCHw0rscGPI715a2uxqV5LyqfX8669YnOqzB9jwu11a81mjHO9SCLzsj3BG9FwmPuzIzx7xsgis9bT4LPKMVRLtOAzU8a5jPuyMmLruyIzK9MeWuvIOyorypNwg8eFyUu1Ize7s5mpA8pSsZPV4wa7xOrPy6KgApu/pGrrvZoPy7VH6rPNdGm7zx3CG8SMLguBAQybs804y8oKoAPRTgPrtT7C+9IGTtuvH9lLvNytM5acsMO9GCirm3SSi8PCTRvGRHMz1IYA48qnufuhoyCr3XKNq85k/xPGY7ubxmLIY8V/U9PACS2rwqmGw7lVw4PYeOtLyWRMu8M3QHOwCLGLxoITu9BX4tPDobQzuHSUg8iamuOgFLAj04JE273G7uPL8UiTy3fxe8hXaEvD+ZqLz25iu9jc8QvN/XrTwJccW8tdzAPEkWsDoUsQC9q69tvGN8ebwR0WE8ZOXxOuG6v7sr3PC81C5cvCbW/zuQis88XkIMPCIG0bpmu888RAgBvO1irTzRr98836nyvF5Nm7zcOOg71N9JPK6M/zyFefQ7T0/XPIWcxjzd8b28UYWePBwkfzxW70G8PaGiPNg/2LzKah+9pYy5u1BoE7wOeJ06j3fTuVR+vjy4zee7RhYwvKQTzLnqMZi6M0I2PIsHBz3IlcA8/7aQu5c4wbteeQW72CZOuhFngrzRxES8Zki6PAI2qTsxg/q7+BW+u6pOBT0prI88/YWoO5luFb1+k/y7JIQUuralGbsAaby82zg1PRBxL7tMd4U8bJypOV9doDxUUiE7QRgXPX5BpToCd608CfGYO5/2qDzbGKi8IdssO2aL5DvDBKq7UBQAvU5dSrwdwPs5pqqzPC7AVztQHtQ82yYXPAzsizzuKRi9IE5NOwviD7xPUPa8x3K6OwOAXzxXLbe8m9GJOxBnxrwqOK08h6SQvHPQ9bxM5B66Ss2bvPvZBz2eybg85t6cvH7irLznARS5e/OfurXw8jzX/Gq6QxPBvAboObwC2Em76oYRPPNvm7yR+0Y8RoPmvFzETzxk/yG8Ha2Ku63cIr2n6j68eb2ZvGXBoLwtGcW7VFWaOxU+N7zjjVc7FJ6tOoJLhjs0GCi98+kYvWvUyzvyoNG8JTYyPJxzC70Fajs9QoayvEeQgzyQDyY7rLHWvMFKJbvPIe28Oa3rPDl4wrsLUJU84v0Uu+hyGTzyz+07Oy/4u3znDbvP0P27qRAJO20q2zvfLgK8W/yau+PBFbue7+A6yjWAvInwPrxCDcu8/OD+uYLrkLtCbwO88TQmPOK4/DviKkW6PZUivBvLLDzLA3q8KKHXugl+oDw/EkW8cxa2PBKU7TyDH+I7hULtu5Yh6ztOibK7+LeZPOCV+7qulyE8ypkSPRYQkjvOxOQ7jU9ZPBtAD7vdwJI6JWaRvBWetTyb6KI8nD1MPKaHKrtTKiI7OWEIvDigoDwgLaq8/AFFu7R8Zzp/A4e71fwLvXfuCDyvEvS7i3AGPLeK67tOExK8t+Mtvd7fFDu57vw8pjAyvD0AI7yQrrY7Ijqbu3LYOjyth/67dZLjPN1rrDz4yJq8ubzNvEEkIzwB4HQ8Vb0lO/viijzgbn+6U2OKOk1X2LwDrwI8ZwhpPEoz07tE51u8suXOu6zyTLxttO+6BcB1POKpXjwFZJW7kMwqPBqZAz2A6wW8abu3PLjPBb1cbTi93liNPOb1/rnCbKQ4G2Crusp+vLsCGp+8qqUwvU/oR7zpYE28nyOxOtMNxTxXcUA8DWe8O9+lEL2m1x+8VVqGOkfE3Tt4kUq8o58WvbXA0rwpU0O8qFKVu9PVKj11Uqe8iq7NPMPH1TuGw0G8Ks5yPJCU7LsoIty8zKPGPKUFuTyhPs28y9KNvOmf9Lyai3G905vwPDtfzTpAMoy7d+xMPKlRrzxdpbe8GQoCPC8PJb1EQM675omEPFLA7Lq88468hmwBvc+g8TtYIaE8kTngOroj1Du6VBo9SBpZPJrAGzleyGW8y5IHOxX41jyYn1G7mQCPuup4YryyeD471dPBO3mHIL3LVbO8uk4/vBdlUL1KL9Q8JMemvMNTFL3zB0y8O0HkPDG50Lo+Svy7yVf/O3FyW7q/6ys8GoCrPB7RCTthJgw8fFAUvFU/nTx3Ey88RdOJvG1Zebzceau7CQgAuxtyITyYfrA8Lb7/PNlleLx2Zga9XmdvPFCzLLu6yxM7Jv/6u298QDvUz5c7FWP6PJdEQTkv0Ku8rAqEPM8m1jsw5wm9YzKrPOLvcLzKxZs8GATGvFhNfDwu4J+8NklevMopPT0/ldo78aT1uz43oLxtERO9viRKPPxkJT2E8Qa9dMdIPAZJr7s8+nY8vZGtvErApDymIIs7+WkavevOBztMu4G5xmvoO32qCjuCCae8INNqvMHF77smOTa8jWzlvCNEKLzOMT68XYp+PBmwkTwbWPm7Rw65vM1ANTz1EJ27+YjYusYBg7y69Bw8lgTnOwQ1VTw1RyQ8kpaNPDSqAT2t+Ry8mj77PB2hXjxSC9y8lgWDu7/FFj1WqaK8OMcRPX7J4juqfn08dcQQPSmPKLxjvrW7w+1aO0tI5bynfpK8MUMiPIurpDxByxy7VP4IvfAEwDpbpQS9kOgPPBdrljw7J008xfyivPlyk7qx3z88x622PO55yzyRIaO70+mAPLMhFb1ZXZK8eGFsvBHznbrXRb66DF8AvIqk5Dwhfp28KksWPGtbmDz8Eu+6xRNVPNjVnbvQyyw7INgIu5sOuDzQ7B67uZePPI83lzxvODS8eJUNPblZFDwOK+m7sbC7PPcTGL05tPw6GMcEO9BfbryvYdo8n64dPTCcSbzZ1UO8MhomPJWcWLyW0mc8K2WUvEblJry4XYy8yOUMvcnUJL0p/6a8H0GTuyyHvLlZLrG7IxMAPEBiurv8uaQ7atZZvHq2tLw0K2c8UqMMvIy5E7xBwLm4BwoSPYKVwrwjB1c8uYyiPMg6Y7wJK+o4EjeSvCv3SD0r4dM8KoSeux6/vLwQhRY91SOHvBcsjLvAsac8InARvPhBirwohBm9kknjuz/zE71zhuU6lOprvFza5zyGL508mOqsur2KDz2LYGu9uBYVvJb1Zbs2way8dgnQO/JO07xTBb27w7kJPCekljwiL6k5r/YWvcTl4jmz5xw82JH7O/nZwrxLFWU7BrioO8i3RLrGm3W6e88QvEX6ozxQ2q46RCKtPFphsLwOFSi8vFsKO5jn3LwBSQQ9w1bTPJK82LxPtt06GJPMu1cRBz33ElA6u+OUuwFtBb3U02s8M6JXO8cxSTyIBg68wlPovJE5KbztyKi732IOvcDDJrwC88W5H1B5vJOnmzr5Ha08IFEVuzVgtbtHpKg8hO5mu95aBbxVhVG7UmCFPCl98jwNhuc7dmzbPOWXsDyWgGS7EvsPvILMKr3AJNq6vgmLO6J3GryjVfE8i51QPAjpAT2Jh4W83qdzO7wW0bv1eFW7wGKfO8aQnzib3Tu8IQzhu7dh2bxnLXy8/umwvLZJgjkCFa45XuxIPMaTELzW44s8PvpWvODEDDyGtpA7kpSPvPXplbtHaJ87OrfLvOBLoDuQ6c47D3mauxm6ersiYBG8FicvvHwnGD2E3pi8bGq8O/U6rry03PQ8kmk7O5x5gTy0VmS7JrnFvPmL5rx37cW65W88u//hpTy0i448sKEqvYk2XbyEZfU7aQ3QPKIJFTy+HoG8SLDhvB8Fhjwm2Bm8xIhuvC9IuDzNZYY7HjEdvOJhhrwLHEg8B9/HvHWqqTzSffY7Pe/zu7O2g7wql508IHvEvNc/8Dth0lk8OY7OO7z7EDz4e8u8fFXGOym0kLs7rPK8mRYwvJghlDqrYHm7stm7PH4zzDpxKSs8THTqvNP/mzt9JiK6liNyPAlJiTwulkK50U+yPMskUzwiU0m7FeWwOySRZLytXF48jc7yPI6Np7zGs4a850SFvLPLCLzo6fK7nZtqPDh+p7zc/Zi7YcwhPINRe7xrsi89hb0RvDBC2zwgaxw98fhuOwCnvDwJZKe8Ch/yvEUmyjyDwly8ImLquH5AgTyJIw092iDtOq6/jjxB4GQ8yS0PPRMUQbwNEfM8Vdg7PAPdfjwXv8U7V5LFPCAfJ7zIrY+7mBhVPOUu3Ds2veg7MHfOvFg+FjxUDja7rT8YvJckMzxsIdk7ZMcGvf1a1LwGYAW8S1lDvJyXPTxRLbA7afA4vU9xdbv+fcs7wrP9O7vMmrmXGZk81SSAvCm33bsrwkq7aQymvAYMGjx/BdC8t/yeO+BUK7wReAI89pUwvY5BJ7seHQG9epsIPZhaqLxAX6w6l15UPA6SxjyjY0C8JrsSvVeLoDzQO8G8pMMaPIKEBL0zQuQ8FKjNOsfK1Dz6M4w8wpUIPazvmLrWW6Y8513Iu4kXR7xQ1008HuMvvNz/ZDz6UPE7vQRfuzkSGrzRKs08IHu5OjLZKjyhDO88zTUjvFhOjDqB4DO7estSO6IaXTpG+dK5dLQCPHOeVzr8efm7I6OgvGdoJTzWJSo73noEvOZKdzw0MpS8/Fq2PDjsIbxtexm7S1Glu9xsoLx3SUA7nFOzvNAjcjzxi667rf4bPTorKDx+ENs8RgtFOytFQ7syfHi8snI+PJRSnjz5a++8/NIKPb+xCDwj2zE8Dyeyu4qgdjvTNvM7qYPDvMnvBD2UEH08keMrPMmgh7rxZjm7xgo9PCl/uzuRHk+81Lbxu7rvDLzugai7J7YSvMmbPTySfx4828P9u+BAR7v9iZe7aNYHuxEYRzyGoHc7gOdxu2lBs7x+IJK7JJPkvIBE0Twd4Rq8cY1Yu7HZkru9ZZ68RxVEvMIZyjvlERC5Y34svKLnc7zKQBM8iuzVPApAo7uBQrq7W/NavBncsLuvNCm8/368vALJIbzmk905foeOvHffALzfrdC8x+78u9kUAzt7/wc7bEn3u1lTB70/CUE7riqNvA== + index: 0 + object: embedding + model: qwen3-embedding:4b + object: list + usage: + prompt_tokens: 4 + total_tokens: 4 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_s3_source/test_create_document_from_s3_etag_changed_md5_same_skips_rechunk.yaml b/tests/cassettes/test_s3_source/test_create_document_from_s3_etag_changed_md5_same_skips_rechunk.yaml new file mode 100644 index 00000000..96a6cc99 --- /dev/null +++ b/tests/cassettes/test_s3_source/test_create_document_from_s3_etag_changed_md5_same_skips_rechunk.yaml @@ -0,0 +1,42 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '87' + content-type: + - application/json + host: + - localhost:11434 + method: POST + parsed_body: + encoding_format: base64 + input: + - S3 hosted content + model: qwen3-embedding:4b + uri: http://localhost:11434/v1/embeddings + response: + headers: + content-type: + - application/json + transfer-encoding: + - chunked + parsed_body: + data: + - embedding: jIfcuDGtJDtS1Q49KmHyOQbTEbpKpkk9+JktPR4H8bw8H5E8G0qevCCB7DwTB1w9BeoVO+MofrtFEPe8hw+pOnVmu7wXz1O9jZrrOR4aqrvnx268x9Tru30nDj2waz297gtOvdd4h7zI8aS8PcRcvW7RRjxGviS8vSM9PBewWr0h0v48z0iru10mTjtM+5e8WJQouvZnIbxUC++7MXipPDdC4DxGwzk8D4lQPGGrDrtt8yy9yKvkOUgJDjxyaDW9mUlCPHojnbwhqOY7f6zoO1OUv7wca6q8XcsCPQ2I/zq13KM8d4vOugSTOb2Z3z29j1mFvHdSmbuCqYu6a/oIvDlaubuJGsO8v6RbvA1cD70eED08pwasO4u4Cj0FlhI8a/civEqoMD2Z0b47tMfpvOxKMbwb+ew8QWuluz9OMjzfAJa7AjO9ut+ILjzEenU7M5YpPH8kxru90a07CWCBOxpOJ71nCZC6rXszPD1hvTyowhQ8oT2nOzboBTtN48M76Pd9vLJYvbzrHhu6uQY6PBoegTxKdk87WVNAPBmhMbyWHwo8TVeqvHsIIjvQz686U52kPJrPkjtDWhG763fLukhpLzvfmdE7Ew8bvb8VY7w+PK+8HVLhO/xFcDup1DU9G+96vIGViDxbBjw87f8hPAKOP7sQBGK8Fs4VPBX0ArzVz4G8Bp+WPGUE4Tw5uOK61p0tPFb3/bwuw1q7pqEmu9oJ/bu0GG+8jZ7/u4q/QTyWATa8uriUOxF/UTyW+ZK7vtORvPikI73hTlY7v1E2PYDnnjlPOlM7o0IxPWb5pLv76Hy8BK/RO6Pysbtcnnw8yq5/vAqp4jwd5Z87zW1RPFFMsjsfH687jXNuvD86VTy8aA488j5VvAtyCry220A6BLgEvO6EfbxbkbY6d7cZvLMd0Ts51Mw2pXI0vDIkGDzoR4y8HAOzPKr/1LspCD89flZOvE6BTjy1nkk8Yrs/vPLvNjyps6o7jNmwO21pFjyWVQ087eaSPIAqVLzifqg8mMNuOwFmoztx1z67WQjevAJOwLwIE8G7J+TpvPD4qTx1AEY8AVFFOW2nzTzNS+Y66NaPurNsGrye0Fc7d6rAuw8q4Ts6c5O8hbPePEOISDxwIWG8g0nLOrL6GTwf3DI8D2mfvJX6H7wU+3c8jddmPHlFVDuprm0897h7uahQHjwTUA+9Qm17PAM+Q7tVqnW7N1jMOpIEDTz5ap08vBeRPLywCjt8yNq8uNV9O2aGk7zNKjq9YqlWu28Z5js1zSo8xChKPIT4HL1vgUM72b0BPMb2pjtSf9S8BDiePLvgTLwMO2m8yRJ+vBDLJDxKCwm8Yrxbu46VV7yOIdK8Vo+dN15hurrT9Z48cBVUu1hoWTxC7GK5qNGuu+tLzrsPCmC77MGJvGUgQ72mcxS8oTBQPIycoDwGaX28VIiBPbA4p7uO7K06u9rLOge8Sb0ykgG9RYC2O0qcYDxgR9A8xVFLPEPslLwi92Q8sHwfvC8xuDpIUHW8Swm9OvZUzDxSAxG8GkFsvHk9FzxdWME8ZfIyvS9vfTxuUZS8XX0TvYDJvTiNk7w88i0tvH1uMrw+7Ao6RGUuvLQcN7wHwCG8WqAtvOVeVTzFpEe7vwiHOnExZDuOgK683cUKvCfcq7tmzxs7uLQlO1S+UDuHFhs8JNA1vTctnzy4xgC7e98cvbaNCDyrrSK7MUE3vWxbSDwS9A+9Cj/guy5YjLu7QYy67Q0IvNf8Hb3QFMY7KEAaPPBWCz3CQdW8WCQjO9sZKjwu69O7/G2ePLeMgj0A9g495+QQvCMarju0fu87vx+kvFfuqTxcp/O8VUOnO/Qtfzz0VTA8vz8lPF6TyjxFVjS9sJLbvB7WlLzvrSa8UHujOx8dKjx0IxA7Y6LgO6x5kDwgXL26PlYVvNn1q7s0yhU8Mj+8uaR4JL3lFWC6lbwpvBFWQzztOke7IMIFvedObDt4qjQ8DRMHPX/PrjxnhQM85MI0u0eQYbwoNL+7xBlNuym5Yjv9/Bu7f4taPFXrrrxqeHI8mJ2fPLtj6TvW3p86gT84O4301DsEHas7fOqvvI57Qbw0pKk7ddKtOxslDLyqL5U8fJpBPDWARTzEmHA8UBuuu3x2KzyglVc7A+0PvSqmqDz80wO8sxGOvEbEkjvdKik9QoXSPAtMVLwp0MO7DIU/PIT5FLgNib28gMNXvQmkjDxcQKc8c2qpvK5/Bbz1gzK8RKaNuywnqLwaCe28dVszOphGgzsvyKY8xSuSvO0EjzyUeay71PfGO5x7grzPnMU8fRcUPW7+5Dw4Kjc7UlCsPKFySDx5luu7FS2UvEv1qbsk0oa7cR7TvGOfxzvNp5U8SFLavK44sDyul/U8r48NPX0ZITv+BDG9ItQnvH7EjbwIr1I8ZSHevF8cyrz5nsE81lDrPFsJnLzCmYO8Ib5ruxQ9h72Q3dU7F0hSPT/Czbw0FBY6nPCOPLTpz7xKYNo7tgcVPY3xBj1oEb28hqItvSXM57sSoYA7bvWFvBonpTxZH0C830jMPCgkGz0z8Z+8Oqa9PL00ajxUm8I8BNFxPdAwOjx5nC89ARO6O1MT6jw40+I8FoghPEvNybuoCkg9GYiLvDKHWDx7nS28GeqDvLDKlzwoQ628zYCvO02psDvxAn47OMvINyXw07tR/ha8466Cu01dSbzsFFC8pQ0aPBX+Orx6DLk8FitRPI87Drtg0V+9eflmvZzHCjxmJne6fcQxvHZTD7xc/jW80+oMvWzEjbxXLky8r8cXPZxnDTykjaS8T1qfPLduKzwGE/s8/1wTvRBy6jxaCpK8kDcxvIM1Szw4jgu8Gj6TvOC7urzYur28vT9/PFN/ojxlgxA8Ks00PEiY0DtyUug8Cj6ou4YqebwvE3+82LfjvIvHQTzHrCW7U6iBO2yNWzzFOI06Mhg4PcYFl7zrwqU8NabxPIUi2bwqumw5RSdNOckhsTvJx1q8JbZpvBSCPDwwj8489cEnPJziHbrhJBC8QIFnPFqG4ryPbsy8Eo0POgYbgjx+78M7v30jugimhjzcKYK9CVfKvF4fkTucMdW8zCxTvJG2lTwuj7U8zmu5O/xFDr1JU7+8aEtoO93ZFDwk0iU85i2LvHyiA7w3VQW9/bSCO/Jw4btR7fE6jpcYPJ5ECbwnIr+8Cn8HvdIRAbqZ54G7trXdPMuTDr2axrK8MjQZPZl687viHdy7YcI6PDGWqTxEA+G88GYBvAqv7bqyJ4g8P53KvDR3WjxxjpE8SSaAvcBZMrxkYBO9bsvwOyMayzzv0AK8R+8hPMNkwDoWmyC9tArGO239iL3NyWW8sSySvIN/N7wZKKm7kYbdu6tQvTt/Dvs8jZ09vBHcgzzhtLO8NWeDvekrDz3qkmu8ReTHvMxHTTwtQro8EIyfPIvz8br8rH88kjMXud73LL3laYi84roKvfoxnzyqE+i8U6XJPC2jhDo0v4O8ftK2O6i/PL0K+JI8ibqivHEFLbxNgZo8Bs3nPIUa6rwZT4u6E4SzPLH+bDwFPr682rlAvSxV+LypKmo9XDQzPPAYGTzId+M8KGohOwPYd7sDVgU9P10RveAx2jtrIzs8gVW3PM1kEz3fL8k4r4zbO7aRwzy1ncW7rAnnOy4M0joQdDi83BK3PN6hTTxSexE9nueRudwdnTyCNwQ9DzBCPSZ/mTtUDTE8RRKBu6LqIDs1Seq85CoMvaiQZbye5Pu62rq2PDPyljxZIIU8tlKbPEKa/zuR7q877vcDPIyiMz2SbTi8ZxeGvKjySL1ajK68W5Xvu63cHbtCEga7Y4a/u2flwrw8FDS83WQkPDshPjw/K3w7BVgyu6JIiLxlU1Q8O/amOrETiDznXWQ7cx0YPT20DD23dY68pmFkvLnOmzwzJaO7RTJOvPfZrLxpgn+8z4tlvKW0FTxc2tU7gXe/Oxj2SjzBk4C8qdutuXN8AT1brw084NuAvFU+IzxHLZc8W5WOPCFI1TxgZac8gsoIO3OOr7xAjOc8ECcZO66s7DysuOs8E1GUPEIbAjxEmfa8kFkQPNQ44rvCaBq9/fTuOfuQzjyLJZ28TgddvElMEDzBvRu8FEVHPR3xKrv3u4e8yRe/PI63HjwToMC86giSvNeDTzzRWfC7jcYXO1A52rs1Vg49USNMO3JtLjyACHE8oXEpvYE2abwQC02817kevSLP1zyL+d0724N6vGnCu7uHLDc8cygSvEp167vFO4y8COM2vQ3MyDvaj6m6np4nvJCxMrwm7AE9arGCPFGvBT3s26U8QDRtu5ZCJryhOL87nXKBPIE5xLyp0Gq8uHPfurFH0rtUlcq8OqXRO4qBm7veJiU8W/aOO7qx4jYXiZk8a7WLu7YWDLzdSaC84U0zPIzceDxUjEu8t8+GPBSQzzzZCBu8HZRqPLo5dTwhioI8Tj4gPI4S0Dy96uW6xKWAvHJ8pTuJ7mi8wo2MOur5ILyDdEA9H38qvYAZcLzYpiE8AJqJvLrJabtHUlo8IWzGPEXiqjzzKHU93a65PMT0dzyQgxO8DPDJOzntqDwljHE84Zz1PPiiHbx/9ak8CNvmvCCHTLx9Lxq8Totju2VNfrxR5u685/hRvdLHZjsastK8CfaXPKHRTTxxJ9s7BQGRPBo6uDwkeqY7gLqlvEaYZDydO2E8iH3yu6JqDT3Czq273X4YPRmHdbyzFg68p0nGO963vbqV1Tm8cdkdOu84cTsTNWK8ZvQmPYSCwTwuECS7+quwO92FDjzoU4U7x8l3vCdoNTykiHK8hHcAPTQLeTwKzB69XYziO6tf0bquyLi7ZQp1vAMIRrpvAFU877mAPAxl2jrDY9S8NMUgPGn06bl+6ci8C/FKuzIlfrzvfiA9ofj3vEUocLxtry88c+dEOx+jDr3ccHc7VMIfPZ4UMrz2Ng28E3wyvBtW6Lw+o+A6003zu+wMCbw3Fzy7mQtXvL9/1LzAq4k8GDimvHNX7jskhg48QVRkvLphRzwFaV+8RMVpvE5JND16rly8KIZ9PHWvFj0Qr2c8X/b+PLyX1jz1I8s7UEIYvfVusrxj0Ie8WTOBO0VCGLzz83e8usNLPCXR3TqZ16M6u2kbvHEAPro6Fdu75XpkPLf9ET2+s2C8h62hPLBbSLzRDA48CeIEuyupbbxkx4G7ERYouZe4Tbz8ViI7+m8yu2nmvTsInJ08TSfyPLLzRDxlL/i7NYofPEJwNDwCJIQ6Zo6QPGXRU70q44m8MCqHvEyTuzwnLwu9YNv0O/SUvzzFwwo72SHtPM9HIbu6Mro8apBPvEO60TrvIZQ79zpavAQUDr3iKtS6yPXGO3YDFr2TVJS86BtTvX/o3TxBheG8GoXFPEJxbrwIB9O8UgrdvBPbjbzhjCu6ZIyHvHu2/zqkYdc80babO3asjTsHlk48nJX/vGX0Krwor528y1s2vXYmajxdcQu8Gy4yPRt0MjxS+HS8r4E1PERw5Dtoh0Y8roSuvDNXUrsFdv88L5hVvHqNAb2ilwM8N7poPKd237wlGf+7NBj4O0WwWbtm+5y8gZ9tvGpNZDxwb5A82a8ovBdQhLuiloc8lBQrPMuMvzwg51k8PxxHvJNAs7ytzU09ZVtIPEhuBr3E9Js8LPbPPCfrM7yTrjA7JjWGPGLRFr2jfIc8673qvBxryDi5ywq8yEL5vKdq8TwBRwG8rbSXPChBiTy2e8k7UrhMOgrh2Lx5nqO7kLwBvQ8VubzpjvK8XOjtvMGeV7yeupO8ztqzvJfdqzwuBUS9NqsVOxUS2jshYha9rHAtPMqiarxZn4M7fW3nvD8ekbz6hOU82e/ovH+hEr0al8y8V2kYPTlgkryE7zW8MZkJOytwLLzBRjq8ZiCPvHDKZrzMOx87tinSPPvmJbtcTAs9ydLMvM/Xtzvzt3g7KUqyPAAZxDxNLHO8kQtlvOwcwTwDD7G8yIeeu6FHCrsYuz48p1Mnu/yrm7vYTo268mtIPBPK1DvcJIO8TzaOPIml8Dt4Zgg8J/XiPPWuKLwgsIU7wcYcPcOXKTy9iLa8ltGLuw5OizuoVjY8L0GLvINOozxM3YC8sGxsvPExTTwAEqE8vD+RvHC2HD36GFo73csZPf/x7LxCxJ887AO0vEOONbsLEb078kRFPOQ9IT1BZzc9xtIyPC+M9jwjccE7g2vJO39Ahzw6YVK8T5hXvcibKj255i+8cW5+vL14oLhpjtS4s2KvvOhgDb1VegA82jm3O3odezglHlo8ar8lPeCKj7wwdxE9UHl2vK1SzjwBOYQ8+q0WPSvcerzCt5S8e/ukPJmJNTyiT6G6gtlpPLsSYbz1PD88Xr/rvG43ADyUwyG8nFfZPFXegTtaqvA7KCubvDbkFL2KcbO89fEAvZ3D1rx+0R07EcWAvJA1aLwt9t88TfKVPHnVgjxwJ4k8H2TqPLsCQTw8pTO7n2aWORWymzyJ77i8NsKBOl9SRzzHK2C84ZcJPFAsWbscwgm6OWs1OwG86zvS/OO8ATKEuzyjk7zuiMI8kQD/PBGP+LviRfA8g3xAO/iSzTtKDzQ9TdERPGnrcryNgSK7frzcvDhh7ry3otY8P0YfO5ZQoTvdX488WwiPPM2RIDxfJpE9fCMJPKxaNbzt9CC83nNXvEMnQrsdYzK9UXzdu/PGoLzNssy83AucvKxsTDyHAJ08FAq/vDAORbovkug8++4HuVHJj7yPFLY5hNchvCm0BD1OU4u7ChpiPB+SALyidRU9z9rDPNu2ADw1SP88wQSCO8y2ijx7WRS8Za9TO2GMhLw59oG8C+mMvB1xqbzKMBu8/TXrPDRFcrrZncq6fQZ5OzcR0TuBm8u7Q6kmuvXq3bvO8PK8ulYHvGc9ZzvvNey8EHQau/Wyibt8y8G8AqdRPQnJwrzcIji8lG6Bu+uCcryNWYa8zxW6PKQoXbx/Kpo8NyVCvZZDHDzFm8i8u7ulvDT6jTxexzm8XlhAvCZyQDwhoq27zpC7POmbqbtEgjG8t3BRvCzbP7xM5cO8XccNvT2KB72cHQg8o4bcPKhnQbssUCa8mUZLPHeuIrz8S1G8xOOPvK61lLzlX7s8zGU0vA9r4DzbClM8o/6fuyLGXzruOX8851dcPGrRlbyUrXE5eXRlvLRtBLw5YNy7I147PPb/sLxI2Hq72kgsPGzZnTy+Is68PPhxOy3WUbz4afQ7yf0ePTCMxzyoHX08hWKeu9inNLwdAiS81dPXPLQqHz0HriS9gJP0PHPoZTxRaoE83BIKPWpU/DyMQ8o8R5/mvORlRDxg1nA7OswdPYNhBby6xJ+7Rj0xPalO3TuhaMe85H33OzYKZLzgDJC8c1YtvFJpnrztsbU8+gKNvDZ+3jzang48tNxevfGokzw0HBM86arIu5kB4bsZ3My87JoLvUNCDT14fjW8+FCQvKwrMD0UmGq8pYruuzJYCD0Dvcg8GmuWvB2vgLyQVqU85smGPKQ+ibsmHAQ9d+xvvOK867xSsQC9eHnSOjqBaD2nVt28jo2ovK9CDjtLpJm8/PN/OtMyFbwsVDe8k1GTvKKmtTxqjxs8RfKjvO0Ljrxg/WU8TK3CvNZYVT2IEKg8ZbKFvH20wrwyibK7YfLavMNOjbxsxO27KX1tvA/3+ztlqzc8tAKPPD38SD2+S804nMePvFAIpjtjJgI5klKbvOA6RjuNmsS7prAkO6Z7Ur1RfbQ7kHoKPJYMX7sEO9G7QSdpvFbeiryVIYS80EygOnNDvDux4vy8BDHpupEwhTy25IQ7/6bdPNsekDrpI6w7gXcFPMbWl7xX/5y8KZ5HPGSMErsqGc28GNsPvOVbgjxhmWc8F6uzO4YU9juviVk7nQ3RO4IB8Ltc56s8iwY/vGB0ejxGI+Q7KKwAPe1bqbpzcVC9qKyKOqoPrjvD1Bm9uHeNuzTBkTszY4k6bvHwvKZlqDyV/z68Q5J/PHze77vQQO67mMTCvLpmMrxz9t+8xF4mO9E3IbxLZCU8XvcrPFm5oDz6hL+8+6cUPRr/ULzda4s8Syeeuz0MpDtMGEu8t+WlPEDUaTzpEHG8DwiMPOhAlTy1ZP87Xk2ivOcb0jvJa428UKSUvKGjZLxA+iu8NjX6OwHQ8rxTtoq6CqLkO6A7jDwKZjY9poqXO+OZE70kmRU8FtvjvHok/rwJpjA8DmWWvJj7dbxWml489BlGOy35P7zUg6k8Xo/LPN4PNLwI/so8Q6SSvMBMpTyxo7I898DdO6eMiDunBVu83gTPvG572zxfUmw8mCGxvKI5rDyqZYM8FdWXu90UKbz416g7aEhUOsM1Bz0hwjC8AqqyPKttdTwZ/eG89uY2PFZ9xzwcmYC8pyMtPLEVYr0JEZo7zp29PNUN+jvvX766tw0NPHypXjwI/Tq8awq+OwYWg7uMXhm88PkRvN9oh7tOFKK8SNCLPO5+KrsTw5260nllPMJzALxwrNG7gbC0OamalbqAjWA86xuFPMyXF7z5ApS8s/2gvPGsuzwtOBY9OPZXPDIizTzTHmo8w36uOyoFJz0cwgq9I6CPvFsjKrueIwq9qew1vPAHPTx32Vc8dFerunDTITzEYLW5HhjXO7rz5DwvuGW8NSNvvDrFgzzgIRs8SAgcPDHTYLyMQd26UqNMvWpuZjrizf484AmrPFXuXLyfvW47zAeLvPXAJzyUrrm8Y6sMPLlWv7oGeg+9gjSHuTZEnLxXdek8qhAqOyfvrLultEU8P4nlOpnBVDxhWra8PI6VvLjMJzwm0PQ7vKaOuol9uTy3ajs8R4gYPbyT0LsI5hI87jk9Opa1vzu6XxS8Ai7NuzOMkTwqvQm6p/LeO/2qsTotLk68ItM5PDxVPTyr5TO9BbpZPNG4CbyWP5C7TeWhugQzxjzJO6c8JkA5vSjdZrzFx6k8lEKNvNsrjryvxQW9fS+VPBaj6rzPNoy79VVpuidxC73Xjnm8paEbPeNcnLw5jR298zOOu9xL0zqE27+6YyA6PGkpxzuXgfE8Oiweu9pYFD3muji7cX5xu7sAdTzZnsm849/pvOOmOTuM6JO8GSCEOx7VVLsiife8l9dbPODYoDwC7we9g6AIvJ58qbyJgGI8EsGVPAbtXLxz5Ic7cxD/vDGXbjxVfhY9OOd4vF0/y7uej8M8nl1NvNyhJz2vEPI8sVthvNMUnbzROgY9xVJ0vMmPNT3bw+Q8zzHAu7NaLzx3xLW7R1wjPCsVDT2xL6S8c5AAPJ4fYLw+ApC8o4kPvNUYybwOGd87f3WKu6VWjDw4oXi8YhiEO6ati7z2hkc8AvNdu/p/DjwlIUg7yQg5OT85J7y6U6E7VmFnvIPrajwUaO28CNI1PZeXoDurMZI8XY0JvUr7frzRn5O7FpCWvPQ2s7zgcoO8E9vDu1uTv7yO9TK9Y4f4PGTKkryhnRQ8saPMvAoU0zvhksA8HfUWPYElMjxuFIE8kF2NumIoED3oqUE7kK0Du5U41bzVCF+7C7KavBKp0jwa58I8MnXJuwtu2Lxl6FM8UxCbO9rMILrRHyO7+B6Lu88ai7wJ5w69hHgLvHbcI7vO3y68GURSPNQvIb2dM0M8AdSyvD4577zy5YC8BhgmvGLKtzx0zFU9aZERO4V8vzs9jFE8Q3gnum2j0zziehw7fccRvcHqhLslE0E8/RdWPKg+XrzBfLQ8O0LBvHn2tzwJ2z+8WOs8u2+1brzeZqU7rFE6vDqgW7zizY26uq8yPFtn2LwNVAG86X9YvBFT1jxcvvu8KsGUvC/woztwiUS8KPymvJv0Fb3I6sU7FPh2PHoM8rpmNMo8lnGRu5fJCT3MdkM7pXhYPIHj+bxDzJy74SHPvLclv7pjSKu8bSt1vDIMy7wNL2I8dFZIvBkB8LywxlO8XfJevL1l3jxNTIo7tcUwveJMAb1DjPC8DSOIvCS53TwdTww8u/GAu8VV+rsNma6820tOOnFmwjzBIi+9R7AMO1t1lbo59Ag8STPbPMSswzsy4wE9hYE8vHWZhDxDJs65rz9WPLEVjzx/Avw7IR00PXfSerxsBoW6NAn9PDoH6Tt+9Aq9KjG+vDF9rzzjgzY8oRuQPDGc+zwtKzg8iEh7PBI9wTzF9O27PMT2PFwrdLxsnA+91ajivBVfRLz4Agw8uIhfvIpKErwcEuC8IJb5vPvEujyy2pi8ymaTu6ILDzzaiUm8QAirutx91jzoh/U8Alj+PNKndDyfCP+7XIElvCXfKD0xY908zBEbPNR2xzrTJJw8rg+xulMxNzwzhhU8ZsykutcD5jt7bfG7E1MuuwGlRjwDjM28C+yRPBcaH7ulSaU8hzY4PA6+uryb4UY6VBFAPA/OXzyZTps8m6bnPD/CP70mY6q7BKAkO8KvM7wAOwK7+Wc0vNcJE73vm2k7STeaPLNXC7xiIdQ6ZHksPOWtirwmgZc8drBgu4BmqztXDdq7Z7Zgvc9TUTs6E/u8Y+E2uC6lmjwYDl+9Lpb6PAipz7wz7qc8CZ9PPLCUQ7yx6l68w5IfPcwcuDzteWu7JrGqu432aLzDzV29tvwoPVo3CrpSuZq8FIzzPOdThzw/E3g8zlPcuoSOt7xKhTk6hOydPAqzWzwDB7W8jq4Mve1o+jzPoe08j3VFujctyDw7Iho9pjIEPDAsr7xTe/u80P4MPAJj+DyAJMI64IE0u2IADrziHwK9RJtCvMhVzbzKuPa8Ci2oO/Zo8bvEW8E8RJ63OCzGzLsGSJQ8cLgdvJArAby7lkE68d26u8BXqDvrlWu7VRI4PNaBvrssjqm8pPsgvSL20jwI16M7ClarO5KIGLz0DpA8WOVBPPZBDLv3syk86SnBPCQ8dbsnhxy9+BHOOl54zDn3St67Mh8euI9wyzwteNU7TzodPA/zdTxCBgu8ZQcSu3vtBLwt2fC7zeXRPAe1Srzn4wy8sJU7u0Gbgzu5uS+9+g0qvNVmRz0aZqi7JoN3OybhMDw/6Yq8eJK7PJJ5Sjq/ayq9jMWDvOqPUDwacg+8lrpZO31USzxEXei7ORYQvLoolrxauwG9YiuNPB7vBLoQSCC8sR21u6mjCr0agb+8uBlXvEjALrwdIwK9iI8ot3jA7DwP97q8TpHMu8IYhjxUlSc8wSqwO2126rwlkUQ71xxmuzfzKToQBVY8Sx7qO0RlCDtwny07zkbNO6scVrzhM/Y7Cge9PEU6Qjzy/9K7WRqpuuig7byIMfK7xkw/PT+KHLzZ3S08lMrNPFEPEb0oqK28b1g/O/LCAT33JMM8OCiovHUU5jupyHW8MtRDOkNTmTxuZi289jCHvIWp0LzHsoC6ZcyMOpqEiTyx/Na8UIxCvNcN6LvHJhW86OWAvO26VbyaisA84cIkvCyIdDwQxoi8t78CPEaW/jsnNYg7IXb3O9P7Yzyjcmm8JxxnPG811jwaC6q8ZBYdOyt+9Twv1gE9g0IYPNVOqzuV7to5g+btOqjWVLwuUL47qJXLO6BeaL2Dwwk93RgIPbbL2rvib0o8tOQPPF08BzyzPNY6dDCuOygAlLwDAG47hhapu6W2/rykXGC7SC4KvbRaqjvBWdc73ZuVPJeF0Dupb1s8yjClvC1p9LuJEfi7Km90vGpBWryVh4u7GQfuOx5U3bzBoGY5qHtvO5jYILuLdti78wYRPL478TyA5jE9SQzSO76jKLz/KjY9ofo5vAZyqzwjJ8C88YMDPJsMKryB3ve8II+5O3egK72suwA9S77tvIuPmjtj7Io82ntPvFNEXz3rPX+8qgp1PDWNVjvmRNm85uY9umpT4Lw4nlY8mI84uYmlMDwTv8s8f932vB19kjx0mVi53i6rPKUk8Ly2r90886xuvHU/9TyfrE08ODgfPJ8M/zyEoaA8OikuPP9fRjx+rAE8GiCJOZ9j9DrPhJs88RQLPbApvrzTD0Y8BJr6u0/rgDxOCwe9164bO7z2bL3r/r27GklxPPvkLrx8AtG865sfvKTxiLv74748I0S7vGIDxbxm3aS7zniQvN3Y87ordRo5AXynvKQlGjy+daY82v/7OzUiHjz1RXm8ws9tPMlE0Tw0QAa88BlIPHFPFryPP+e883AMvI/rtby9tYW8NNAhPOIuYLx1H3c83zueujqOnzz22Ag7yaiQPNYFqbwhKRK8GKIUPYriprnJKLE81RtKvLYTg7wPHRS9FJKvvHTSarxGUga80bwzvKRAorxXnhc90+zsO9rjhTzkpNA8IQkkvHRWhDwMJLE8jUmZvL6/zDsllxm8lzKFPA2KKjs2DRe7+/yFvDRH2TyyNRy99OP/OxX1EL0hgyu8sFCbvKHJhTvyvQm8AJ+nu6L+hryg1Uk86Z2YvHQ8Dz2EIxA6QGravAyRhDsbAZ+7E7sKPCsnCTwBLrY84oDVO+Nyt7to9nM8TSE9vCN7z7tKNfO8n/qTvEX6XbzYNRy8TAEIvPzTzDzXSBK7nVGbvJNHhTx2ZLQ8saRaPCTYhDveCCa895AcO1t7srw+Ad68YJEBPCA1kjtkE1K8HUkLvK6G4zyP3G674rKgOwD6RTzVN3m8/fEbvZJOy7qMz9k79+37uxjxKTxHi/Q7Cp94PBkhYzzJMJE87sXSPLyuCr0GoFg8ibMAPA0J0rwLwIi84ZEmuu8EjjvkXjQ8IIMoPWXjB7wY6P88iEu0PNveQLxfV326aaEKPXQ2hzx3AEq8pMrsPNSuLT0xtKG7gJUTvWJ7ITx40M+7iJqHvBgCEzshn2q8xnPdPAYBFrwHHYw8G1AWPUpzN7wO9pK879NZPM76Nbxo6My8n/uvPOenITy1L788FIeXO0RCkLvSOZS8R3R/vKcIiztRWhE7i+ubPBrp7brUnmm8/S2uvDiB0rqDf9O8DvqSvEkcIT1zgcE8xbEqvc97N7wCHBS9WvBCuo3Y8TqN1MI7G8nRu5DIKr1C09G8hcgePET86jxstig8jBWcO1IZLbyqwrk74LbAOvC5n7tmHco7t1sDvJ7QwryjmQq8rqtGvPoz8zw6a7S6AjBIO7THCz1e+Yc8+bTsO+1dfLu/RSS80Uu7PHuvlTySMte8gvxIvAi9izzBFVm8RZZLPCDTrbuhcnQ8QndpPP0zSzx8iCc8QbGlPHvh1DvUOS48o9mhPLoT47tB2ww8giLMvHG66TuqAsg8climPEHaLj1wzj48OfXyuuQBrbzTe5e8oIVKO3QZHLucgbm8p4MvPAAncjpmS7Y6aPYkPa3w3bu4yz67L4cevASUPrxqOrE8fO8PvR9MU7sYpIm7ytunO2mQ0Lj3DKI8oBKXOdapAjz7jyQ8/HLXvK+mNDxSR9846PrfPAUecrv7tlc8gn4nvJIAnTxFojK8X0xUvCfrsDuqLUE8dhgKvLgZzLzvgjo8mhiXPJrVgLziO988FvhEvLKBlzy9HlE7faYXPFeFi7wdoxW95xb+O5G45jvk55A8Dy6HvPSgILyHVuc7oZNvvFL8gLwGVCi8Mf0+vEoDgTyjWBG8fWkEPDZNGbwOzbi8p0EAPWsWVjwtDKc8MGGsvN+YV7zggAG6HlQHvIpr0rvcMnI8yYB1u0QCkbxyIGy8EA0YvX+udLyTUja7RYvuO0DQq7wVL0E75m26PBfrWjrj7VA8QeH+vIxo/7ztmKY7kZezvA== + index: 0 + object: embedding + model: qwen3-embedding:4b + object: list + usage: + prompt_tokens: 5 + total_tokens: 5 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_s3_source/test_create_document_from_s3_new.yaml b/tests/cassettes/test_s3_source/test_create_document_from_s3_new.yaml new file mode 100644 index 00000000..96a6cc99 --- /dev/null +++ b/tests/cassettes/test_s3_source/test_create_document_from_s3_new.yaml @@ -0,0 +1,42 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '87' + content-type: + - application/json + host: + - localhost:11434 + method: POST + parsed_body: + encoding_format: base64 + input: + - S3 hosted content + model: qwen3-embedding:4b + uri: http://localhost:11434/v1/embeddings + response: + headers: + content-type: + - application/json + transfer-encoding: + - chunked + parsed_body: + data: + - embedding: jIfcuDGtJDtS1Q49KmHyOQbTEbpKpkk9+JktPR4H8bw8H5E8G0qevCCB7DwTB1w9BeoVO+MofrtFEPe8hw+pOnVmu7wXz1O9jZrrOR4aqrvnx268x9Tru30nDj2waz297gtOvdd4h7zI8aS8PcRcvW7RRjxGviS8vSM9PBewWr0h0v48z0iru10mTjtM+5e8WJQouvZnIbxUC++7MXipPDdC4DxGwzk8D4lQPGGrDrtt8yy9yKvkOUgJDjxyaDW9mUlCPHojnbwhqOY7f6zoO1OUv7wca6q8XcsCPQ2I/zq13KM8d4vOugSTOb2Z3z29j1mFvHdSmbuCqYu6a/oIvDlaubuJGsO8v6RbvA1cD70eED08pwasO4u4Cj0FlhI8a/civEqoMD2Z0b47tMfpvOxKMbwb+ew8QWuluz9OMjzfAJa7AjO9ut+ILjzEenU7M5YpPH8kxru90a07CWCBOxpOJ71nCZC6rXszPD1hvTyowhQ8oT2nOzboBTtN48M76Pd9vLJYvbzrHhu6uQY6PBoegTxKdk87WVNAPBmhMbyWHwo8TVeqvHsIIjvQz686U52kPJrPkjtDWhG763fLukhpLzvfmdE7Ew8bvb8VY7w+PK+8HVLhO/xFcDup1DU9G+96vIGViDxbBjw87f8hPAKOP7sQBGK8Fs4VPBX0ArzVz4G8Bp+WPGUE4Tw5uOK61p0tPFb3/bwuw1q7pqEmu9oJ/bu0GG+8jZ7/u4q/QTyWATa8uriUOxF/UTyW+ZK7vtORvPikI73hTlY7v1E2PYDnnjlPOlM7o0IxPWb5pLv76Hy8BK/RO6Pysbtcnnw8yq5/vAqp4jwd5Z87zW1RPFFMsjsfH687jXNuvD86VTy8aA488j5VvAtyCry220A6BLgEvO6EfbxbkbY6d7cZvLMd0Ts51Mw2pXI0vDIkGDzoR4y8HAOzPKr/1LspCD89flZOvE6BTjy1nkk8Yrs/vPLvNjyps6o7jNmwO21pFjyWVQ087eaSPIAqVLzifqg8mMNuOwFmoztx1z67WQjevAJOwLwIE8G7J+TpvPD4qTx1AEY8AVFFOW2nzTzNS+Y66NaPurNsGrye0Fc7d6rAuw8q4Ts6c5O8hbPePEOISDxwIWG8g0nLOrL6GTwf3DI8D2mfvJX6H7wU+3c8jddmPHlFVDuprm0897h7uahQHjwTUA+9Qm17PAM+Q7tVqnW7N1jMOpIEDTz5ap08vBeRPLywCjt8yNq8uNV9O2aGk7zNKjq9YqlWu28Z5js1zSo8xChKPIT4HL1vgUM72b0BPMb2pjtSf9S8BDiePLvgTLwMO2m8yRJ+vBDLJDxKCwm8Yrxbu46VV7yOIdK8Vo+dN15hurrT9Z48cBVUu1hoWTxC7GK5qNGuu+tLzrsPCmC77MGJvGUgQ72mcxS8oTBQPIycoDwGaX28VIiBPbA4p7uO7K06u9rLOge8Sb0ykgG9RYC2O0qcYDxgR9A8xVFLPEPslLwi92Q8sHwfvC8xuDpIUHW8Swm9OvZUzDxSAxG8GkFsvHk9FzxdWME8ZfIyvS9vfTxuUZS8XX0TvYDJvTiNk7w88i0tvH1uMrw+7Ao6RGUuvLQcN7wHwCG8WqAtvOVeVTzFpEe7vwiHOnExZDuOgK683cUKvCfcq7tmzxs7uLQlO1S+UDuHFhs8JNA1vTctnzy4xgC7e98cvbaNCDyrrSK7MUE3vWxbSDwS9A+9Cj/guy5YjLu7QYy67Q0IvNf8Hb3QFMY7KEAaPPBWCz3CQdW8WCQjO9sZKjwu69O7/G2ePLeMgj0A9g495+QQvCMarju0fu87vx+kvFfuqTxcp/O8VUOnO/Qtfzz0VTA8vz8lPF6TyjxFVjS9sJLbvB7WlLzvrSa8UHujOx8dKjx0IxA7Y6LgO6x5kDwgXL26PlYVvNn1q7s0yhU8Mj+8uaR4JL3lFWC6lbwpvBFWQzztOke7IMIFvedObDt4qjQ8DRMHPX/PrjxnhQM85MI0u0eQYbwoNL+7xBlNuym5Yjv9/Bu7f4taPFXrrrxqeHI8mJ2fPLtj6TvW3p86gT84O4301DsEHas7fOqvvI57Qbw0pKk7ddKtOxslDLyqL5U8fJpBPDWARTzEmHA8UBuuu3x2KzyglVc7A+0PvSqmqDz80wO8sxGOvEbEkjvdKik9QoXSPAtMVLwp0MO7DIU/PIT5FLgNib28gMNXvQmkjDxcQKc8c2qpvK5/Bbz1gzK8RKaNuywnqLwaCe28dVszOphGgzsvyKY8xSuSvO0EjzyUeay71PfGO5x7grzPnMU8fRcUPW7+5Dw4Kjc7UlCsPKFySDx5luu7FS2UvEv1qbsk0oa7cR7TvGOfxzvNp5U8SFLavK44sDyul/U8r48NPX0ZITv+BDG9ItQnvH7EjbwIr1I8ZSHevF8cyrz5nsE81lDrPFsJnLzCmYO8Ib5ruxQ9h72Q3dU7F0hSPT/Czbw0FBY6nPCOPLTpz7xKYNo7tgcVPY3xBj1oEb28hqItvSXM57sSoYA7bvWFvBonpTxZH0C830jMPCgkGz0z8Z+8Oqa9PL00ajxUm8I8BNFxPdAwOjx5nC89ARO6O1MT6jw40+I8FoghPEvNybuoCkg9GYiLvDKHWDx7nS28GeqDvLDKlzwoQ628zYCvO02psDvxAn47OMvINyXw07tR/ha8466Cu01dSbzsFFC8pQ0aPBX+Orx6DLk8FitRPI87Drtg0V+9eflmvZzHCjxmJne6fcQxvHZTD7xc/jW80+oMvWzEjbxXLky8r8cXPZxnDTykjaS8T1qfPLduKzwGE/s8/1wTvRBy6jxaCpK8kDcxvIM1Szw4jgu8Gj6TvOC7urzYur28vT9/PFN/ojxlgxA8Ks00PEiY0DtyUug8Cj6ou4YqebwvE3+82LfjvIvHQTzHrCW7U6iBO2yNWzzFOI06Mhg4PcYFl7zrwqU8NabxPIUi2bwqumw5RSdNOckhsTvJx1q8JbZpvBSCPDwwj8489cEnPJziHbrhJBC8QIFnPFqG4ryPbsy8Eo0POgYbgjx+78M7v30jugimhjzcKYK9CVfKvF4fkTucMdW8zCxTvJG2lTwuj7U8zmu5O/xFDr1JU7+8aEtoO93ZFDwk0iU85i2LvHyiA7w3VQW9/bSCO/Jw4btR7fE6jpcYPJ5ECbwnIr+8Cn8HvdIRAbqZ54G7trXdPMuTDr2axrK8MjQZPZl687viHdy7YcI6PDGWqTxEA+G88GYBvAqv7bqyJ4g8P53KvDR3WjxxjpE8SSaAvcBZMrxkYBO9bsvwOyMayzzv0AK8R+8hPMNkwDoWmyC9tArGO239iL3NyWW8sSySvIN/N7wZKKm7kYbdu6tQvTt/Dvs8jZ09vBHcgzzhtLO8NWeDvekrDz3qkmu8ReTHvMxHTTwtQro8EIyfPIvz8br8rH88kjMXud73LL3laYi84roKvfoxnzyqE+i8U6XJPC2jhDo0v4O8ftK2O6i/PL0K+JI8ibqivHEFLbxNgZo8Bs3nPIUa6rwZT4u6E4SzPLH+bDwFPr682rlAvSxV+LypKmo9XDQzPPAYGTzId+M8KGohOwPYd7sDVgU9P10RveAx2jtrIzs8gVW3PM1kEz3fL8k4r4zbO7aRwzy1ncW7rAnnOy4M0joQdDi83BK3PN6hTTxSexE9nueRudwdnTyCNwQ9DzBCPSZ/mTtUDTE8RRKBu6LqIDs1Seq85CoMvaiQZbye5Pu62rq2PDPyljxZIIU8tlKbPEKa/zuR7q877vcDPIyiMz2SbTi8ZxeGvKjySL1ajK68W5Xvu63cHbtCEga7Y4a/u2flwrw8FDS83WQkPDshPjw/K3w7BVgyu6JIiLxlU1Q8O/amOrETiDznXWQ7cx0YPT20DD23dY68pmFkvLnOmzwzJaO7RTJOvPfZrLxpgn+8z4tlvKW0FTxc2tU7gXe/Oxj2SjzBk4C8qdutuXN8AT1brw084NuAvFU+IzxHLZc8W5WOPCFI1TxgZac8gsoIO3OOr7xAjOc8ECcZO66s7DysuOs8E1GUPEIbAjxEmfa8kFkQPNQ44rvCaBq9/fTuOfuQzjyLJZ28TgddvElMEDzBvRu8FEVHPR3xKrv3u4e8yRe/PI63HjwToMC86giSvNeDTzzRWfC7jcYXO1A52rs1Vg49USNMO3JtLjyACHE8oXEpvYE2abwQC02817kevSLP1zyL+d0724N6vGnCu7uHLDc8cygSvEp167vFO4y8COM2vQ3MyDvaj6m6np4nvJCxMrwm7AE9arGCPFGvBT3s26U8QDRtu5ZCJryhOL87nXKBPIE5xLyp0Gq8uHPfurFH0rtUlcq8OqXRO4qBm7veJiU8W/aOO7qx4jYXiZk8a7WLu7YWDLzdSaC84U0zPIzceDxUjEu8t8+GPBSQzzzZCBu8HZRqPLo5dTwhioI8Tj4gPI4S0Dy96uW6xKWAvHJ8pTuJ7mi8wo2MOur5ILyDdEA9H38qvYAZcLzYpiE8AJqJvLrJabtHUlo8IWzGPEXiqjzzKHU93a65PMT0dzyQgxO8DPDJOzntqDwljHE84Zz1PPiiHbx/9ak8CNvmvCCHTLx9Lxq8Totju2VNfrxR5u685/hRvdLHZjsastK8CfaXPKHRTTxxJ9s7BQGRPBo6uDwkeqY7gLqlvEaYZDydO2E8iH3yu6JqDT3Czq273X4YPRmHdbyzFg68p0nGO963vbqV1Tm8cdkdOu84cTsTNWK8ZvQmPYSCwTwuECS7+quwO92FDjzoU4U7x8l3vCdoNTykiHK8hHcAPTQLeTwKzB69XYziO6tf0bquyLi7ZQp1vAMIRrpvAFU877mAPAxl2jrDY9S8NMUgPGn06bl+6ci8C/FKuzIlfrzvfiA9ofj3vEUocLxtry88c+dEOx+jDr3ccHc7VMIfPZ4UMrz2Ng28E3wyvBtW6Lw+o+A6003zu+wMCbw3Fzy7mQtXvL9/1LzAq4k8GDimvHNX7jskhg48QVRkvLphRzwFaV+8RMVpvE5JND16rly8KIZ9PHWvFj0Qr2c8X/b+PLyX1jz1I8s7UEIYvfVusrxj0Ie8WTOBO0VCGLzz83e8usNLPCXR3TqZ16M6u2kbvHEAPro6Fdu75XpkPLf9ET2+s2C8h62hPLBbSLzRDA48CeIEuyupbbxkx4G7ERYouZe4Tbz8ViI7+m8yu2nmvTsInJ08TSfyPLLzRDxlL/i7NYofPEJwNDwCJIQ6Zo6QPGXRU70q44m8MCqHvEyTuzwnLwu9YNv0O/SUvzzFwwo72SHtPM9HIbu6Mro8apBPvEO60TrvIZQ79zpavAQUDr3iKtS6yPXGO3YDFr2TVJS86BtTvX/o3TxBheG8GoXFPEJxbrwIB9O8UgrdvBPbjbzhjCu6ZIyHvHu2/zqkYdc80babO3asjTsHlk48nJX/vGX0Krwor528y1s2vXYmajxdcQu8Gy4yPRt0MjxS+HS8r4E1PERw5Dtoh0Y8roSuvDNXUrsFdv88L5hVvHqNAb2ilwM8N7poPKd237wlGf+7NBj4O0WwWbtm+5y8gZ9tvGpNZDxwb5A82a8ovBdQhLuiloc8lBQrPMuMvzwg51k8PxxHvJNAs7ytzU09ZVtIPEhuBr3E9Js8LPbPPCfrM7yTrjA7JjWGPGLRFr2jfIc8673qvBxryDi5ywq8yEL5vKdq8TwBRwG8rbSXPChBiTy2e8k7UrhMOgrh2Lx5nqO7kLwBvQ8VubzpjvK8XOjtvMGeV7yeupO8ztqzvJfdqzwuBUS9NqsVOxUS2jshYha9rHAtPMqiarxZn4M7fW3nvD8ekbz6hOU82e/ovH+hEr0al8y8V2kYPTlgkryE7zW8MZkJOytwLLzBRjq8ZiCPvHDKZrzMOx87tinSPPvmJbtcTAs9ydLMvM/Xtzvzt3g7KUqyPAAZxDxNLHO8kQtlvOwcwTwDD7G8yIeeu6FHCrsYuz48p1Mnu/yrm7vYTo268mtIPBPK1DvcJIO8TzaOPIml8Dt4Zgg8J/XiPPWuKLwgsIU7wcYcPcOXKTy9iLa8ltGLuw5OizuoVjY8L0GLvINOozxM3YC8sGxsvPExTTwAEqE8vD+RvHC2HD36GFo73csZPf/x7LxCxJ887AO0vEOONbsLEb078kRFPOQ9IT1BZzc9xtIyPC+M9jwjccE7g2vJO39Ahzw6YVK8T5hXvcibKj255i+8cW5+vL14oLhpjtS4s2KvvOhgDb1VegA82jm3O3odezglHlo8ar8lPeCKj7wwdxE9UHl2vK1SzjwBOYQ8+q0WPSvcerzCt5S8e/ukPJmJNTyiT6G6gtlpPLsSYbz1PD88Xr/rvG43ADyUwyG8nFfZPFXegTtaqvA7KCubvDbkFL2KcbO89fEAvZ3D1rx+0R07EcWAvJA1aLwt9t88TfKVPHnVgjxwJ4k8H2TqPLsCQTw8pTO7n2aWORWymzyJ77i8NsKBOl9SRzzHK2C84ZcJPFAsWbscwgm6OWs1OwG86zvS/OO8ATKEuzyjk7zuiMI8kQD/PBGP+LviRfA8g3xAO/iSzTtKDzQ9TdERPGnrcryNgSK7frzcvDhh7ry3otY8P0YfO5ZQoTvdX488WwiPPM2RIDxfJpE9fCMJPKxaNbzt9CC83nNXvEMnQrsdYzK9UXzdu/PGoLzNssy83AucvKxsTDyHAJ08FAq/vDAORbovkug8++4HuVHJj7yPFLY5hNchvCm0BD1OU4u7ChpiPB+SALyidRU9z9rDPNu2ADw1SP88wQSCO8y2ijx7WRS8Za9TO2GMhLw59oG8C+mMvB1xqbzKMBu8/TXrPDRFcrrZncq6fQZ5OzcR0TuBm8u7Q6kmuvXq3bvO8PK8ulYHvGc9ZzvvNey8EHQau/Wyibt8y8G8AqdRPQnJwrzcIji8lG6Bu+uCcryNWYa8zxW6PKQoXbx/Kpo8NyVCvZZDHDzFm8i8u7ulvDT6jTxexzm8XlhAvCZyQDwhoq27zpC7POmbqbtEgjG8t3BRvCzbP7xM5cO8XccNvT2KB72cHQg8o4bcPKhnQbssUCa8mUZLPHeuIrz8S1G8xOOPvK61lLzlX7s8zGU0vA9r4DzbClM8o/6fuyLGXzruOX8851dcPGrRlbyUrXE5eXRlvLRtBLw5YNy7I147PPb/sLxI2Hq72kgsPGzZnTy+Is68PPhxOy3WUbz4afQ7yf0ePTCMxzyoHX08hWKeu9inNLwdAiS81dPXPLQqHz0HriS9gJP0PHPoZTxRaoE83BIKPWpU/DyMQ8o8R5/mvORlRDxg1nA7OswdPYNhBby6xJ+7Rj0xPalO3TuhaMe85H33OzYKZLzgDJC8c1YtvFJpnrztsbU8+gKNvDZ+3jzang48tNxevfGokzw0HBM86arIu5kB4bsZ3My87JoLvUNCDT14fjW8+FCQvKwrMD0UmGq8pYruuzJYCD0Dvcg8GmuWvB2vgLyQVqU85smGPKQ+ibsmHAQ9d+xvvOK867xSsQC9eHnSOjqBaD2nVt28jo2ovK9CDjtLpJm8/PN/OtMyFbwsVDe8k1GTvKKmtTxqjxs8RfKjvO0Ljrxg/WU8TK3CvNZYVT2IEKg8ZbKFvH20wrwyibK7YfLavMNOjbxsxO27KX1tvA/3+ztlqzc8tAKPPD38SD2+S804nMePvFAIpjtjJgI5klKbvOA6RjuNmsS7prAkO6Z7Ur1RfbQ7kHoKPJYMX7sEO9G7QSdpvFbeiryVIYS80EygOnNDvDux4vy8BDHpupEwhTy25IQ7/6bdPNsekDrpI6w7gXcFPMbWl7xX/5y8KZ5HPGSMErsqGc28GNsPvOVbgjxhmWc8F6uzO4YU9juviVk7nQ3RO4IB8Ltc56s8iwY/vGB0ejxGI+Q7KKwAPe1bqbpzcVC9qKyKOqoPrjvD1Bm9uHeNuzTBkTszY4k6bvHwvKZlqDyV/z68Q5J/PHze77vQQO67mMTCvLpmMrxz9t+8xF4mO9E3IbxLZCU8XvcrPFm5oDz6hL+8+6cUPRr/ULzda4s8Syeeuz0MpDtMGEu8t+WlPEDUaTzpEHG8DwiMPOhAlTy1ZP87Xk2ivOcb0jvJa428UKSUvKGjZLxA+iu8NjX6OwHQ8rxTtoq6CqLkO6A7jDwKZjY9poqXO+OZE70kmRU8FtvjvHok/rwJpjA8DmWWvJj7dbxWml489BlGOy35P7zUg6k8Xo/LPN4PNLwI/so8Q6SSvMBMpTyxo7I898DdO6eMiDunBVu83gTPvG572zxfUmw8mCGxvKI5rDyqZYM8FdWXu90UKbz416g7aEhUOsM1Bz0hwjC8AqqyPKttdTwZ/eG89uY2PFZ9xzwcmYC8pyMtPLEVYr0JEZo7zp29PNUN+jvvX766tw0NPHypXjwI/Tq8awq+OwYWg7uMXhm88PkRvN9oh7tOFKK8SNCLPO5+KrsTw5260nllPMJzALxwrNG7gbC0OamalbqAjWA86xuFPMyXF7z5ApS8s/2gvPGsuzwtOBY9OPZXPDIizTzTHmo8w36uOyoFJz0cwgq9I6CPvFsjKrueIwq9qew1vPAHPTx32Vc8dFerunDTITzEYLW5HhjXO7rz5DwvuGW8NSNvvDrFgzzgIRs8SAgcPDHTYLyMQd26UqNMvWpuZjrizf484AmrPFXuXLyfvW47zAeLvPXAJzyUrrm8Y6sMPLlWv7oGeg+9gjSHuTZEnLxXdek8qhAqOyfvrLultEU8P4nlOpnBVDxhWra8PI6VvLjMJzwm0PQ7vKaOuol9uTy3ajs8R4gYPbyT0LsI5hI87jk9Opa1vzu6XxS8Ai7NuzOMkTwqvQm6p/LeO/2qsTotLk68ItM5PDxVPTyr5TO9BbpZPNG4CbyWP5C7TeWhugQzxjzJO6c8JkA5vSjdZrzFx6k8lEKNvNsrjryvxQW9fS+VPBaj6rzPNoy79VVpuidxC73Xjnm8paEbPeNcnLw5jR298zOOu9xL0zqE27+6YyA6PGkpxzuXgfE8Oiweu9pYFD3muji7cX5xu7sAdTzZnsm849/pvOOmOTuM6JO8GSCEOx7VVLsiife8l9dbPODYoDwC7we9g6AIvJ58qbyJgGI8EsGVPAbtXLxz5Ic7cxD/vDGXbjxVfhY9OOd4vF0/y7uej8M8nl1NvNyhJz2vEPI8sVthvNMUnbzROgY9xVJ0vMmPNT3bw+Q8zzHAu7NaLzx3xLW7R1wjPCsVDT2xL6S8c5AAPJ4fYLw+ApC8o4kPvNUYybwOGd87f3WKu6VWjDw4oXi8YhiEO6ati7z2hkc8AvNdu/p/DjwlIUg7yQg5OT85J7y6U6E7VmFnvIPrajwUaO28CNI1PZeXoDurMZI8XY0JvUr7frzRn5O7FpCWvPQ2s7zgcoO8E9vDu1uTv7yO9TK9Y4f4PGTKkryhnRQ8saPMvAoU0zvhksA8HfUWPYElMjxuFIE8kF2NumIoED3oqUE7kK0Du5U41bzVCF+7C7KavBKp0jwa58I8MnXJuwtu2Lxl6FM8UxCbO9rMILrRHyO7+B6Lu88ai7wJ5w69hHgLvHbcI7vO3y68GURSPNQvIb2dM0M8AdSyvD4577zy5YC8BhgmvGLKtzx0zFU9aZERO4V8vzs9jFE8Q3gnum2j0zziehw7fccRvcHqhLslE0E8/RdWPKg+XrzBfLQ8O0LBvHn2tzwJ2z+8WOs8u2+1brzeZqU7rFE6vDqgW7zizY26uq8yPFtn2LwNVAG86X9YvBFT1jxcvvu8KsGUvC/woztwiUS8KPymvJv0Fb3I6sU7FPh2PHoM8rpmNMo8lnGRu5fJCT3MdkM7pXhYPIHj+bxDzJy74SHPvLclv7pjSKu8bSt1vDIMy7wNL2I8dFZIvBkB8LywxlO8XfJevL1l3jxNTIo7tcUwveJMAb1DjPC8DSOIvCS53TwdTww8u/GAu8VV+rsNma6820tOOnFmwjzBIi+9R7AMO1t1lbo59Ag8STPbPMSswzsy4wE9hYE8vHWZhDxDJs65rz9WPLEVjzx/Avw7IR00PXfSerxsBoW6NAn9PDoH6Tt+9Aq9KjG+vDF9rzzjgzY8oRuQPDGc+zwtKzg8iEh7PBI9wTzF9O27PMT2PFwrdLxsnA+91ajivBVfRLz4Agw8uIhfvIpKErwcEuC8IJb5vPvEujyy2pi8ymaTu6ILDzzaiUm8QAirutx91jzoh/U8Alj+PNKndDyfCP+7XIElvCXfKD0xY908zBEbPNR2xzrTJJw8rg+xulMxNzwzhhU8ZsykutcD5jt7bfG7E1MuuwGlRjwDjM28C+yRPBcaH7ulSaU8hzY4PA6+uryb4UY6VBFAPA/OXzyZTps8m6bnPD/CP70mY6q7BKAkO8KvM7wAOwK7+Wc0vNcJE73vm2k7STeaPLNXC7xiIdQ6ZHksPOWtirwmgZc8drBgu4BmqztXDdq7Z7Zgvc9TUTs6E/u8Y+E2uC6lmjwYDl+9Lpb6PAipz7wz7qc8CZ9PPLCUQ7yx6l68w5IfPcwcuDzteWu7JrGqu432aLzDzV29tvwoPVo3CrpSuZq8FIzzPOdThzw/E3g8zlPcuoSOt7xKhTk6hOydPAqzWzwDB7W8jq4Mve1o+jzPoe08j3VFujctyDw7Iho9pjIEPDAsr7xTe/u80P4MPAJj+DyAJMI64IE0u2IADrziHwK9RJtCvMhVzbzKuPa8Ci2oO/Zo8bvEW8E8RJ63OCzGzLsGSJQ8cLgdvJArAby7lkE68d26u8BXqDvrlWu7VRI4PNaBvrssjqm8pPsgvSL20jwI16M7ClarO5KIGLz0DpA8WOVBPPZBDLv3syk86SnBPCQ8dbsnhxy9+BHOOl54zDn3St67Mh8euI9wyzwteNU7TzodPA/zdTxCBgu8ZQcSu3vtBLwt2fC7zeXRPAe1Srzn4wy8sJU7u0Gbgzu5uS+9+g0qvNVmRz0aZqi7JoN3OybhMDw/6Yq8eJK7PJJ5Sjq/ayq9jMWDvOqPUDwacg+8lrpZO31USzxEXei7ORYQvLoolrxauwG9YiuNPB7vBLoQSCC8sR21u6mjCr0agb+8uBlXvEjALrwdIwK9iI8ot3jA7DwP97q8TpHMu8IYhjxUlSc8wSqwO2126rwlkUQ71xxmuzfzKToQBVY8Sx7qO0RlCDtwny07zkbNO6scVrzhM/Y7Cge9PEU6Qjzy/9K7WRqpuuig7byIMfK7xkw/PT+KHLzZ3S08lMrNPFEPEb0oqK28b1g/O/LCAT33JMM8OCiovHUU5jupyHW8MtRDOkNTmTxuZi289jCHvIWp0LzHsoC6ZcyMOpqEiTyx/Na8UIxCvNcN6LvHJhW86OWAvO26VbyaisA84cIkvCyIdDwQxoi8t78CPEaW/jsnNYg7IXb3O9P7Yzyjcmm8JxxnPG811jwaC6q8ZBYdOyt+9Twv1gE9g0IYPNVOqzuV7to5g+btOqjWVLwuUL47qJXLO6BeaL2Dwwk93RgIPbbL2rvib0o8tOQPPF08BzyzPNY6dDCuOygAlLwDAG47hhapu6W2/rykXGC7SC4KvbRaqjvBWdc73ZuVPJeF0Dupb1s8yjClvC1p9LuJEfi7Km90vGpBWryVh4u7GQfuOx5U3bzBoGY5qHtvO5jYILuLdti78wYRPL478TyA5jE9SQzSO76jKLz/KjY9ofo5vAZyqzwjJ8C88YMDPJsMKryB3ve8II+5O3egK72suwA9S77tvIuPmjtj7Io82ntPvFNEXz3rPX+8qgp1PDWNVjvmRNm85uY9umpT4Lw4nlY8mI84uYmlMDwTv8s8f932vB19kjx0mVi53i6rPKUk8Ly2r90886xuvHU/9TyfrE08ODgfPJ8M/zyEoaA8OikuPP9fRjx+rAE8GiCJOZ9j9DrPhJs88RQLPbApvrzTD0Y8BJr6u0/rgDxOCwe9164bO7z2bL3r/r27GklxPPvkLrx8AtG865sfvKTxiLv74748I0S7vGIDxbxm3aS7zniQvN3Y87ordRo5AXynvKQlGjy+daY82v/7OzUiHjz1RXm8ws9tPMlE0Tw0QAa88BlIPHFPFryPP+e883AMvI/rtby9tYW8NNAhPOIuYLx1H3c83zueujqOnzz22Ag7yaiQPNYFqbwhKRK8GKIUPYriprnJKLE81RtKvLYTg7wPHRS9FJKvvHTSarxGUga80bwzvKRAorxXnhc90+zsO9rjhTzkpNA8IQkkvHRWhDwMJLE8jUmZvL6/zDsllxm8lzKFPA2KKjs2DRe7+/yFvDRH2TyyNRy99OP/OxX1EL0hgyu8sFCbvKHJhTvyvQm8AJ+nu6L+hryg1Uk86Z2YvHQ8Dz2EIxA6QGravAyRhDsbAZ+7E7sKPCsnCTwBLrY84oDVO+Nyt7to9nM8TSE9vCN7z7tKNfO8n/qTvEX6XbzYNRy8TAEIvPzTzDzXSBK7nVGbvJNHhTx2ZLQ8saRaPCTYhDveCCa895AcO1t7srw+Ad68YJEBPCA1kjtkE1K8HUkLvK6G4zyP3G674rKgOwD6RTzVN3m8/fEbvZJOy7qMz9k79+37uxjxKTxHi/Q7Cp94PBkhYzzJMJE87sXSPLyuCr0GoFg8ibMAPA0J0rwLwIi84ZEmuu8EjjvkXjQ8IIMoPWXjB7wY6P88iEu0PNveQLxfV326aaEKPXQ2hzx3AEq8pMrsPNSuLT0xtKG7gJUTvWJ7ITx40M+7iJqHvBgCEzshn2q8xnPdPAYBFrwHHYw8G1AWPUpzN7wO9pK879NZPM76Nbxo6My8n/uvPOenITy1L788FIeXO0RCkLvSOZS8R3R/vKcIiztRWhE7i+ubPBrp7brUnmm8/S2uvDiB0rqDf9O8DvqSvEkcIT1zgcE8xbEqvc97N7wCHBS9WvBCuo3Y8TqN1MI7G8nRu5DIKr1C09G8hcgePET86jxstig8jBWcO1IZLbyqwrk74LbAOvC5n7tmHco7t1sDvJ7QwryjmQq8rqtGvPoz8zw6a7S6AjBIO7THCz1e+Yc8+bTsO+1dfLu/RSS80Uu7PHuvlTySMte8gvxIvAi9izzBFVm8RZZLPCDTrbuhcnQ8QndpPP0zSzx8iCc8QbGlPHvh1DvUOS48o9mhPLoT47tB2ww8giLMvHG66TuqAsg8climPEHaLj1wzj48OfXyuuQBrbzTe5e8oIVKO3QZHLucgbm8p4MvPAAncjpmS7Y6aPYkPa3w3bu4yz67L4cevASUPrxqOrE8fO8PvR9MU7sYpIm7ytunO2mQ0Lj3DKI8oBKXOdapAjz7jyQ8/HLXvK+mNDxSR9846PrfPAUecrv7tlc8gn4nvJIAnTxFojK8X0xUvCfrsDuqLUE8dhgKvLgZzLzvgjo8mhiXPJrVgLziO988FvhEvLKBlzy9HlE7faYXPFeFi7wdoxW95xb+O5G45jvk55A8Dy6HvPSgILyHVuc7oZNvvFL8gLwGVCi8Mf0+vEoDgTyjWBG8fWkEPDZNGbwOzbi8p0EAPWsWVjwtDKc8MGGsvN+YV7zggAG6HlQHvIpr0rvcMnI8yYB1u0QCkbxyIGy8EA0YvX+udLyTUja7RYvuO0DQq7wVL0E75m26PBfrWjrj7VA8QeH+vIxo/7ztmKY7kZezvA== + index: 0 + object: embedding + model: qwen3-embedding:4b + object: list + usage: + prompt_tokens: 5 + total_tokens: 5 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_s3_source/test_create_document_from_s3_skips_when_etag_unchanged.yaml b/tests/cassettes/test_s3_source/test_create_document_from_s3_skips_when_etag_unchanged.yaml new file mode 100644 index 00000000..96a6cc99 --- /dev/null +++ b/tests/cassettes/test_s3_source/test_create_document_from_s3_skips_when_etag_unchanged.yaml @@ -0,0 +1,42 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '87' + content-type: + - application/json + host: + - localhost:11434 + method: POST + parsed_body: + encoding_format: base64 + input: + - S3 hosted content + model: qwen3-embedding:4b + uri: http://localhost:11434/v1/embeddings + response: + headers: + content-type: + - application/json + transfer-encoding: + - chunked + parsed_body: + data: + - embedding: jIfcuDGtJDtS1Q49KmHyOQbTEbpKpkk9+JktPR4H8bw8H5E8G0qevCCB7DwTB1w9BeoVO+MofrtFEPe8hw+pOnVmu7wXz1O9jZrrOR4aqrvnx268x9Tru30nDj2waz297gtOvdd4h7zI8aS8PcRcvW7RRjxGviS8vSM9PBewWr0h0v48z0iru10mTjtM+5e8WJQouvZnIbxUC++7MXipPDdC4DxGwzk8D4lQPGGrDrtt8yy9yKvkOUgJDjxyaDW9mUlCPHojnbwhqOY7f6zoO1OUv7wca6q8XcsCPQ2I/zq13KM8d4vOugSTOb2Z3z29j1mFvHdSmbuCqYu6a/oIvDlaubuJGsO8v6RbvA1cD70eED08pwasO4u4Cj0FlhI8a/civEqoMD2Z0b47tMfpvOxKMbwb+ew8QWuluz9OMjzfAJa7AjO9ut+ILjzEenU7M5YpPH8kxru90a07CWCBOxpOJ71nCZC6rXszPD1hvTyowhQ8oT2nOzboBTtN48M76Pd9vLJYvbzrHhu6uQY6PBoegTxKdk87WVNAPBmhMbyWHwo8TVeqvHsIIjvQz686U52kPJrPkjtDWhG763fLukhpLzvfmdE7Ew8bvb8VY7w+PK+8HVLhO/xFcDup1DU9G+96vIGViDxbBjw87f8hPAKOP7sQBGK8Fs4VPBX0ArzVz4G8Bp+WPGUE4Tw5uOK61p0tPFb3/bwuw1q7pqEmu9oJ/bu0GG+8jZ7/u4q/QTyWATa8uriUOxF/UTyW+ZK7vtORvPikI73hTlY7v1E2PYDnnjlPOlM7o0IxPWb5pLv76Hy8BK/RO6Pysbtcnnw8yq5/vAqp4jwd5Z87zW1RPFFMsjsfH687jXNuvD86VTy8aA488j5VvAtyCry220A6BLgEvO6EfbxbkbY6d7cZvLMd0Ts51Mw2pXI0vDIkGDzoR4y8HAOzPKr/1LspCD89flZOvE6BTjy1nkk8Yrs/vPLvNjyps6o7jNmwO21pFjyWVQ087eaSPIAqVLzifqg8mMNuOwFmoztx1z67WQjevAJOwLwIE8G7J+TpvPD4qTx1AEY8AVFFOW2nzTzNS+Y66NaPurNsGrye0Fc7d6rAuw8q4Ts6c5O8hbPePEOISDxwIWG8g0nLOrL6GTwf3DI8D2mfvJX6H7wU+3c8jddmPHlFVDuprm0897h7uahQHjwTUA+9Qm17PAM+Q7tVqnW7N1jMOpIEDTz5ap08vBeRPLywCjt8yNq8uNV9O2aGk7zNKjq9YqlWu28Z5js1zSo8xChKPIT4HL1vgUM72b0BPMb2pjtSf9S8BDiePLvgTLwMO2m8yRJ+vBDLJDxKCwm8Yrxbu46VV7yOIdK8Vo+dN15hurrT9Z48cBVUu1hoWTxC7GK5qNGuu+tLzrsPCmC77MGJvGUgQ72mcxS8oTBQPIycoDwGaX28VIiBPbA4p7uO7K06u9rLOge8Sb0ykgG9RYC2O0qcYDxgR9A8xVFLPEPslLwi92Q8sHwfvC8xuDpIUHW8Swm9OvZUzDxSAxG8GkFsvHk9FzxdWME8ZfIyvS9vfTxuUZS8XX0TvYDJvTiNk7w88i0tvH1uMrw+7Ao6RGUuvLQcN7wHwCG8WqAtvOVeVTzFpEe7vwiHOnExZDuOgK683cUKvCfcq7tmzxs7uLQlO1S+UDuHFhs8JNA1vTctnzy4xgC7e98cvbaNCDyrrSK7MUE3vWxbSDwS9A+9Cj/guy5YjLu7QYy67Q0IvNf8Hb3QFMY7KEAaPPBWCz3CQdW8WCQjO9sZKjwu69O7/G2ePLeMgj0A9g495+QQvCMarju0fu87vx+kvFfuqTxcp/O8VUOnO/Qtfzz0VTA8vz8lPF6TyjxFVjS9sJLbvB7WlLzvrSa8UHujOx8dKjx0IxA7Y6LgO6x5kDwgXL26PlYVvNn1q7s0yhU8Mj+8uaR4JL3lFWC6lbwpvBFWQzztOke7IMIFvedObDt4qjQ8DRMHPX/PrjxnhQM85MI0u0eQYbwoNL+7xBlNuym5Yjv9/Bu7f4taPFXrrrxqeHI8mJ2fPLtj6TvW3p86gT84O4301DsEHas7fOqvvI57Qbw0pKk7ddKtOxslDLyqL5U8fJpBPDWARTzEmHA8UBuuu3x2KzyglVc7A+0PvSqmqDz80wO8sxGOvEbEkjvdKik9QoXSPAtMVLwp0MO7DIU/PIT5FLgNib28gMNXvQmkjDxcQKc8c2qpvK5/Bbz1gzK8RKaNuywnqLwaCe28dVszOphGgzsvyKY8xSuSvO0EjzyUeay71PfGO5x7grzPnMU8fRcUPW7+5Dw4Kjc7UlCsPKFySDx5luu7FS2UvEv1qbsk0oa7cR7TvGOfxzvNp5U8SFLavK44sDyul/U8r48NPX0ZITv+BDG9ItQnvH7EjbwIr1I8ZSHevF8cyrz5nsE81lDrPFsJnLzCmYO8Ib5ruxQ9h72Q3dU7F0hSPT/Czbw0FBY6nPCOPLTpz7xKYNo7tgcVPY3xBj1oEb28hqItvSXM57sSoYA7bvWFvBonpTxZH0C830jMPCgkGz0z8Z+8Oqa9PL00ajxUm8I8BNFxPdAwOjx5nC89ARO6O1MT6jw40+I8FoghPEvNybuoCkg9GYiLvDKHWDx7nS28GeqDvLDKlzwoQ628zYCvO02psDvxAn47OMvINyXw07tR/ha8466Cu01dSbzsFFC8pQ0aPBX+Orx6DLk8FitRPI87Drtg0V+9eflmvZzHCjxmJne6fcQxvHZTD7xc/jW80+oMvWzEjbxXLky8r8cXPZxnDTykjaS8T1qfPLduKzwGE/s8/1wTvRBy6jxaCpK8kDcxvIM1Szw4jgu8Gj6TvOC7urzYur28vT9/PFN/ojxlgxA8Ks00PEiY0DtyUug8Cj6ou4YqebwvE3+82LfjvIvHQTzHrCW7U6iBO2yNWzzFOI06Mhg4PcYFl7zrwqU8NabxPIUi2bwqumw5RSdNOckhsTvJx1q8JbZpvBSCPDwwj8489cEnPJziHbrhJBC8QIFnPFqG4ryPbsy8Eo0POgYbgjx+78M7v30jugimhjzcKYK9CVfKvF4fkTucMdW8zCxTvJG2lTwuj7U8zmu5O/xFDr1JU7+8aEtoO93ZFDwk0iU85i2LvHyiA7w3VQW9/bSCO/Jw4btR7fE6jpcYPJ5ECbwnIr+8Cn8HvdIRAbqZ54G7trXdPMuTDr2axrK8MjQZPZl687viHdy7YcI6PDGWqTxEA+G88GYBvAqv7bqyJ4g8P53KvDR3WjxxjpE8SSaAvcBZMrxkYBO9bsvwOyMayzzv0AK8R+8hPMNkwDoWmyC9tArGO239iL3NyWW8sSySvIN/N7wZKKm7kYbdu6tQvTt/Dvs8jZ09vBHcgzzhtLO8NWeDvekrDz3qkmu8ReTHvMxHTTwtQro8EIyfPIvz8br8rH88kjMXud73LL3laYi84roKvfoxnzyqE+i8U6XJPC2jhDo0v4O8ftK2O6i/PL0K+JI8ibqivHEFLbxNgZo8Bs3nPIUa6rwZT4u6E4SzPLH+bDwFPr682rlAvSxV+LypKmo9XDQzPPAYGTzId+M8KGohOwPYd7sDVgU9P10RveAx2jtrIzs8gVW3PM1kEz3fL8k4r4zbO7aRwzy1ncW7rAnnOy4M0joQdDi83BK3PN6hTTxSexE9nueRudwdnTyCNwQ9DzBCPSZ/mTtUDTE8RRKBu6LqIDs1Seq85CoMvaiQZbye5Pu62rq2PDPyljxZIIU8tlKbPEKa/zuR7q877vcDPIyiMz2SbTi8ZxeGvKjySL1ajK68W5Xvu63cHbtCEga7Y4a/u2flwrw8FDS83WQkPDshPjw/K3w7BVgyu6JIiLxlU1Q8O/amOrETiDznXWQ7cx0YPT20DD23dY68pmFkvLnOmzwzJaO7RTJOvPfZrLxpgn+8z4tlvKW0FTxc2tU7gXe/Oxj2SjzBk4C8qdutuXN8AT1brw084NuAvFU+IzxHLZc8W5WOPCFI1TxgZac8gsoIO3OOr7xAjOc8ECcZO66s7DysuOs8E1GUPEIbAjxEmfa8kFkQPNQ44rvCaBq9/fTuOfuQzjyLJZ28TgddvElMEDzBvRu8FEVHPR3xKrv3u4e8yRe/PI63HjwToMC86giSvNeDTzzRWfC7jcYXO1A52rs1Vg49USNMO3JtLjyACHE8oXEpvYE2abwQC02817kevSLP1zyL+d0724N6vGnCu7uHLDc8cygSvEp167vFO4y8COM2vQ3MyDvaj6m6np4nvJCxMrwm7AE9arGCPFGvBT3s26U8QDRtu5ZCJryhOL87nXKBPIE5xLyp0Gq8uHPfurFH0rtUlcq8OqXRO4qBm7veJiU8W/aOO7qx4jYXiZk8a7WLu7YWDLzdSaC84U0zPIzceDxUjEu8t8+GPBSQzzzZCBu8HZRqPLo5dTwhioI8Tj4gPI4S0Dy96uW6xKWAvHJ8pTuJ7mi8wo2MOur5ILyDdEA9H38qvYAZcLzYpiE8AJqJvLrJabtHUlo8IWzGPEXiqjzzKHU93a65PMT0dzyQgxO8DPDJOzntqDwljHE84Zz1PPiiHbx/9ak8CNvmvCCHTLx9Lxq8Totju2VNfrxR5u685/hRvdLHZjsastK8CfaXPKHRTTxxJ9s7BQGRPBo6uDwkeqY7gLqlvEaYZDydO2E8iH3yu6JqDT3Czq273X4YPRmHdbyzFg68p0nGO963vbqV1Tm8cdkdOu84cTsTNWK8ZvQmPYSCwTwuECS7+quwO92FDjzoU4U7x8l3vCdoNTykiHK8hHcAPTQLeTwKzB69XYziO6tf0bquyLi7ZQp1vAMIRrpvAFU877mAPAxl2jrDY9S8NMUgPGn06bl+6ci8C/FKuzIlfrzvfiA9ofj3vEUocLxtry88c+dEOx+jDr3ccHc7VMIfPZ4UMrz2Ng28E3wyvBtW6Lw+o+A6003zu+wMCbw3Fzy7mQtXvL9/1LzAq4k8GDimvHNX7jskhg48QVRkvLphRzwFaV+8RMVpvE5JND16rly8KIZ9PHWvFj0Qr2c8X/b+PLyX1jz1I8s7UEIYvfVusrxj0Ie8WTOBO0VCGLzz83e8usNLPCXR3TqZ16M6u2kbvHEAPro6Fdu75XpkPLf9ET2+s2C8h62hPLBbSLzRDA48CeIEuyupbbxkx4G7ERYouZe4Tbz8ViI7+m8yu2nmvTsInJ08TSfyPLLzRDxlL/i7NYofPEJwNDwCJIQ6Zo6QPGXRU70q44m8MCqHvEyTuzwnLwu9YNv0O/SUvzzFwwo72SHtPM9HIbu6Mro8apBPvEO60TrvIZQ79zpavAQUDr3iKtS6yPXGO3YDFr2TVJS86BtTvX/o3TxBheG8GoXFPEJxbrwIB9O8UgrdvBPbjbzhjCu6ZIyHvHu2/zqkYdc80babO3asjTsHlk48nJX/vGX0Krwor528y1s2vXYmajxdcQu8Gy4yPRt0MjxS+HS8r4E1PERw5Dtoh0Y8roSuvDNXUrsFdv88L5hVvHqNAb2ilwM8N7poPKd237wlGf+7NBj4O0WwWbtm+5y8gZ9tvGpNZDxwb5A82a8ovBdQhLuiloc8lBQrPMuMvzwg51k8PxxHvJNAs7ytzU09ZVtIPEhuBr3E9Js8LPbPPCfrM7yTrjA7JjWGPGLRFr2jfIc8673qvBxryDi5ywq8yEL5vKdq8TwBRwG8rbSXPChBiTy2e8k7UrhMOgrh2Lx5nqO7kLwBvQ8VubzpjvK8XOjtvMGeV7yeupO8ztqzvJfdqzwuBUS9NqsVOxUS2jshYha9rHAtPMqiarxZn4M7fW3nvD8ekbz6hOU82e/ovH+hEr0al8y8V2kYPTlgkryE7zW8MZkJOytwLLzBRjq8ZiCPvHDKZrzMOx87tinSPPvmJbtcTAs9ydLMvM/Xtzvzt3g7KUqyPAAZxDxNLHO8kQtlvOwcwTwDD7G8yIeeu6FHCrsYuz48p1Mnu/yrm7vYTo268mtIPBPK1DvcJIO8TzaOPIml8Dt4Zgg8J/XiPPWuKLwgsIU7wcYcPcOXKTy9iLa8ltGLuw5OizuoVjY8L0GLvINOozxM3YC8sGxsvPExTTwAEqE8vD+RvHC2HD36GFo73csZPf/x7LxCxJ887AO0vEOONbsLEb078kRFPOQ9IT1BZzc9xtIyPC+M9jwjccE7g2vJO39Ahzw6YVK8T5hXvcibKj255i+8cW5+vL14oLhpjtS4s2KvvOhgDb1VegA82jm3O3odezglHlo8ar8lPeCKj7wwdxE9UHl2vK1SzjwBOYQ8+q0WPSvcerzCt5S8e/ukPJmJNTyiT6G6gtlpPLsSYbz1PD88Xr/rvG43ADyUwyG8nFfZPFXegTtaqvA7KCubvDbkFL2KcbO89fEAvZ3D1rx+0R07EcWAvJA1aLwt9t88TfKVPHnVgjxwJ4k8H2TqPLsCQTw8pTO7n2aWORWymzyJ77i8NsKBOl9SRzzHK2C84ZcJPFAsWbscwgm6OWs1OwG86zvS/OO8ATKEuzyjk7zuiMI8kQD/PBGP+LviRfA8g3xAO/iSzTtKDzQ9TdERPGnrcryNgSK7frzcvDhh7ry3otY8P0YfO5ZQoTvdX488WwiPPM2RIDxfJpE9fCMJPKxaNbzt9CC83nNXvEMnQrsdYzK9UXzdu/PGoLzNssy83AucvKxsTDyHAJ08FAq/vDAORbovkug8++4HuVHJj7yPFLY5hNchvCm0BD1OU4u7ChpiPB+SALyidRU9z9rDPNu2ADw1SP88wQSCO8y2ijx7WRS8Za9TO2GMhLw59oG8C+mMvB1xqbzKMBu8/TXrPDRFcrrZncq6fQZ5OzcR0TuBm8u7Q6kmuvXq3bvO8PK8ulYHvGc9ZzvvNey8EHQau/Wyibt8y8G8AqdRPQnJwrzcIji8lG6Bu+uCcryNWYa8zxW6PKQoXbx/Kpo8NyVCvZZDHDzFm8i8u7ulvDT6jTxexzm8XlhAvCZyQDwhoq27zpC7POmbqbtEgjG8t3BRvCzbP7xM5cO8XccNvT2KB72cHQg8o4bcPKhnQbssUCa8mUZLPHeuIrz8S1G8xOOPvK61lLzlX7s8zGU0vA9r4DzbClM8o/6fuyLGXzruOX8851dcPGrRlbyUrXE5eXRlvLRtBLw5YNy7I147PPb/sLxI2Hq72kgsPGzZnTy+Is68PPhxOy3WUbz4afQ7yf0ePTCMxzyoHX08hWKeu9inNLwdAiS81dPXPLQqHz0HriS9gJP0PHPoZTxRaoE83BIKPWpU/DyMQ8o8R5/mvORlRDxg1nA7OswdPYNhBby6xJ+7Rj0xPalO3TuhaMe85H33OzYKZLzgDJC8c1YtvFJpnrztsbU8+gKNvDZ+3jzang48tNxevfGokzw0HBM86arIu5kB4bsZ3My87JoLvUNCDT14fjW8+FCQvKwrMD0UmGq8pYruuzJYCD0Dvcg8GmuWvB2vgLyQVqU85smGPKQ+ibsmHAQ9d+xvvOK867xSsQC9eHnSOjqBaD2nVt28jo2ovK9CDjtLpJm8/PN/OtMyFbwsVDe8k1GTvKKmtTxqjxs8RfKjvO0Ljrxg/WU8TK3CvNZYVT2IEKg8ZbKFvH20wrwyibK7YfLavMNOjbxsxO27KX1tvA/3+ztlqzc8tAKPPD38SD2+S804nMePvFAIpjtjJgI5klKbvOA6RjuNmsS7prAkO6Z7Ur1RfbQ7kHoKPJYMX7sEO9G7QSdpvFbeiryVIYS80EygOnNDvDux4vy8BDHpupEwhTy25IQ7/6bdPNsekDrpI6w7gXcFPMbWl7xX/5y8KZ5HPGSMErsqGc28GNsPvOVbgjxhmWc8F6uzO4YU9juviVk7nQ3RO4IB8Ltc56s8iwY/vGB0ejxGI+Q7KKwAPe1bqbpzcVC9qKyKOqoPrjvD1Bm9uHeNuzTBkTszY4k6bvHwvKZlqDyV/z68Q5J/PHze77vQQO67mMTCvLpmMrxz9t+8xF4mO9E3IbxLZCU8XvcrPFm5oDz6hL+8+6cUPRr/ULzda4s8Syeeuz0MpDtMGEu8t+WlPEDUaTzpEHG8DwiMPOhAlTy1ZP87Xk2ivOcb0jvJa428UKSUvKGjZLxA+iu8NjX6OwHQ8rxTtoq6CqLkO6A7jDwKZjY9poqXO+OZE70kmRU8FtvjvHok/rwJpjA8DmWWvJj7dbxWml489BlGOy35P7zUg6k8Xo/LPN4PNLwI/so8Q6SSvMBMpTyxo7I898DdO6eMiDunBVu83gTPvG572zxfUmw8mCGxvKI5rDyqZYM8FdWXu90UKbz416g7aEhUOsM1Bz0hwjC8AqqyPKttdTwZ/eG89uY2PFZ9xzwcmYC8pyMtPLEVYr0JEZo7zp29PNUN+jvvX766tw0NPHypXjwI/Tq8awq+OwYWg7uMXhm88PkRvN9oh7tOFKK8SNCLPO5+KrsTw5260nllPMJzALxwrNG7gbC0OamalbqAjWA86xuFPMyXF7z5ApS8s/2gvPGsuzwtOBY9OPZXPDIizTzTHmo8w36uOyoFJz0cwgq9I6CPvFsjKrueIwq9qew1vPAHPTx32Vc8dFerunDTITzEYLW5HhjXO7rz5DwvuGW8NSNvvDrFgzzgIRs8SAgcPDHTYLyMQd26UqNMvWpuZjrizf484AmrPFXuXLyfvW47zAeLvPXAJzyUrrm8Y6sMPLlWv7oGeg+9gjSHuTZEnLxXdek8qhAqOyfvrLultEU8P4nlOpnBVDxhWra8PI6VvLjMJzwm0PQ7vKaOuol9uTy3ajs8R4gYPbyT0LsI5hI87jk9Opa1vzu6XxS8Ai7NuzOMkTwqvQm6p/LeO/2qsTotLk68ItM5PDxVPTyr5TO9BbpZPNG4CbyWP5C7TeWhugQzxjzJO6c8JkA5vSjdZrzFx6k8lEKNvNsrjryvxQW9fS+VPBaj6rzPNoy79VVpuidxC73Xjnm8paEbPeNcnLw5jR298zOOu9xL0zqE27+6YyA6PGkpxzuXgfE8Oiweu9pYFD3muji7cX5xu7sAdTzZnsm849/pvOOmOTuM6JO8GSCEOx7VVLsiife8l9dbPODYoDwC7we9g6AIvJ58qbyJgGI8EsGVPAbtXLxz5Ic7cxD/vDGXbjxVfhY9OOd4vF0/y7uej8M8nl1NvNyhJz2vEPI8sVthvNMUnbzROgY9xVJ0vMmPNT3bw+Q8zzHAu7NaLzx3xLW7R1wjPCsVDT2xL6S8c5AAPJ4fYLw+ApC8o4kPvNUYybwOGd87f3WKu6VWjDw4oXi8YhiEO6ati7z2hkc8AvNdu/p/DjwlIUg7yQg5OT85J7y6U6E7VmFnvIPrajwUaO28CNI1PZeXoDurMZI8XY0JvUr7frzRn5O7FpCWvPQ2s7zgcoO8E9vDu1uTv7yO9TK9Y4f4PGTKkryhnRQ8saPMvAoU0zvhksA8HfUWPYElMjxuFIE8kF2NumIoED3oqUE7kK0Du5U41bzVCF+7C7KavBKp0jwa58I8MnXJuwtu2Lxl6FM8UxCbO9rMILrRHyO7+B6Lu88ai7wJ5w69hHgLvHbcI7vO3y68GURSPNQvIb2dM0M8AdSyvD4577zy5YC8BhgmvGLKtzx0zFU9aZERO4V8vzs9jFE8Q3gnum2j0zziehw7fccRvcHqhLslE0E8/RdWPKg+XrzBfLQ8O0LBvHn2tzwJ2z+8WOs8u2+1brzeZqU7rFE6vDqgW7zizY26uq8yPFtn2LwNVAG86X9YvBFT1jxcvvu8KsGUvC/woztwiUS8KPymvJv0Fb3I6sU7FPh2PHoM8rpmNMo8lnGRu5fJCT3MdkM7pXhYPIHj+bxDzJy74SHPvLclv7pjSKu8bSt1vDIMy7wNL2I8dFZIvBkB8LywxlO8XfJevL1l3jxNTIo7tcUwveJMAb1DjPC8DSOIvCS53TwdTww8u/GAu8VV+rsNma6820tOOnFmwjzBIi+9R7AMO1t1lbo59Ag8STPbPMSswzsy4wE9hYE8vHWZhDxDJs65rz9WPLEVjzx/Avw7IR00PXfSerxsBoW6NAn9PDoH6Tt+9Aq9KjG+vDF9rzzjgzY8oRuQPDGc+zwtKzg8iEh7PBI9wTzF9O27PMT2PFwrdLxsnA+91ajivBVfRLz4Agw8uIhfvIpKErwcEuC8IJb5vPvEujyy2pi8ymaTu6ILDzzaiUm8QAirutx91jzoh/U8Alj+PNKndDyfCP+7XIElvCXfKD0xY908zBEbPNR2xzrTJJw8rg+xulMxNzwzhhU8ZsykutcD5jt7bfG7E1MuuwGlRjwDjM28C+yRPBcaH7ulSaU8hzY4PA6+uryb4UY6VBFAPA/OXzyZTps8m6bnPD/CP70mY6q7BKAkO8KvM7wAOwK7+Wc0vNcJE73vm2k7STeaPLNXC7xiIdQ6ZHksPOWtirwmgZc8drBgu4BmqztXDdq7Z7Zgvc9TUTs6E/u8Y+E2uC6lmjwYDl+9Lpb6PAipz7wz7qc8CZ9PPLCUQ7yx6l68w5IfPcwcuDzteWu7JrGqu432aLzDzV29tvwoPVo3CrpSuZq8FIzzPOdThzw/E3g8zlPcuoSOt7xKhTk6hOydPAqzWzwDB7W8jq4Mve1o+jzPoe08j3VFujctyDw7Iho9pjIEPDAsr7xTe/u80P4MPAJj+DyAJMI64IE0u2IADrziHwK9RJtCvMhVzbzKuPa8Ci2oO/Zo8bvEW8E8RJ63OCzGzLsGSJQ8cLgdvJArAby7lkE68d26u8BXqDvrlWu7VRI4PNaBvrssjqm8pPsgvSL20jwI16M7ClarO5KIGLz0DpA8WOVBPPZBDLv3syk86SnBPCQ8dbsnhxy9+BHOOl54zDn3St67Mh8euI9wyzwteNU7TzodPA/zdTxCBgu8ZQcSu3vtBLwt2fC7zeXRPAe1Srzn4wy8sJU7u0Gbgzu5uS+9+g0qvNVmRz0aZqi7JoN3OybhMDw/6Yq8eJK7PJJ5Sjq/ayq9jMWDvOqPUDwacg+8lrpZO31USzxEXei7ORYQvLoolrxauwG9YiuNPB7vBLoQSCC8sR21u6mjCr0agb+8uBlXvEjALrwdIwK9iI8ot3jA7DwP97q8TpHMu8IYhjxUlSc8wSqwO2126rwlkUQ71xxmuzfzKToQBVY8Sx7qO0RlCDtwny07zkbNO6scVrzhM/Y7Cge9PEU6Qjzy/9K7WRqpuuig7byIMfK7xkw/PT+KHLzZ3S08lMrNPFEPEb0oqK28b1g/O/LCAT33JMM8OCiovHUU5jupyHW8MtRDOkNTmTxuZi289jCHvIWp0LzHsoC6ZcyMOpqEiTyx/Na8UIxCvNcN6LvHJhW86OWAvO26VbyaisA84cIkvCyIdDwQxoi8t78CPEaW/jsnNYg7IXb3O9P7Yzyjcmm8JxxnPG811jwaC6q8ZBYdOyt+9Twv1gE9g0IYPNVOqzuV7to5g+btOqjWVLwuUL47qJXLO6BeaL2Dwwk93RgIPbbL2rvib0o8tOQPPF08BzyzPNY6dDCuOygAlLwDAG47hhapu6W2/rykXGC7SC4KvbRaqjvBWdc73ZuVPJeF0Dupb1s8yjClvC1p9LuJEfi7Km90vGpBWryVh4u7GQfuOx5U3bzBoGY5qHtvO5jYILuLdti78wYRPL478TyA5jE9SQzSO76jKLz/KjY9ofo5vAZyqzwjJ8C88YMDPJsMKryB3ve8II+5O3egK72suwA9S77tvIuPmjtj7Io82ntPvFNEXz3rPX+8qgp1PDWNVjvmRNm85uY9umpT4Lw4nlY8mI84uYmlMDwTv8s8f932vB19kjx0mVi53i6rPKUk8Ly2r90886xuvHU/9TyfrE08ODgfPJ8M/zyEoaA8OikuPP9fRjx+rAE8GiCJOZ9j9DrPhJs88RQLPbApvrzTD0Y8BJr6u0/rgDxOCwe9164bO7z2bL3r/r27GklxPPvkLrx8AtG865sfvKTxiLv74748I0S7vGIDxbxm3aS7zniQvN3Y87ordRo5AXynvKQlGjy+daY82v/7OzUiHjz1RXm8ws9tPMlE0Tw0QAa88BlIPHFPFryPP+e883AMvI/rtby9tYW8NNAhPOIuYLx1H3c83zueujqOnzz22Ag7yaiQPNYFqbwhKRK8GKIUPYriprnJKLE81RtKvLYTg7wPHRS9FJKvvHTSarxGUga80bwzvKRAorxXnhc90+zsO9rjhTzkpNA8IQkkvHRWhDwMJLE8jUmZvL6/zDsllxm8lzKFPA2KKjs2DRe7+/yFvDRH2TyyNRy99OP/OxX1EL0hgyu8sFCbvKHJhTvyvQm8AJ+nu6L+hryg1Uk86Z2YvHQ8Dz2EIxA6QGravAyRhDsbAZ+7E7sKPCsnCTwBLrY84oDVO+Nyt7to9nM8TSE9vCN7z7tKNfO8n/qTvEX6XbzYNRy8TAEIvPzTzDzXSBK7nVGbvJNHhTx2ZLQ8saRaPCTYhDveCCa895AcO1t7srw+Ad68YJEBPCA1kjtkE1K8HUkLvK6G4zyP3G674rKgOwD6RTzVN3m8/fEbvZJOy7qMz9k79+37uxjxKTxHi/Q7Cp94PBkhYzzJMJE87sXSPLyuCr0GoFg8ibMAPA0J0rwLwIi84ZEmuu8EjjvkXjQ8IIMoPWXjB7wY6P88iEu0PNveQLxfV326aaEKPXQ2hzx3AEq8pMrsPNSuLT0xtKG7gJUTvWJ7ITx40M+7iJqHvBgCEzshn2q8xnPdPAYBFrwHHYw8G1AWPUpzN7wO9pK879NZPM76Nbxo6My8n/uvPOenITy1L788FIeXO0RCkLvSOZS8R3R/vKcIiztRWhE7i+ubPBrp7brUnmm8/S2uvDiB0rqDf9O8DvqSvEkcIT1zgcE8xbEqvc97N7wCHBS9WvBCuo3Y8TqN1MI7G8nRu5DIKr1C09G8hcgePET86jxstig8jBWcO1IZLbyqwrk74LbAOvC5n7tmHco7t1sDvJ7QwryjmQq8rqtGvPoz8zw6a7S6AjBIO7THCz1e+Yc8+bTsO+1dfLu/RSS80Uu7PHuvlTySMte8gvxIvAi9izzBFVm8RZZLPCDTrbuhcnQ8QndpPP0zSzx8iCc8QbGlPHvh1DvUOS48o9mhPLoT47tB2ww8giLMvHG66TuqAsg8climPEHaLj1wzj48OfXyuuQBrbzTe5e8oIVKO3QZHLucgbm8p4MvPAAncjpmS7Y6aPYkPa3w3bu4yz67L4cevASUPrxqOrE8fO8PvR9MU7sYpIm7ytunO2mQ0Lj3DKI8oBKXOdapAjz7jyQ8/HLXvK+mNDxSR9846PrfPAUecrv7tlc8gn4nvJIAnTxFojK8X0xUvCfrsDuqLUE8dhgKvLgZzLzvgjo8mhiXPJrVgLziO988FvhEvLKBlzy9HlE7faYXPFeFi7wdoxW95xb+O5G45jvk55A8Dy6HvPSgILyHVuc7oZNvvFL8gLwGVCi8Mf0+vEoDgTyjWBG8fWkEPDZNGbwOzbi8p0EAPWsWVjwtDKc8MGGsvN+YV7zggAG6HlQHvIpr0rvcMnI8yYB1u0QCkbxyIGy8EA0YvX+udLyTUja7RYvuO0DQq7wVL0E75m26PBfrWjrj7VA8QeH+vIxo/7ztmKY7kZezvA== + index: 0 + object: embedding + model: qwen3-embedding:4b + object: list + usage: + prompt_tokens: 5 + total_tokens: 5 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_s3_source.py b/tests/test_s3_source.py index 061accac..41688dd2 100644 --- a/tests/test_s3_source.py +++ b/tests/test_s3_source.py @@ -66,6 +66,7 @@ def test_make_s3_store_missing_obstore_raises_actionable_error(monkeypatch): @pytest.mark.asyncio +@pytest.mark.vcr() async def test_create_document_from_s3_new(fake_obstore_io, temp_db_path): head_async, get_async = fake_obstore_io text = b"S3 hosted content" @@ -84,6 +85,7 @@ async def test_create_document_from_s3_new(fake_obstore_io, temp_db_path): @pytest.mark.asyncio +@pytest.mark.vcr() async def test_create_document_from_s3_skips_when_etag_unchanged( fake_obstore_io, temp_db_path ): @@ -106,6 +108,7 @@ async def test_create_document_from_s3_skips_when_etag_unchanged( @pytest.mark.asyncio +@pytest.mark.vcr() async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk( fake_obstore_io, temp_db_path ): @@ -137,6 +140,7 @@ async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk( @pytest.mark.asyncio +@pytest.mark.vcr() async def test_create_document_from_s3_etag_changed_md5_changed_rechunks( fake_obstore_io, temp_db_path ): From 9f8c9fe3b1faa4ab4dfcc68680c6dc1125eb888f Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 5 May 2026 10:42:57 +0300 Subject: [PATCH 7/7] cover s3 uri override, unsupported extension, watcher error path --- ..._create_document_from_s3_uri_override.yaml | 42 +++++++++++++++++++ tests/test_s3_monitor.py | 31 ++++++++++++++ tests/test_s3_source.py | 34 +++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 tests/cassettes/test_s3_source/test_create_document_from_s3_uri_override.yaml diff --git a/tests/cassettes/test_s3_source/test_create_document_from_s3_uri_override.yaml b/tests/cassettes/test_s3_source/test_create_document_from_s3_uri_override.yaml new file mode 100644 index 00000000..b564c8c9 --- /dev/null +++ b/tests/cassettes/test_s3_source/test_create_document_from_s3_uri_override.yaml @@ -0,0 +1,42 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '93' + content-type: + - application/json + host: + - localhost:11434 + method: POST + parsed_body: + encoding_format: base64 + input: + - override target content + model: qwen3-embedding:4b + uri: http://localhost:11434/v1/embeddings + response: + headers: + content-type: + - application/json + transfer-encoding: + - chunked + parsed_body: + data: + - embedding: 3+sUuYLH4jznrKk8fYDHvL2dfLrMgiY9o+4sPdlAEr0N7ic8tJETO4zV9rx6ZwQ9NZaDOfK6qztQ4xy9XgwFvbCtH726+T871KbSO2lFQrsxXMa8mZ83O5k//jxxuoq8SrbgOx54rDwuPLi8NIGXvc90mjy+72c7rxdSvGQQeb30rag8xJXVurdkyju3C4W8OBUcPM6NG7x3JZO8PxRcPJiYqzufPJm8aMI1PK1kqTrQqJY7MoQDPec3RTvqgIA86XtWvMZo4Lv79x48Oa8IPObpGzwVc/K8/Ybsu2w+bDwc3Au7Vqq7OtV75br8BDQ8mfHSus1XtTpDdPa6DCvbvG6En7usn228A9ANvH6j0TpU+/Q7SYB6u+LtUDz9shQ8gpRjO6pRLD1Iwz+72vUpvXTFUrsXhDy5IzuzPEPg4zsD2vy7TZ2CvPB6n7tRSX886c5jPBFaWrzlwXc8If2puI4yATy39tm7Id/UPFjua7siKck78xeGvEl+tDqYCAa8lWq5vLgeB7zluAu94wUSOzJsLbsli9c7825NPbnh6LoxeWC8yD2zvLLe8bw5jow8Vf+CO6TBH7zcips61Qr1ukwelbqZ4ZS8WFbKvOzptLydVLm7jLoPPCKTsjzY/LQ8OLtEuyGqDD2YZXA85PZdvD+MN7tEaME80+B8PKmYDLzzmOu8bIGjPDMeED24Eu472BLPOjxRy7zR1nY8BQjru3XtCrxwkQy6WJkrPPoXCzv6BjO8hQMyuzLLMzwWTEa8wt2TPJ7o4DsTUbs7AAbmPIj+hDtBpnM8kquGPFhyz7zVHB27zR4jPCRwQruwUiM9M+Wzu2SOLjzuqWE8KXY4PHszfrzyfY88hvyrvOlbQz1vmEu8Fx+BOV/murz1VjG7Ged0u4sgbrpXwjC7QVNpvP8c/zuCT4S8HlUOvQ9naLxAOMK8XrXrvEGLNrsvIyA6lyk7vC8DVzxEvqo8ScGOvFTFWDxbS647zi0KPHkg7DvgBHk84cccublQdDw5Nfu8WJLSOlE12TrXiB68q/l2u+Bko7xcSci7g59EvXhQxDxc9TU8+OYPu55Iej2NL547pHJAPN3E5rvTr447zpl5O8iVJjr8KxO9ELCFPHNf6TsMwyG85XR/vIJ11Dthi1o8KdCYvCjPrru70Ow8TvofPWFKHzz6Zlw7qwjFvN5XGTxfRjG9KUmuPCGibrsWXta7MtOpPCRa7DvFtz28zc4EPINy6bocWI68vQBNPL9nyjsxqyE8MO48vO+7Orz/t7u8ttiWPLn0JDyzne88HTPMPLnrqTsj2QC849ovOmQcxbxhCxe8UCTKvDuQPbzxOjq8qhQSu+SBuLstfl+8mVjRO1djv7sEecw7rw8XPK9aIDy/67A8Bc2sPFV3IzoVYZa7b2vRO09Tr7zC40W8fJhhPHisfLyzSla73n83PTxnlbw2qLW6VK5FPMKUwLvHrTc8TKAvOs1XnLtG/e48RkhaPFjSKbr1zum4JEf1vNixWTosbg08uGcLvN/+JLyvlnI8t8KhuxwTpDpeXbY8xKVivEh4pzw+FZK8hRo5PfvcZLtvszw8Shy9Oib5DLy8AYa8LyokvMn4nbxEYz692Ur6vKZvALr3qc48tkvkO7L8kTxWho686ImZPF+oSzz8UMO8VtwMuvxIOLwh5pA5Vu9VvZ56xbl5Wak7srAFvQaegrxAiuY6W/rDvFUSsbuCtqK7mqACvfVXjzvrp+A8/x82vJQ3lLzRDJU8l+Y0vXweIj0sQaQ7usrduwcQErpb+5a8SXAEumj5ST1wEdg7y1W9O+e+8DuH25s6ltkcPIwT9Lsbn4O8rP6tvEVWfD1uH6a7DJyFvFGSBz0TE3Q8S0dsvMN+R71Egrm85o/wO8jRwDyupT696w0UPKObsDs1xQy9CiqqvPkftTwICXk8mmxnPM+1y7z+0xK8TwrzvCJGwzxdcTY82K3vvI6PuDzyVpA8c/wXPTHs+DzNPXc8FoBBPASBfTv5P6K8x0SAvNQ4Cj1K0888Lx4EvY8I17yb7+E8GeTAPCTSszzzIle8meooPDAf1jxsncQ8TpDsvFOB8btOIys8pIiyvOxj6ruANII82wurPOHUcbdvSsw7lQaKu6szBzvXjKu8OrDSvIiTejy3wzW8nqNivFbP1TwMty09cFnLPK48prtFfPS8a2KIPFbrBDwAOMm8g/dAu1fEiTysopu7B8Y0vCPyzrzvuMI7zPCNu8MWorxlWZ47H7hjOyPdibzPyOQ7hdIXPEbPNT1hT6O84vFNvJ61yzud9kA90zEqu9HLRj2lvS68PDA9Oj4ngryQPhG8dvJIuf6bmbwkj0i7cuGwvJ7/pznF26w8tQtpO+PFpbzs9w09zjMoPZT2Y7s4iTi88Gs/OnxYSjyMKHY8Usm6u7iJFT26Sr88fjJku1yk3bwfc4m8nOsRvfqUoL2B8a87krafPLjc4jvOql27yxLFvJdZtzzSSy28mBJKOy+SHj2F8HC6uTrTvBLjQrsMewI9oyOqOt9DPzx6YJo7/UBevJP/Pz36kLi8EO8XPXIPMr3S0T48pSmNPGf3fDwot+48KVktPa+b6jzKbEi7uGJqvA4EyLsSdik9gWlbPHYh/rvobDa7fTaPvDgCZTzIwNm8QQRJu7BACT2bGUK8L3ETPOBkWzv6/Qy8WnRBvHYqyTyHoSo8dmZwvNMgyzxgLMI8l7NKPECYT7y+CQ+9o6N6vSbHczwbRHy81ULSu2KtQb3ISrE4fHgTPExKebwmf6c8/Zq9PHxClbyv4OK8teXkPIVzIT2vIws9Fja4PLsdpjxz5/a67iudPL32RD3+ujA8bsYdvftIXbzehYu6mwKPPLwAkzziz/m7veQ+u4wFZDxKpGm89A5TvGhhL7vG8Ni8WPt7O3q+ojxIYBm9zaO1PIAJgzx02ha8JhyOPOEuLTzVf+O8hSG+PEq8sTzZRtm7XGg3vNMQKDz4xg08X4mZvMH2XDtaTpo8GVYYPTiMvLvCQq67f06VPLxqs7zXPAU7kUc4vFql3TyEhac8RuIjPCbCKDtXrIO8TXnAumzdyrwwlV+991OzO1Yti7ywvLY8l2GBPEI3hLvJYB0859oZPeeiRzwf/Ce874/SPFpM3LwzlIO805gvuwuiIDzgd5u7JyZFO1gphbwbdFy8Jxb2vE/HkbyWfsG7460NPV5kmLymwfm8eOefO4/icrshnZK8kKQNPegfnTzN1E69aU6PPJFEu7vF49y6YMg5vC4k1zsblGO8mjEHvUSIA7ui4Ki87pGhu1/siLxAIAS8n1soPFelVzyJQa68m5xxOnOtzry+tcO7CbRtvMoytzzHJxg9bt4avI1ohjsccQw9UAzgvFneLTzxHDG8LuxrvVgBurtTSKq7TYYYvVzX1TvVagw94aPAPLLrWTugjuM76EfwOxeCWb22jPS8ycYGvalwsrtznBq94SxVPJkzhzwoKXC8/vNSPLB6qLw5Xxc9dIH8vKLtxTxezeu8y2abvAsNG73e82S8yuqAvGOaSLwvyCu9Cgkfvd4pSrzbi8U85HSKPCZyXDwOsA48Gcg/u7HjiTw3uuy7SSoEvdEvfbob4487sJ4jPHHbxTupZyS8+UPYPLvWfjy5LiK6QQqVPNZ8mTwZUEG76CMPPUx7vzsSR6o8mYoRPB+he7xMqqk8116iPMpfQTyUev27beYVPWKM97wB+507Y3ZIvBSRDzr/M7O6wwPCPDa8QLphK6G8PQK3PPLuGrwQDxy80jlmPGBdbz0JNow8A6PavG6TY7tI6Ki8iFOUu74qFzw5XJO807EPPDNAprzl9h682TGavLp3NjxHhC68oVMovO9swrzzZBY952ODPHxjMLwNn7C8ANEAPXJcrDwrVqw8rIRMvJ44Mz0LqIO8yficPByd+jvATJq7cgFCPIAvoTzxHQ07MnxNPGRP1DvC+AM92IRXvX8GD7qAOeA6p2/Uu5R+Aj0oD0W85+iJPKxUsDz99iU9A6Wzudp6jTzOKmc9wNJ4PG4pBT0hH8I8IydfO+DpNz2rEES8isusPM7Py7sy5Y28yzvKOj/I4zx29/a8H5eDvJtBDz1Oles7kk+zPH/NEz33QGw8ZogCPcoT57om+6m8pCpxvG4uUjzvLfg76x2kPKBv3rzyB1g7zUfwuxq1t7zHcTQ775qmvO1Pv7wFphi84Xy5vJ5LgDyFUJ06bz5lO1zVA7zpFAC8AmBjPJSTorwI8Wu8nQMovQLCETywLIA7vzdLvEuSi7zB+nQ8/hNbO11GnDzQE8U8oyTWu2IFHjsIr0G89m7tPEHIJb0NEsM7YbOyPDnHmDvx5Lq8jhDLPAppJbwRfno8uaQFvYR78zuv/EY81Ty1O9cJvDtESom8u38YuzPPNT2Nnqe7IsL3vI64KTxmQ/q7cKETvEpg3TwXNPc61ThJvLe0Frzi1tO5LEpdvQFqdLylpbA7efC6u+TwPjxYPvw8zLM8vJArzbwWMj28APaROzQ7sTyF+dW89tI9PLszuDvKurY8XFgcPSF9N7u6/Io8p5/FPOC0Cj3dCti7vwArPAFN/rvp4qw7u9bAvI0OCrz9L4Y8ssJnu+KJ7bwuzhS87FCgO7r81DyKqJK8twDOO6kBrbwa9bc8emVIu4rFxzqidv88H+bfvCLi3jxwhwi8wGa0vEug97mydCK8oRpxPCitC73VreQ7NhzNPIWJJTxxotg8JO4mvOj1D7x6XSO89cENPdiaRjxIP5o7ByMMvMSyjTx2BIa8tn7tvDd1vTzFJbq8r5moOxzmBzyqfFa8m16WPOxRBDvDXe07eIuEPN/6FDyTVJ28OnEAPPKFmLw1afc7bpGkPJ7eKbkZfFG9DvCNvBXagLz3XFY8WckjPDjmKDxuYB49tB3RtRT+Gr0QUe27jBQnPafhK7wm9UM8kiS0u2w+xbdYLaW7XPrrvAcSSrwh2fC83GwWvOat+7zKM4O8OJ8IPHOChzyEVRY84QXMu1ygs7tCIIW6/LxKvD9j3jtgnQ68cgeTu6YOljzl+By8nOAAvGGqRzyPbkU8XAz6vOKBc73Uxqg8EAy/PMryFb2QoCE8LHZnOx1hrjqeK6Y47MGvvKt8pTqZ2YK7dj01PMvcwDxV8ia7YdgDPA/kQby7yKy7G53ovKEh7zrmK9g8oZ5/O1VO0zsqGjG8PE5fPA6JtztfLw89hrXYPM3bIDzGPHa8jisOPUnBL7pwbpW8TAG/uWPHa7z0aQm86LAcvYXdUjsSZL28cBxku2b9DTz/j+m8sqjyO1XCtruAeVk8jwULvYW9s7tcs5i7h6h3PB5T1bxfyus7qz7pPEXp7buJKpK8rcbVvN5/PjwJymC5Dn5KPeSdoLy3jfm87sCpvMk3xjtX8V+85Bjku6rwBTxDWti8/KzEPGMU6ryHR0u7J5Tyu4mSa7xsPJS8jcd7vEOs1bvw9Z+8vbYIuwVyiruZpAy8RCrwO7VYizxXTfO82OqovCcdbDzTZq88sTM7vBPE6bxy3S06Utnmu8qd7zxWn+A7IRb0PDyos7sPyIg5pssivL6GC7xz5pw78tC6O5s/ErxoY0Q7r0cWPMJHRjwV4fM71qvju0jQgbxK/+k8M66Mu7oKA71sikw8+y5DPPG4lzxkeUi8qbeKPAath7yuv7E88mBRvNaEbDz/kNq7Jr33vBiMjTuxd+07v/AovOXJmzzDLNQ8cl93uvsiQjzBsCW8U0eeOzsNMb3c1M28bbgLvTUmtDxxCLy7vMpKvEFIDTujD9S88PpOu+zciLwR6fu8Ty2ZPCpA2jr1oCA8ajwwvZqeobzGVfQ84P2LOgglR7wb28C8t/XBPEUpAzsCcFc8CYiEvO/AmjxkKEm8VmNNvMVLCrxa2p26XxEFPRYjuDuLdZs8KmNQveAKwDzHR8m6V0jMu23nmLtcZNU64wlgO8lLrDx4ArS8AUt+O3jffLwelx8773a9POKmBD2lCzG8Ij55PNkRXDzuFEa8P5bru89kvzyof8+5FYbVPHf8RzuBEpA8BlbiPDUkSzymrxa9HuEsvBh7L7vnkL67oSYqPPuclzpmFLe8n9wHvC6rlTz6fv88Ba36uzT4bTwiQ3O8VkqMPNR5Przd54Q8gbhgvFY2xjxzUS2803GPOkMJwTyL2zM9mI7BvB2+RTxUSdM7JN0mPAvuBj00w8e8ZNHUvCW6mjzApte80Zgmu1N3x7sv+1072341vAuZqLxmUWo87yx7vG9IV7zNy4S8rwMBPYZrOjsaUL88xxUivFiqhLw40h28ZCyhPDJIxLz/ajG9QjfmOQuNYTxlNwY9hlcyu2VbkLwfA6o8BPH7vNOhgjqG6UC7OJL6PJ1dCL23A468F4NyOfOJ+ryI4pO8PeZmvMMgorwfUbG79KwuPMld4bsFWrg8CD13PBvyq7sYgq88cTlvPUY1ILwMFkW83jWYvBWgLzy+QKq8iViLPJHXPbxiwLY7CKTYPA1z0zoU0yS81mrfvL0etznVINK8ydM6uugC8buA8pk8/0WWPERgIDwLPm884wSavOQDBTwu0GQ9n9J2PMuV57xhTtc8V4J/vKWVoLzxQg+8Wcr8vFVVsjyo5ua7nX5OPNjITbvXjQc9d5oOPdGoDb0UgXO75LZHvOPDEbrhXBC9Ysqau54WGb3+kSy89Z/gumQPjjwN0Zg87yn0vKNWhDyPevs8yI4yu5+JorzDvxy8We+TvC7qtTyUvqG8CtDPPHQXqzykXMg8ESAfu0zhDT2H6YW6PGO/PFxgGTwkXN28DaVYPCQ/r7yRGpO8atsXvYOCPjzjWr28OpVIOyKsfTwVLsQ79W79PMA2krueGhC8kvQrPNzWx7wQLba837+TPFMLuDz8FnS8DK8/OUm6XTzq5du7Rx+APLURBrw6uBW7QSajO4dXA7zieBu8TFsWOPczozyoGVM7lQeZPBsCt7uVqbM83x7ivLqBqDwvhCU8ihILPKOefjy/ACK7tLFWvGYphDzkGau8acPrvLkdVbzjJJ+7xfz4vDHynbySg5g7kEs1u66oYjwOXow8D+8FPfFRvLwKFK881E6WvGTM0LyqgAI9PuJ8vEsxqzxz6Cq6gLEJPOffaLvpr6A81aa/PNqFkLtr4AG8d/FUveqvC723RpY8id0RPWkmETuGGLy8M6C2O4763zwe9W68kXg3vEvz1jvp9kS7FHZHPGsVp7sxRC48eddpPCu6OTyUXSI7fZE8PPRe+zwLVFs6S9TNuocgU7sbxGw8vOcRPbqtDz2qoqO8xzuIOwZXOTwri0W7XesKPWN4/juzFz08NeRePKIDzzvOFTK7/uoNvGuuDD2oTea8IbKlu9v4YLyRnK86ltPrO5XRWjtlWnY8sFMIvQOlAj039Qy8FlZ+vH/ycznN23q8D7ItvbdkCj0vrM88jOs/O88UCD1foxq9Vwd4vImAEjvdORE8+M7ku4sqK70qO/C7FRqHPFyz8rzjbye88ZDkO0wr0bw9hqi8C3esvAm0DT0z8oU5SXehO9tquzwENBW8so08vG3PprtZIP+61zOrvP1zlTyWqqg7OczYu7o8u7zolaq6ZSVwvJROMz1sdRw9gX5pPINQy7tCcaE8ojy+u0O50TzdYp27e4R2PGN90jxpkNY73+YFPJ7DGjwkZTI8vG44u42aUjxSeuE80lQRvLBY/7rWQiG8v1hJvAq6O7trB787VJ7hvHIcGL0VIBk8zye1vHHKTzwEfac7ixIEPY2svDxxgq+7Oe81PHRiODwszsq8voxAPSPvHLyTWwk96mjlPK1sBL0SWae8cH3iPJNBGDurTKy8eAZ6vDmuIrwHjME8LcdXu2+Wfjydzac8ACLdOVqGG7zlJ4U8QVAevKY0vDypAwk8mLyfusnZ2Tt2V5m88GExPCbUerxsyyQ73y/BvGwp0Ly2tZ67wzaSvGuc0Ty9h7u7RiZSO/qLU7zh8G28YFwvvXGj+LtwLxS9x7c4PIeGELwzf7c76/LAu/YHhjw4g2K9dfS3PJmZSbvWZZW86AHXux7ZBjxP5SM81HMjN6sxmLotXVO9QEFeOzOU1jySjia8+nZmvIHqQrvtu2q85q/XOYI4Sjt+bQc8YQiIPKVsQLul+lE8RiPTOlg3gjyqmJI8mmtYPHQOVb3mn3o88Rj2OtM3BLzex5I8s23Tu11exrvrjPs5uZMCvAD8pTwUkQO8Y4Tgu2etVDzNNKI8aUWlupz7FT1Ys7o8wkYgveIjFzsvhcK7f3ebvK7j87yUvh08Z94DvFnSpTxxoUU9FJyPPIhsm7w7YWe8DkP0u/xvkT337e28C5EFvJDgYrtE9fm8VcKWPDed3zsV1Rm9IWdEPAx1rbv6Qe47Mix+u3hQ3jzBmWG8cxf0OqIzjjw6n0Q7UsblOwSKVjzyhaK8IxlbvLIgUzwxDPS8VfD6PLUpDrzLw988T4AKvI+D2TiqnES8ud5XOkrs1jzPiBQ91D1tPJ2jmbzjxQq8z+2KPBooAjz8I5k8VKOQvOjSC7yDKNi8+s0ivFF5ODxhAv68aKmUuyyizrzmCiy96jBSvOKZXzwr5+c6Kdy1u/aHlrwaFAa8IjlpPPvv6TxUDs68L1DSPOGyCjuJc5A8vjlyPACMWDwgkr+79n7AvFgamTyKSSk7ZDzZux1XqLzrFkK8z8pfumb6+jybDyC9wW81PDrXgjzynqO8auMSvA1EjLs3Ogs9t9+5OthC1TyOptu6jS4SvfmOqzwuiXs81M/KvCqsOjxrvgC7Dbbcu+E3uzvFSgY9CFVFPE4uzrsVeUA8bZ/8Oy37HzxJOVW7MapevKWLrDzOVZe8LJ2DPHh/GTpW78C8n03JOzgkgDsAbwG9MXlePKSEODxxLoG7zGShuxs+3zy35gE8pcgtvW/t5zsyCr48BxrDOnN/yLzq3ui7MoCOPKEylrycqhk7qfKWvOEkQ7xAvE08UO9ePP2v5zmnfD69DMPGu8WkJDypWew7SRW3O9We5Lx/Bu07KMO0vFgZvDwZQny4c95AvAXoJD10YJi8BX3gvKNMj7ywv8M75M8OPaRwcTzxvGY8bCqTPCybpTuBc8i8bVK3vE7RqLzokVc8Z+ilux1LPLx5e6y8IjMyvUEYQjv8Y0M9oobNu6Y0Cbz7bXM8PgrtvIVCGD2H6gg94om+vI7qIbxUs1M8LbTuOk9bRjxoGK08OZDaPMON4DvTmdi6jVUqPcfZrjxkMRa9J3b1O1zBt7zXhP+8hxNSvIsVQbxCMw07AZOQuzA5vztszPC7YBkWPPGxLz1dZLW740qIvL0q/zxJ3ee7PEMXPWyXMTsEpLQ8ekw5vOPj3LzY9Us8EU8KPbpQ27u1wsc8TSzEvL5AjDlxsjs8xXO7vGBymrw8rZQ8bu/Gu/o8gztKOdS8hmgIPXmdYbyKYMw8GGouvKWeJj00rMs8eIGqPHWYSjxURrI7zpzpu1coKD1VyK68eDVlvENINTxNtiq9iNdCvNW/CT2gb7Y8/k4DvHhllrwbjoi85NTBO3BV4LuSPAg8XnINOySkUbvxANa8RUmHPIOfHTyEUIi83f7GPGPmO7wVQ/E6mk62Ot1KJ72/uVK8envkvAsLNT1OBNA8xUFBvGFfTLsNAj+6wEsjvAg3Vjz1Khw9D5/yvBNbkjxBK7g72ff6PL/WTbqjLD07wWMjujYc7Dv5CL67PkqmvKU7brwGUr67nt0bPAoHa7y9ZQ87UAlDvCimI7xJPgk95FAKvagJSzzgQri8WPsNvWTXmDx0gfS7XCWwvDF6RLsqYII7lmVOvAxS17tDBg882MHsvH/QoLoMrWm8p/8Gu/PMNrxrvBg97a74vIa12TxgbBw8lJVQvBkf0bu4p9i8TL/7PBU/jTyP29o8+uQtO+kslbzbBr87DQfEvLEgcrwuDL67uRn/O5G7KTvXKHm8s9yzu72X1DsTKh68o7Tau3wFpTzHxPC8cy0FPCpjC7smeKC8Lk5ZPEailDw4Ttk7ezEyvIw8FLyCttS8WKoEPWY5TTsLtyw8qrGYPLcLbrwILX48kysNPfrgkjylbYk7Acf1u9HFOjuUTwu8BhUBPKDmwDzG6u26LLVnvMU/vDxtxju6FSfFPG4i6TuyESC9j2YKvYSYRrwieG+71AUrO2rkfDrYx+K84jwmvVqzAjzAVG48WP4yPACE7Ds8fQi6k1YEvEQBNzsUtCk74haaPEXoBT2ZFh28CK3UvBZ+FjxvbA47mPwwvMo3ED2lkhw88Ru1vDU6Trxooh88kuL8u/T76zoqzDC8NTWLPCuGGrzCdSg7q7oFPBD4NzzoYgS8e0iMO/jr1zzB73g7F26RPH9GibvLN5O8ALK7OxNMPr143Oy7LTs5O0KYHrzbZkE8hRrLOaKfUrw3/Xy8bliJvLGdAz2N4IE7LRuzuz5pG72VraE73IjPPCwXZDysir67AGklvSMY4LxlmJW85HakO3G3SjxqJBG9nvhQO7SRkbzk2Zc8v38RPfQ8jrumRr+7ibuhuxPJtjy4FIK74l2KvOtVhLxGJ0G9/SdGPTbd4ruI5pm7FNavPDc2Fj1ZDKq8+ak7vOHorrx6jxu9mfkjPfU0sTviVBC81yZoPMBi8zxJ+EE9zVFdPKMUODxIkTU8hnouu9/YSDzLXvu8hOIrvK2lijx9fg88Q+rqO8ME8rur/gu9TYAtOQlKfbzFxkO8PaGRvHYaf7wS4I88OVb9O/jSoLyww+O6mYPVPM9dobsvIua8vXeyu9y6A7yLw5g6VGuZu8g18jyyXLO8j8fGu+lAVLyjAge8X6V6vFwhfrwbNkA7wk9hvOSjvDojB3y8xZ5/PFIijbx7zQi9RLg7vJqBwzy2+yS84W7hO/SubzyD3Nu7UQOtPJEsIrxTQJG8NFsVvLLb1Lyz8nO8aKZbO2jAIbsMFKe7sKSDvCmU+jtapgu9U40AveqIXzww26O6cRDuu0+01TuVCi+8UTu0PLcIvDy2MwG9V/SLvOVrwDvd75m8R0ZPvIBKUjyVFNm7saWpvComNjzCQs28Mv8CPHvxJrsTVSc4WlfuvAN7Br3+4Bi94+cjOygQGbxthQe9LojquzqtmLzHRtS7cjRPvBcNKD0LSCq81Hu2vOfu2rytwX88K6aNPCVRczqfjmy74IzuO1YwIz2RzJG84UpsPD9s6run3ge6ltXCPOEnpLtP6nC8yDRvPOzaJbw3GJc7zZrmPHyGk7xxg8c7na6evKeXO71NJoE71uNdO3ydNj2+P2k8W9YUvdaO5Twl/xg8HsIMvFzSDT08bli8tyTVvCzZxbxWb6k8H5UpPdcu7jzx9bm84VYyvF5gNL26yq68JqWwvOwknTqLMQU9X5yhu0rQyDyp8eS8jLW6PKAnCrrnp1U84iuHPHjjKzoGM5O82gGaOo/wczxZW5K7laMgvNIvezycWp68l1FMvPio6Du58XW88q15PL3ZsLwUHTe8v8SWO9oNlrzHBwA9QL1aPFRf8Tukohk8Y+oAPY+NAzv0i3w8ziBfvAHXnDx1Tfy8WlJ0vCz6M70sD5W82UqgvAjCtbwGpeA7q406PSgZgTsS3EM77H/tvHjipbzzDc08/PoWvTrInLsoK+W8eNPgPIfm5Tv70pS8VH06O/O3t7zjkYg85o2sPCfoej2+5IE8bNiDPLkprDtZtG89tT8svQ4qVbo2cPU8gF2JPFKhwLs4QdW8Jsztu3fFiLzhyi+6JX1NvESrjTy9ORM8sMSfO3mnoDwshA297g62PDo0qLydgJ67NfnJO0hlBb32NCg9W8ysPHMNkjx/1wM9rOV7vAhrzby7ToY88ietOeZDAb0kjwc8InIUPL8G4zvURHi8iu6KvDuGMzwzUxs9FKyYPMQTirt1HdO6aHQFvH1+C7tGBpw8SnnePP5cqbwPVu88fenAurNBBj3praU7jOf2PHheK73YZ3m8s0azPKHlo7oxTZo7YEe5vOfp4bwvBey7/A7UvGCLY7xu0G480eCeupnh8Ls1GuE8e258vE24hLyx5Cw91iVGvFq4CbxlXUW8fF7iPNvOSzzgLPG6OFsxvC4qJ7zTtia8w+hnO9PRg7x6Uhc8gtokPHXATTtCveI8jyk+PUJmQT07AO673NKNPJ0hljv4M0i7TCDlPPPeuTkzyFs8RwrEPLjA0Lyrxxe9uoMJvDoMHDu7qJG8US1JvPOPPLytfYM8gNUDPIqhnTkk07y70eRjPF6LHTvYlXy8LhervCEU2TvqyeK6xwmUOzr7ejzB5J28Km8YvDdfizym4i69FqaAu9mJqrwkRzM8H4hNPB8V6ro20zA6MtaavOHC9ry6tbG8M407vPDYwDxYJ+I7PfYgO4TGsTzBMsc87OFWPJ+wVzxRMX68ZjnevAjMqDyaFqo67Pt+vL4aNDw4rxq92WM6vHCOKrzV4oC6JwQdvQJnAj1gDPU8QrTgOvFE9LtZZC08+976u2kHNLv4kuc7NoAevJMEhzwzqb28Qvg3O7u2gbwIRpG83pS7PEdar7dr0lS8ZVdXvALoZrwwO0W7qgqUvHR7BD2Hwra7nksbvNfqAz2Xoh69kmOKPNV3qLuPIhO8Dq7QPE52Dbzm1So92slqPLXAnrsCNHg8f4EIO6dZAr2xXJk8gpYhPZmrlbxjSyU8DrRyO1FyPrwJzY08YyNOPJX5Fz1vAIY8qwATPMW+ursdIlu62DHIvH0U5Tz401y8pjBbvHz+GD11i6A8lM6sO1pP4DtKZkg8vZc9PJG5RjzU2fs79uBEPPA4fbu+Gb08f2VvPCP7hLsxb7u8uiSXPKwJ7jps7dw79MXevAjeWLwv6Ig8gEAiu/VfmzyCNJq7Co5gvGYNirxGR3q8RxjAvKnWXDwIPoo8+wGrvOhzf7ww9S87Xi4YPEwourvLze88EqavvJ0Z7LwLMki8zBzIu0fYlzwBK8W7GRfSPH0OHbyqx8g7fDL0vPFwsDjrs2S8gtqoPPswV72aLgC8takuO5Xv0jzWxoI7ZU/Yu3e+w7wNlXw8WeSwPMPmw7wRZm08IuVyO1LQmDwYuks8fWylu9W9jzyOTVk8nXS7PONV7rwjxa47W4hIuxy64DwjTFY8LTihvD/ypjsmF5M8/cggvGEHgzzoXPA8mJ+MvInaAz3oLDg8nLC6PNSBcTvQOiY8ejUZOdpIezuMXxK7X88LPc99sbsXrrS7qGVrPPwGujzKshW7+LNjvGgXsDw5fLq7x+MFvWuI8Lygpx48OausO5+x3jzNuV28ZxRYPBeWFTzNPP47VivMPHNkBTxIFTo7qqgWO0APCT3NCfO8DOgGPEnEi7w3Ht+7faQHvO1YtzzmaWo8aYa0u8JCsrsYRGw8yw4XvLSQw7t6QpO8aEyGOw/y+TqR54o6XXHpO/6QX7uhYxk8zXNkvDySGLtSnbC8/j8pvPV5kDy/H448rIApPLnqvTvoWKs8PvnvuiK8MryZ0re7km6zvKYwLz3A1Ui9mwxqPBwQmzpUiRO9sCRuvF3muzwIONI7n+edvChauLyhiyU8+XTAOiw1wLz1j9I8GNYKPSGRuTteeS68qoeKPAJ6CryPnG08uu+iu2bwu7yui4e8s+aSPMwUnbyUiMS8daHvvMFxD72gtgA8bcJSOw== + index: 0 + object: embedding + model: qwen3-embedding:4b + object: list + usage: + prompt_tokens: 4 + total_tokens: 4 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_s3_monitor.py b/tests/test_s3_monitor.py index b125c592..36aad9a4 100644 --- a/tests/test_s3_monitor.py +++ b/tests/test_s3_monitor.py @@ -317,6 +317,37 @@ async def test_s3_watcher_invalid_uri_rejected(): ) +@pytest.mark.asyncio +async def test_s3_watcher_upsert_failure_does_not_abort_sweep(s3_listing): + """A failing upsert doesn't propagate; the refresh keeps processing siblings.""" + set_batches, _ = s3_listing + set_batches([[_meta("incoming/bad.txt", "abc"), _meta("incoming/good.txt", "def")]]) + + from haiku.rag.monitor import S3Watcher + + rag = AsyncMock(spec=HaikuRAG) + rag.list_documents.return_value = [] + + good_doc = Document( + id="good-id", content="...", uri="s3://my-bucket/incoming/good.txt" + ) + + async def maybe_fail(uri, **_): + if uri.endswith("bad.txt"): + raise RuntimeError("boom") + return good_doc + + rag.create_document_from_source.side_effect = maybe_fail + + watcher = S3Watcher(client=rag, entry=_entry(), supported_extensions=[".txt"]) + + # The failing upsert must not propagate out of refresh(). + await watcher.refresh() + + # Both objects were attempted — the first failure didn't abort the sibling. + assert rag.create_document_from_source.await_count == 2 + + @pytest.mark.asyncio async def test_serve_starts_one_s3_task_per_entry(monkeypatch, s3_listing): """`serve` wires one S3Watcher task per MonitorConfig.s3 entry.""" diff --git a/tests/test_s3_source.py b/tests/test_s3_source.py index 41688dd2..1b157566 100644 --- a/tests/test_s3_source.py +++ b/tests/test_s3_source.py @@ -169,3 +169,37 @@ async def test_create_document_from_s3_rejects_invalid_uri( async with HaikuRAG(temp_db_path, create=True) as client: with pytest.raises(ValueError, match="Invalid S3 URI"): await client.create_document_from_source("s3://only-bucket-no-key") + + +@pytest.mark.asyncio +async def test_create_document_from_s3_rejects_unsupported_extension( + fake_obstore_io, temp_db_path +): + head_async, _ = fake_obstore_io + head_async.return_value = _meta('"abc"') + + async with HaikuRAG(temp_db_path, create=True) as client: + with pytest.raises(ValueError, match="Unsupported content type"): + await client.create_document_from_source("s3://my-bucket/file.unsupported") + + +@pytest.mark.asyncio +@pytest.mark.vcr() +async def test_create_document_from_s3_uri_override(fake_obstore_io, temp_db_path): + """`uri=` kwarg overrides the s3:// URL as the stored document identifier.""" + head_async, get_async = fake_obstore_io + head_async.return_value = _meta('"abc"') + get_async.return_value = _get_result(b"override target content") + + async with HaikuRAG(temp_db_path, create=True) as client: + doc = await client.create_document_from_source( + "s3://my-bucket/file.txt", uri="arxiv:2401.00001" + ) + + assert doc.uri == "arxiv:2401.00001" + # The override is the canonical identifier — lookup by it must hit the doc. + looked_up = await client.get_document_by_uri("arxiv:2401.00001") + assert looked_up is not None + assert looked_up.id == doc.id + # The s3:// URL is NOT a stored identifier. + assert await client.get_document_by_uri("s3://my-bucket/file.txt") is None