From d009da06d6053030cc92d4b274341584010d4c16 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 29 Apr 2026 13:03:14 +0300 Subject: [PATCH] support s3:// document sources --- haiku_rag_slim/haiku/rag/client/__init__.py | 10 +- haiku_rag_slim/haiku/rag/client/documents.py | 144 +++++++++- haiku_rag_slim/haiku/rag/config/__init__.py | 2 + haiku_rag_slim/haiku/rag/config/models.py | 10 + haiku_rag_slim/haiku/rag/s3.py | 42 +++ haiku_rag_slim/pyproject.toml | 2 + pyproject.toml | 9 + tests/test_s3_source.py | 261 +++++++++++++++++++ uv.lock | 82 +++++- 9 files changed, 549 insertions(+), 13 deletions(-) create mode 100644 haiku_rag_slim/haiku/rag/s3.py create mode 100644 tests/test_s3_source.py diff --git a/haiku_rag_slim/haiku/rag/client/__init__.py b/haiku_rag_slim/haiku/rag/client/__init__.py index cb95771e..1413757f 100644 --- a/haiku_rag_slim/haiku/rag/client/__init__.py +++ b/haiku_rag_slim/haiku/rag/client/__init__.py @@ -210,10 +210,18 @@ class HaikuRAG: title: str | None = None, metadata: dict | None = None, uri: str | None = None, + storage_options: dict[str, str] | None = None, ) -> Document | list[Document]: from haiku.rag.client.documents import create_document_from_source - return await create_document_from_source(self, source, title, metadata, uri=uri) + return await create_document_from_source( + self, + source, + title, + metadata, + uri=uri, + storage_options=storage_options, + ) async def update_document( self, diff --git a/haiku_rag_slim/haiku/rag/client/documents.py b/haiku_rag_slim/haiku/rag/client/documents.py index e104cce3..a0b0003a 100644 --- a/haiku_rag_slim/haiku/rag/client/documents.py +++ b/haiku_rag_slim/haiku/rag/client/documents.py @@ -197,6 +197,7 @@ async def create_document_from_source( title: str | None = None, metadata: dict | None = None, uri: str | None = None, + storage_options: dict[str, str] | None = None, ) -> Document | list[Document]: """Create or update document(s) from a file path, directory, or URL. @@ -221,6 +222,15 @@ async def create_document_from_source( return await _create_or_update_document_from_url( client, source_str, title=title, metadata=metadata, uri=uri ) + elif parsed_url.scheme == "s3": + return await _create_or_update_document_from_s3( + client, + source_str, + storage_options=storage_options, + title=title, + metadata=metadata, + uri=uri, + ) elif parsed_url.scheme == "file": source_path = Path(parsed_url.path) else: @@ -441,6 +451,138 @@ async def _create_or_update_document_from_url( ) +async def _create_or_update_document_from_s3( + client: "HaikuRAG", + url: str, + *, + storage_options: dict[str, str] | None = None, + title: str | None = None, + metadata: dict | None = None, + uri: str | None = None, +) -> Document: + """Create or update a document from an s3:// URL. + + Two-stage change detection: + - HeadObject ETag matches stored metadata["etag"] → skip GetObject and re-chunk. + - ETag differs but content MD5 matches → refresh etag only, no re-chunk. + - Otherwise download, convert, chunk, embed. + + ``uri`` overrides the s3:// URL as the stored document identifier. + """ + from haiku.rag.client.processing import get_extension_from_content_type_or_url + from haiku.rag.embeddings import embed_chunks + from haiku.rag.s3 import make_s3_session + + metadata = metadata or {} + stored_uri = uri if uri is not None else url + + parsed = urlparse(url) + bucket = parsed.netloc + key = parsed.path.lstrip("/") + if not bucket or not key: + raise ValueError(f"Invalid S3 URI: {url}") + + converter = get_converter(client._config) + supported_extensions = converter.supported_extensions + + session, client_kwargs = make_s3_session(storage_options) + + async with session.client("s3", **client_kwargs) as s3: + head = await s3.head_object(Bucket=bucket, Key=key) + etag = head["ETag"].strip('"') + content_type = (head.get("ContentType") or "").lower() + if not content_type: + content_type = "application/octet-stream" + + existing_doc = await client.get_document_by_uri(stored_uri) + if existing_doc and existing_doc.metadata.get("etag") == etag: + updated = False + if title is not None and title != existing_doc.title: + existing_doc.title = title + updated = True + + merged_metadata = {**(existing_doc.metadata or {}), **metadata} + if merged_metadata != existing_doc.metadata: + existing_doc.metadata = merged_metadata + updated = True + + if updated: + return await client.document_repository.update(existing_doc) + return existing_doc + + file_extension = get_extension_from_content_type_or_url(url, content_type) + if file_extension not in supported_extensions: + raise ValueError( + f"Unsupported content type/extension: {content_type}/{file_extension}" + ) + + get_resp = await s3.get_object(Bucket=bucket, Key=key) + body = await get_resp["Body"].read() + + md5_hash = hashlib.md5(body, usedforsecurity=False).hexdigest() + + with tempfile.NamedTemporaryFile( + mode="wb", suffix=file_extension, delete=False + ) as temp_file: + temp_file.write(body) + temp_file.flush() + temp_path = Path(temp_file.name) + + metadata.update({"contentType": content_type, "md5": md5_hash, "etag": etag}) + + if existing_doc and existing_doc.metadata.get("md5") == md5_hash: + temp_path.unlink(missing_ok=True) + merged_metadata = {**(existing_doc.metadata or {}), **metadata} + updated = False + if merged_metadata != existing_doc.metadata: + existing_doc.metadata = merged_metadata + updated = True + if title is not None and title != existing_doc.title: + existing_doc.title = title + updated = True + if updated: + return await client.document_repository.update(existing_doc) + return existing_doc + + try: + docling_document = await client.convert(temp_path) + chunks = await client.chunk(docling_document) + embedded_chunks = await embed_chunks(chunks, client._config) + finally: + temp_path.unlink(missing_ok=True) + + stored_content = docling_document.export_to_markdown() + + if existing_doc: + existing_doc.content = stored_content + existing_doc.metadata = metadata + existing_doc.set_docling(docling_document) + if title is not None: + existing_doc.title = title + elif existing_doc.title is None: + existing_doc.title = await resolve_title( + client._config, docling_document, stored_content + ) + return await _update_document_with_chunks( + client, existing_doc, embedded_chunks, docling_document + ) + else: + if title is None: + title = await resolve_title( + client._config, docling_document, stored_content + ) + document = Document( + content=stored_content, + uri=stored_uri, + title=title, + metadata=metadata, + ) + document.set_docling(docling_document) + return await _store_document_with_chunks( + client, document, embedded_chunks, docling_document + ) + + async def update_document( client: "HaikuRAG", document_id: str, @@ -525,7 +667,7 @@ def check_source_accessible(uri: str) -> bool: try: if parsed_url.scheme == "file": return Path(parsed_url.path).exists() - elif parsed_url.scheme in ("http", "https"): + elif parsed_url.scheme in ("http", "https", "s3"): return True return False except Exception: diff --git a/haiku_rag_slim/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py index 75e637bc..e5567237 100644 --- a/haiku_rag_slim/haiku/rag/config/__init__.py +++ b/haiku_rag_slim/haiku/rag/config/__init__.py @@ -18,6 +18,7 @@ from haiku.rag.config.models import ( QAConfig, RerankingConfig, ResearchConfig, + S3MonitorEntry, StorageConfig, ) @@ -37,6 +38,7 @@ __all__ = [ "QAConfig", "RerankingConfig", "ResearchConfig", + "S3MonitorEntry", "StorageConfig", "find_config_file", "generate_default_config", diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index 29d36407..a46cedc3 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -51,11 +51,21 @@ class StorageConfig(BaseModel): vacuum_retention_seconds: int = 86400 +class S3MonitorEntry(BaseModel): + uri: str + storage_options: dict[str, str] = Field(default_factory=dict) + poll_interval: int = 300 + ignore_patterns: list[str] = [] + include_patterns: list[str] = [] + delete_orphans: bool = False + + class MonitorConfig(BaseModel): directories: list[Path] = [] ignore_patterns: list[str] = [] include_patterns: list[str] = [] delete_orphans: bool = False + s3: list[S3MonitorEntry] = [] class LanceDBConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/s3.py b/haiku_rag_slim/haiku/rag/s3.py new file mode 100644 index 00000000..9c9d489f --- /dev/null +++ b/haiku_rag_slim/haiku/rag/s3.py @@ -0,0 +1,42 @@ +from typing import Any + + +def make_s3_session( + storage_options: dict[str, str] | None, +) -> tuple[Any, dict[str, Any]]: + """Build an aioboto3 Session and S3 client kwargs from LanceDB-style options. + + Accepts the same dict shape as `LanceDBConfig.storage_options` so users can + copy-paste their LanceDB credentials. Recognized keys: aws_access_key_id, + aws_secret_access_key, aws_session_token, region (or region_name), endpoint + (or endpoint_url), allow_http. Empty/missing keys fall back to the boto3 + default credential chain (environment variables, IAM role, AWS profile). + """ + try: + import aioboto3 # type: ignore[import-not-found] # ty: ignore[unresolved-import] + except ImportError as e: + raise ImportError( + "aioboto3 is required for s3:// sources. " + "Install with: pip install haiku.rag-slim[s3]" + ) from e + + storage_options = storage_options or {} + session_kwargs: dict[str, Any] = {} + client_kwargs: dict[str, Any] = {} + + for key in ("aws_access_key_id", "aws_secret_access_key", "aws_session_token"): + if key in storage_options: + session_kwargs[key] = storage_options[key] + + region = storage_options.get("region") or storage_options.get("region_name") + if region: + session_kwargs["region_name"] = region + + endpoint = storage_options.get("endpoint") or storage_options.get("endpoint_url") + if endpoint: + client_kwargs["endpoint_url"] = endpoint + + if str(storage_options.get("allow_http", "")).lower() == "true": + client_kwargs["use_ssl"] = False + + return aioboto3.Session(**session_kwargs), client_kwargs diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 6705793e..a8d3789d 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -43,6 +43,8 @@ dependencies = [ [project.optional-dependencies] # Document processing docling = ["docling>=2.91.0", "opencv-python-headless>=4.13.0.92"] +# S3 / object-storage monitoring +s3 = ["aioboto3>=13"] # Embedding providers voyageai = ["pydantic-ai-slim[voyageai]"] # Rerankers diff --git a/pyproject.toml b/pyproject.toml index fe94c4c4..2539c311 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ haiku-rag = "haiku.rag.cli:cli" [project.optional-dependencies] tui = ["textual>=1.0.0"] +s3 = ["haiku.rag-slim[s3]==0.45.0"] [build-system] requires = ["hatchling"] @@ -57,6 +58,14 @@ exclude = [ [tool.hatch.build.targets.wheel] only-include = ["/README.md"] +[tool.uv] +conflicts = [ + [ + { package = "haiku.rag-slim", extra = "s3" }, + { package = "pydantic-ai-slim", extra = "bedrock" }, + ], +] + [tool.uv.workspace] members = ["haiku_rag_slim", "evaluations"] diff --git a/tests/test_s3_source.py b/tests/test_s3_source.py new file mode 100644 index 00000000..c5f0ba7b --- /dev/null +++ b/tests/test_s3_source.py @@ -0,0 +1,261 @@ +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from haiku.rag.client import HaikuRAG + + +@pytest.fixture +def fake_aioboto3(monkeypatch): + """Install a fake aioboto3 module in sys.modules. + + Returns the module itself; callers configure `Session` to control behavior. + """ + fake = MagicMock() + monkeypatch.setitem(sys.modules, "aioboto3", fake) + return fake + + +@pytest.fixture +def fake_s3_client(fake_aioboto3): + """Configure fake_aioboto3.Session to return a controllable S3 client. + + Returns a MagicMock representing the S3 client (head_object, get_object). + Tests override its return values per-case. + """ + s3_client = MagicMock() + s3_client.head_object = AsyncMock() + s3_client.get_object = AsyncMock() + + client_ctx = AsyncMock() + client_ctx.__aenter__.return_value = s3_client + client_ctx.__aexit__.return_value = None + + session = MagicMock() + session.client.return_value = client_ctx + + fake_aioboto3.Session.return_value = session + return s3_client + + +def _streaming_body(data: bytes) -> AsyncMock: + body = AsyncMock() + body.read.return_value = data + return body + + +def test_make_s3_session_translates_lancedb_keys(fake_aioboto3): + from haiku.rag.s3 import make_s3_session + + storage_options = { + "endpoint": "http://seaweed:8333", + "region": "us-east-1", + "aws_access_key_id": "AKIA", + "aws_secret_access_key": "secret", + "allow_http": "true", + } + + session, client_kwargs = make_s3_session(storage_options) + + fake_aioboto3.Session.assert_called_once_with( + aws_access_key_id="AKIA", + aws_secret_access_key="secret", + region_name="us-east-1", + ) + assert client_kwargs == { + "endpoint_url": "http://seaweed:8333", + "use_ssl": False, + } + assert session is fake_aioboto3.Session.return_value + + +def test_make_s3_session_accepts_native_aliases(fake_aioboto3): + from haiku.rag.s3 import make_s3_session + + _, client_kwargs = make_s3_session( + {"region_name": "eu-west-1", "endpoint_url": "https://s3"} + ) + + fake_aioboto3.Session.assert_called_once_with(region_name="eu-west-1") + assert client_kwargs == {"endpoint_url": "https://s3"} + + +def test_make_s3_session_no_options_uses_default_chain(fake_aioboto3): + from haiku.rag.s3 import make_s3_session + + session, client_kwargs = make_s3_session(None) + + fake_aioboto3.Session.assert_called_once_with() + assert client_kwargs == {} + + +def test_make_s3_session_allow_http_only_when_truthy(fake_aioboto3): + from haiku.rag.s3 import make_s3_session + + _, ck = make_s3_session({"allow_http": "false"}) + assert "use_ssl" not in ck + + _, ck = make_s3_session({"allow_http": "True"}) + assert ck["use_ssl"] is False + + +def test_make_s3_session_missing_aioboto3_raises_actionable_error(monkeypatch): + monkeypatch.setitem(sys.modules, "aioboto3", None) + + from haiku.rag.s3 import make_s3_session + + with pytest.raises(ImportError, match=r"haiku\.rag-slim\[s3\]"): + make_s3_session({}) + + +@pytest.mark.asyncio +async def test_create_document_from_s3_new(fake_s3_client, temp_db_path): + text = b"S3 hosted content" + fake_s3_client.head_object.return_value = { + "ETag": '"abc123"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + + async with HaikuRAG(temp_db_path, create=True) as client: + doc = await client.create_document_from_source("s3://my-bucket/folder/file.txt") + + assert doc.uri == "s3://my-bucket/folder/file.txt" + assert doc.metadata["etag"] == "abc123" # quotes stripped + assert doc.metadata["md5"] # real content MD5 + assert doc.metadata["md5"] != "abc123" + fake_s3_client.head_object.assert_awaited_once() + fake_s3_client.get_object.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_create_document_from_s3_skips_when_etag_unchanged( + fake_s3_client, temp_db_path +): + text = b"S3 hosted content" + fake_s3_client.head_object.return_value = { + "ETag": '"abc123"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + + async with HaikuRAG(temp_db_path, create=True) as client: + first = await client.create_document_from_source("s3://my-bucket/file.txt") + + # Second call with the same ETag must not GetObject. + fake_s3_client.get_object.reset_mock() + # Re-arm body so a stray call would still produce something readable. + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + second = await client.create_document_from_source("s3://my-bucket/file.txt") + + assert second.id == first.id + assert fake_s3_client.head_object.await_count == 2 + fake_s3_client.get_object.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk( + fake_s3_client, temp_db_path +): + """Multipart re-upload of same content: etag changes, MD5 doesn't. + + Expected: GetObject runs to verify, but no re-chunk; only metadata.etag updates. + """ + text = b"S3 hosted content" + fake_s3_client.head_object.return_value = { + "ETag": '"abc123"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + + async with HaikuRAG(temp_db_path, create=True) as client: + first = await client.create_document_from_source("s3://my-bucket/file.txt") + original_md5 = first.metadata["md5"] + original_updated_at = first.updated_at + + # Same bytes, different ETag (simulating a multipart re-upload). + fake_s3_client.head_object.return_value = { + "ETag": '"def456-2"', # multipart-style ETag + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)} + + second = await client.create_document_from_source("s3://my-bucket/file.txt") + + assert second.id == first.id + assert second.metadata["md5"] == original_md5 # MD5 unchanged + assert second.metadata["etag"] == "def456-2" # ETag refreshed + assert second.updated_at >= original_updated_at + # GetObject ran once (initial create) plus once more for the etag-changed compare. + assert fake_s3_client.get_object.await_count == 2 + + +@pytest.mark.asyncio +async def test_create_document_from_s3_etag_changed_md5_changed_rechunks( + fake_s3_client, temp_db_path +): + fake_s3_client.head_object.return_value = { + "ETag": '"abc123"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(b"original text")} + + async with HaikuRAG(temp_db_path, create=True) as client: + first = await client.create_document_from_source("s3://my-bucket/file.txt") + + fake_s3_client.head_object.return_value = { + "ETag": '"new999"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = { + "Body": _streaming_body(b"different text now") + } + + second = await client.create_document_from_source("s3://my-bucket/file.txt") + + assert second.id == first.id + assert second.metadata["md5"] != first.metadata["md5"] + assert second.metadata["etag"] == "new999" + assert "different text now" in second.content + + +@pytest.mark.asyncio +async def test_create_document_from_s3_rejects_invalid_uri( + fake_s3_client, temp_db_path +): + async with HaikuRAG(temp_db_path, create=True) as client: + with pytest.raises(ValueError, match="Invalid S3 URI"): + await client.create_document_from_source("s3://only-bucket-no-key") + + +@pytest.mark.asyncio +async def test_create_document_from_s3_passes_storage_options( + fake_aioboto3, fake_s3_client, temp_db_path +): + fake_s3_client.head_object.return_value = { + "ETag": '"abc"', + "ContentType": "text/plain", + } + fake_s3_client.get_object.return_value = {"Body": _streaming_body(b"hello")} + + async with HaikuRAG(temp_db_path, create=True) as client: + await client.create_document_from_source( + "s3://bucket/key.txt", + storage_options={ + "endpoint": "http://seaweed:8333", + "region": "us-east-1", + "allow_http": "true", + "aws_access_key_id": "AKIA", + "aws_secret_access_key": "secret", + }, + ) + + fake_aioboto3.Session.assert_called_with( + aws_access_key_id="AKIA", + aws_secret_access_key="secret", + region_name="us-east-1", + ) + fake_aioboto3.Session.return_value.client.assert_called_with( + "s3", endpoint_url="http://seaweed:8333", use_ssl=False + ) diff --git a/uv.lock b/uv.lock index 59c2370a..cf73fe76 100644 --- a/uv.lock +++ b/uv.lock @@ -872,14 +872,14 @@ wheels = [ [[package]] name = "docling" -version = "2.92.0" +version = "2.93.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docling-slim", extra = ["standard"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/87/343d558da23e0626ba027d2e27da25f1c9022964807bfe80778205550a65/docling-2.92.0.tar.gz", hash = "sha256:e8e393ce1a9520c6a61acc67977a6c2d7c2588d98c7446a935b01f9877af9f93", size = 8727, upload-time = "2026-04-29T07:41:25.438Z" } +sdist = { url = "https://files.pythonhosted.org/packages/df/e7/41a7ef58935b914332a7924e0a9d817d50214201aeca7507944404229161/docling-2.93.0.tar.gz", hash = "sha256:c8b4455466d9a6314c27cf84debfdf491532298f65364ec3449acaad517bfea6", size = 8726, upload-time = "2026-05-07T11:55:38.745Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/ad/e3c776dac9639ba7a9fff4c3fe3687258b6eb651d83c4885fad71b3b6aa4/docling-2.92.0-py3-none-any.whl", hash = "sha256:bd85e102e34bb5a90d3be5c2179855fedbdeb9f45318dfba340bf54f1b3268b2", size = 4829, upload-time = "2026-04-29T07:41:23.992Z" }, + { url = "https://files.pythonhosted.org/packages/47/6f/3befa6e4cd42f3a9cf284165d18ca1039ab676ab36da327242aa701e6b63/docling-2.93.0-py3-none-any.whl", hash = "sha256:30a2dc2db733d6a24095e624cc133ce67e9edc46e778f982cf719e137bcee200", size = 4827, upload-time = "2026-05-07T11:55:37.457Z" }, ] [[package]] @@ -969,7 +969,7 @@ wheels = [ [[package]] name = "docling-slim" -version = "2.92.0" +version = "2.93.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -981,9 +981,9 @@ dependencies = [ { name = "requests" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/45/ff4d565ccf69694b2ab3b0fcc7cd403243b8650c735bb1a42a48ba82bcaf/docling_slim-2.92.0.tar.gz", hash = "sha256:f54a2159a46cf00f4738888594c5a81372048d8a0d1a15dd279a7390641a04fa", size = 387431, upload-time = "2026-04-29T07:40:05.913Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/51/2ee874abcd62b990f0a86abec2e3b87b4cc00731b675ed446fefff6199d9/docling_slim-2.93.0.tar.gz", hash = "sha256:2962f4fc5bdf9dd6d67d6f36f09334f7187039985c0fd4b2d4b1d375e4799157", size = 390036, upload-time = "2026-05-07T11:54:14.562Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/91/12/bdf2579d85e43e5944a146c1c4eab708103784a043fb12d793b524c1cd02/docling_slim-2.92.0-py3-none-any.whl", hash = "sha256:e6b7ea5b955c5c47e9bc36f828268b2b78c32f544842036d43d529746783e380", size = 503467, upload-time = "2026-04-29T07:40:03.714Z" }, + { url = "https://files.pythonhosted.org/packages/f3/3f/f2195f79a62fd6cd10c768c04c758c9d6bd0c804a46175d0f3f10a784fca/docling_slim-2.93.0-py3-none-any.whl", hash = "sha256:98e3db67f7976f051f132e6a6e04f73e7fcd4017877eddf9e849e0969c39fbe7", size = 506046, upload-time = "2026-05-07T11:54:10.884Z" }, ] [package.optional-dependencies] @@ -1445,6 +1445,9 @@ dependencies = [ ] [package.optional-dependencies] +s3 = [ + { name = "haiku-rag-slim", extra = ["s3"] }, +] tui = [ { name = "textual" }, ] @@ -1455,6 +1458,7 @@ dev = [ { name = "haiku-rag-evals" }, { name = "mkdocs" }, { name = "mkdocs-material" }, + { name = "obstore" }, { name = "pre-commit" }, { name = "pydantic-ai-slim", extra = ["anthropic", "bedrock", "google", "groq"] }, { name = "pytest" }, @@ -1469,9 +1473,10 @@ dev = [ [package.metadata] requires-dist = [ { name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "tui"], editable = "haiku_rag_slim" }, + { name = "haiku-rag-slim", extras = ["s3"], marker = "extra == 's3'", editable = "haiku_rag_slim" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=1.0.0" }, ] -provides-extras = ["tui"] +provides-extras = ["tui", "s3"] [package.metadata.requires-dev] dev = [ @@ -1479,6 +1484,7 @@ dev = [ { name = "haiku-rag-evals", editable = "evaluations" }, { name = "mkdocs", specifier = ">=1.6.1" }, { name = "mkdocs-material", specifier = ">=9.7.6" }, + { name = "obstore", specifier = ">=0.9,<0.10" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pydantic-ai-slim", extras = ["anthropic"] }, { name = "pydantic-ai-slim", extras = ["bedrock"] }, @@ -1571,6 +1577,9 @@ mistral = [ mxbai = [ { name = "mxbai-rerank" }, ] +s3 = [ + { name = "obstore" }, +] tui = [ { name = "textual" }, { name = "textual-image" }, @@ -1596,6 +1605,7 @@ requires-dist = [ { name = "jsonpatch", specifier = ">=1.33" }, { name = "lancedb", specifier = "==0.30.2" }, { name = "mxbai-rerank", marker = "extra == 'mxbai'", specifier = ">=0.1.6" }, + { name = "obstore", marker = "extra == 's3'", specifier = ">=0.9,<0.10" }, { name = "opencv-python-headless", marker = "extra == 'docling'", specifier = ">=4.13.0.92" }, { name = "pathspec", specifier = ">=1.0.4" }, { name = "pydantic", specifier = ">=2.12.5" }, @@ -1620,7 +1630,7 @@ requires-dist = [ { name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a11" }, { name = "zstandard", marker = "python_full_version < '3.14'", specifier = ">=0.23.0" }, ] -provides-extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] +provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] [[package]] name = "haiku-skills" @@ -2878,6 +2888,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, ] +[[package]] +name = "obstore" +version = "0.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/7a/3a37b0bf0da898478029fcc511a0d2a7252689b1f29e46db7ae74a219c74/obstore-0.9.4.tar.gz", hash = "sha256:e2b93f1372c59da2c7e74122fc6dc4b713d84fd4528b5b500ef7f548425496b5", size = 124167, upload-time = "2026-04-22T19:51:05.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/25/4449a0066796b91e282d7604a66387bba399b14752598c748ea9557c4c32/obstore-0.9.4-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0d17cd04e7f22960050a85f8daa6e274d693e8fb3b97b81eeaa293c6f9e62eb4", size = 4090743, upload-time = "2026-04-22T19:49:26.461Z" }, + { url = "https://files.pythonhosted.org/packages/93/91/639fe5f5644593b9f4bea66f8f29c7bfd4de3b3381fb74b4f7df678f505f/obstore-0.9.4-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4beec92710fb8826fb357baf28fb79a91ee07dcdfe73777207aa762164aaa35", size = 3876313, upload-time = "2026-04-22T19:49:28.107Z" }, + { url = "https://files.pythonhosted.org/packages/ce/71/d6675f845ebe1e3927f2dce6a2a4d5a393359274762ee00c5e6855d5f468/obstore-0.9.4-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d523c8c365ab60afb8d232614a00a92bea439a9f5c55b92486c23a47af038a1e", size = 4029950, upload-time = "2026-04-22T19:49:30.279Z" }, + { url = "https://files.pythonhosted.org/packages/0e/3a/5915a173f5c6a95f9ec186a7e29b0ce6a23bd9b04c2b0b29a351dbe2baf6/obstore-0.9.4-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee0483619088337ee365cb344fceee337e2670ec4de2a1da92ac7f6b2220f18e", size = 4129455, upload-time = "2026-04-22T19:49:31.934Z" }, + { url = "https://files.pythonhosted.org/packages/b5/a9/63c31d2d436c06c4d39ed5cb154fe54202b303854532ec09537c4ce0755b/obstore-0.9.4-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:83da348bf0a7dd84e5839c0cd54d79dcd08e0729c394e566f73a605b93b9e998", size = 4416727, upload-time = "2026-04-22T19:49:34.016Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fa/23c5c6db02be0e13abcbe01c1ca94c5f7876e8c58e74cb9ac2b57b068866/obstore-0.9.4-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f282a17200bcc37b8d7a1d02a146ed41812eb6e76fd0a4c9a154f02da1b8031f", size = 4311520, upload-time = "2026-04-22T19:49:35.905Z" }, + { url = "https://files.pythonhosted.org/packages/86/f0/49f6b02dab9c05e3fd79d6129e4d9e7e9874d6e5e05369ca3b3b80a48aaa/obstore-0.9.4-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d29dcfceaa0a205ded2263d29a2a3aa206819d549e0325c1f2106f79e2658584", size = 4220536, upload-time = "2026-04-22T19:49:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/50/ab/d0bfd6d68422e7d8f2204d91736c7e62767e0576ad749da442a71e7773b2/obstore-0.9.4-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:caecb912723ab8e9da8da26def249d66da4318959df2bafc0a55af64f3255902", size = 4105099, upload-time = "2026-04-22T19:49:40.384Z" }, + { url = "https://files.pythonhosted.org/packages/66/3b/f595d0ee354f9daa69438991f8818602f34bc59498c8468456a02d45fb27/obstore-0.9.4-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c1c06fec8837595a2829b5f7536d0d01e940ce10b07ad2a8594fec1cfd0b7d5", size = 4294206, upload-time = "2026-04-22T19:49:42.016Z" }, + { url = "https://files.pythonhosted.org/packages/60/54/3c5af2d59258aaa9e5bef05320658ea6e9b1f3897a3a977bf7f54a0b6ec1/obstore-0.9.4-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c132795a789ec5ade31bf4d5b55ed321fb41d9749e9145520bf19063e1da5f7b", size = 4265047, upload-time = "2026-04-22T19:49:43.983Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/a8ba1feb81b9833b253147839da40405ec6bfa51feb3abfe909c800208a5/obstore-0.9.4-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:c6e342360a5d0ae71486bc5f8311778aa144ec1a905c23593f8ef57b5bceae24", size = 4255361, upload-time = "2026-04-22T19:49:45.864Z" }, + { url = "https://files.pythonhosted.org/packages/15/f7/3ccc0288111e057f8ba3d99bee14f95d9e9bb00acaf6e9700e0eb4cd82c3/obstore-0.9.4-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aeb6f7e7e862550f5020a10692ef6f02d5ba4912dba08942eb59bb7d73f93fe0", size = 4439378, upload-time = "2026-04-22T19:49:47.581Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/3ac8b5772743c60064f3c7e02d27f346dbb58feaa99a49ee09798d1cfb00/obstore-0.9.4-cp311-abi3-win_amd64.whl", hash = "sha256:a58ef942292841f99d69ac11d19d05544c835447c8c09dacbfb7409c6374c4a1", size = 4191594, upload-time = "2026-04-22T19:49:49.308Z" }, + { url = "https://files.pythonhosted.org/packages/9d/81/8f6b6509f8df603261cdb5ddb521c49891457775669c6ad857812bf4a7c1/obstore-0.9.4-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:fff17f59390ed307afcd1fb18c56076c1f911dd9f5c2636b7d7133c4d07f8c3f", size = 4071300, upload-time = "2026-04-22T19:49:51.386Z" }, + { url = "https://files.pythonhosted.org/packages/ab/fe/0c74ddf3ab9b24ef356925bfb613bc7846f869220361a784b63f754d8563/obstore-0.9.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4527c4c7889f1bd1f1952017d74774870e14e199d6b50b9e72f291f9498d898c", size = 3870593, upload-time = "2026-04-22T19:49:53.481Z" }, + { url = "https://files.pythonhosted.org/packages/73/fa/260ec94f9a7b4f4c8afbdd016710bed0736615488d3ac0c5620f9179bfcd/obstore-0.9.4-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a57c2016e3e569de35050f95c679ffe61813c4e3cb6d6028c4c3f57231021eb4", size = 4023990, upload-time = "2026-04-22T19:49:55.644Z" }, + { url = "https://files.pythonhosted.org/packages/8d/84/5b8e2b9607fb93c96a39a4cfa6d37bd3049ebf7265d0e9f8afa938bf32fe/obstore-0.9.4-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd5327cee4fb3578b51beb1c92915cc3a05ffe794be40f50bd68d27e97d78c5c", size = 4119971, upload-time = "2026-04-22T19:49:57.745Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2b/e6c093acb7e62009d5b1678d82839903287c29d4a6e1dfbea8fbf41313d5/obstore-0.9.4-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12b1e6105eafe02d8973dbeb2d274eeac2271c67f1126ffa16f18ddea8dd5443", size = 4407147, upload-time = "2026-04-22T19:49:59.928Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3d/5c93a9adee8f045b89d5f21b337f53667499db770bda129f805723ab14e4/obstore-0.9.4-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b0d378248fda4e36652808d73eaaeb7e67154427e6c724248c9b0b9b03e70a6", size = 4312215, upload-time = "2026-04-22T19:50:01.534Z" }, + { url = "https://files.pythonhosted.org/packages/9a/de/507f60b4e6a8c0cad9f93a51a7b28132c9db49e20aadbcd542fa2abc57c4/obstore-0.9.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78fb77c346abd2bcdfa071d7166be2bdc38c28573ae5a230746df6158a5593e", size = 4216936, upload-time = "2026-04-22T19:50:03.244Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ff/612bd5f8258349bfe9e8c349d184b5ea3333038d4cce0d003eefafb2160c/obstore-0.9.4-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:f4e5a6dfe6877fb599868d560d6fcf4d7416cadbdf3bd947254b53830c2f11c0", size = 4105091, upload-time = "2026-04-22T19:50:05.038Z" }, + { url = "https://files.pythonhosted.org/packages/e3/73/b083b99e7bc0b529bee7b4437cafd7cc7d9f59c10995a48b6c26447fdf7f/obstore-0.9.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f8114a2b84268c991232d89b105d9239299b6afb56e4941a61c09f3a89033022", size = 4292570, upload-time = "2026-04-22T19:50:06.823Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cd/3c4555f98db9a49432bc0afa68bfc33dd47bdfa3699c915b4b0e887577e3/obstore-0.9.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:9d7b959f5f74532a142fb449c0bef5814dfe3fa5c43c31ac4284a15221a75aaf", size = 4261946, upload-time = "2026-04-22T19:50:08.789Z" }, + { url = "https://files.pythonhosted.org/packages/96/f8/bdc66df3d0dfdcfb3931a585a7fb3b74336619baf6d3540b1425b424232b/obstore-0.9.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a8e9101fc2659dd938e7ae06512075bc0a8f02ab28d2ee438d6fca8b4f3bdfba", size = 4245595, upload-time = "2026-04-22T19:50:10.765Z" }, + { url = "https://files.pythonhosted.org/packages/d7/22/1aa58ea676293e5b888391c8433ff6ab8f66622aae30427287f9daac6d46/obstore-0.9.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:538384255545b5c575497fcab26389c8f01707402b6ddcdd73b769b66311635d", size = 4436599, upload-time = "2026-04-22T19:50:12.585Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9e/b52f2c97be27952d488cf1980af0c635f9947003e5744e3e1dc6252f0040/obstore-0.9.4-cp313-cp313t-win_amd64.whl", hash = "sha256:eef1c772657bb1293adad0d671ca1ff1e1dcae84ec4dfbf1a34e47c2a1f134ac", size = 4180463, upload-time = "2026-04-22T19:50:14.288Z" }, + { url = "https://files.pythonhosted.org/packages/19/76/c53583f95c6811057abd3116756dca46785318d564a0e99c207cbb2d8938/obstore-0.9.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e009e7437770c85beae4c32cb79f662f0a9922676ef127e943d107a5c082d38d", size = 4071302, upload-time = "2026-04-22T19:50:15.967Z" }, + { url = "https://files.pythonhosted.org/packages/2f/23/ac3b9c05a09b3d5f178ed6f288c5d6913df8f7386059590194e0fee65d15/obstore-0.9.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ac5f3ad314bd4592fe484b79c229518be7bb5f6218bed33c20742026d5caf860", size = 3870813, upload-time = "2026-04-22T19:50:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ec/c3458e0f24d2d1a4f185f541905b07e51c91b3fec589b1600c77d511e585/obstore-0.9.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:db79d5ebc4177360565ffcec4abd49930cf052cdbeb94e3a3ece2e2d08f087d0", size = 4024237, upload-time = "2026-04-22T19:50:19.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/6cf468a200e491fdc6c04075e2fbbac1707bbecd243f0f56ae1e75d052ed/obstore-0.9.4-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05b565d89c3115fb74385852dd628e12f6645a1bba97523dceae016b538a3f33", size = 4119635, upload-time = "2026-04-22T19:50:21.605Z" }, + { url = "https://files.pythonhosted.org/packages/81/fb/b44d002767fa5af95ab4ca8e16c3a9057fc11f13de03f498b99adf0c4e50/obstore-0.9.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dfc4fc98403d8fbb316eb04257c8122b6f1dda37e80869491fdacf60a815e4c", size = 4406906, upload-time = "2026-04-22T19:50:23.654Z" }, + { url = "https://files.pythonhosted.org/packages/4b/18/9a75ad5082cd581c4a55f0e62bedf4b030a8b53824976fc1f030eff225b3/obstore-0.9.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c69af620fd3d06a8cfb62d25faf1adb6ccc97cc572f47ee04dddcde5a5e5444e", size = 4311826, upload-time = "2026-04-22T19:50:25.458Z" }, + { url = "https://files.pythonhosted.org/packages/8d/03/b0f945b31f40364a7ed4dbc5677abc66331fcf478732f4d643e17e56bb13/obstore-0.9.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa78c0230e0b9d49b25ed18980e1751331ddfe05782d6ce97579a9ccda8229ea", size = 4217086, upload-time = "2026-04-22T19:50:27.266Z" }, + { url = "https://files.pythonhosted.org/packages/1a/26/bdd85264c806802086f21d73cc7c95a5baca5feeeac4bce8acb97142163f/obstore-0.9.4-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:c828719f0bb310a9cf0e0f08cb62a0b8cc550138617cb03ac897900aec9d3d47", size = 4105560, upload-time = "2026-04-22T19:50:29.324Z" }, + { url = "https://files.pythonhosted.org/packages/6a/36/4a4a6a398e5f145edd1886388ebe5e6f6bbaf74950a5dea1a6ceae63e6b5/obstore-0.9.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:49a0455519f284b6bc2e0694298114926aff1d1f3d5d344e9163e03b446826cc", size = 4292582, upload-time = "2026-04-22T19:50:31.028Z" }, + { url = "https://files.pythonhosted.org/packages/39/4c/9caa197cd2eba726e9a5285db34027049b9527a23e1a7e08479678ad6a4a/obstore-0.9.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf437309fc0fe852591ae50405300490229f876ea06574651fd753ca3fd23f25", size = 4261613, upload-time = "2026-04-22T19:50:32.868Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/a3fbe6fb3ee1c57fd4943ddbb21848eea3925b77e0789614c857d86b795e/obstore-0.9.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d83dbd20b6a5d42e35794ef64046de39040854829ec4f1eb2f6dfb54df48cc3d", size = 4245638, upload-time = "2026-04-22T19:50:35.009Z" }, + { url = "https://files.pythonhosted.org/packages/56/a7/d18e168f318327d63512dfa7cf3b5e89ed9bfba6d6a8917ad7d4700b8657/obstore-0.9.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5a0c337f37f30a2d66555d69bf3abd840457a279c57ede93bd02e014721ed364", size = 4437226, upload-time = "2026-04-22T19:50:36.635Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ce/66aadd155db1e273c6ec2236c0fb904666d10c2e3b791b40624c272e586c/obstore-0.9.4-cp314-cp314t-win_amd64.whl", hash = "sha256:24e37a1c713c95a964e119f8ef879415a495432162e74e80ed29d645aeeca114", size = 4180746, upload-time = "2026-04-22T19:50:38.396Z" }, +] + [[package]] name = "omegaconf" version = "2.3.0" @@ -3835,16 +3895,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.14.0" +version = "2.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/42/98/c8345dccdc31de4228c039a98f6467a941e39558da41c1744fbe29fa5666/pydantic_settings-2.14.0.tar.gz", hash = "sha256:24285fd4b0e0c06507dd9fdfd331ee23794305352aaec8fc4eb92d4047aeb67d", size = 235709, upload-time = "2026-04-20T13:37:40.293Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/01/dd/bebff3040138f00ae8a102d426b27349b9a49acc310fcae7f92112d867e3/pydantic_settings-2.14.0-py3-none-any.whl", hash = "sha256:fc8d5d692eb7092e43c8647c1c35a3ecd00e040fcf02ed86f4cb5458ca62182e", size = 60940, upload-time = "2026-04-20T13:37:38.586Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, ] [[package]]