danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

support s3:// document sources

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-04-29 13:03:14 +03:00
parent 8b8f44e7b2
commit d009da06d6
No known key found for this signature in database
9 changed files with 549 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
haiku_rag_slim/haiku/rag/client/__init__.py
View file

@ -210,10 +210,18 @@ class HaikuRAG:
title: str | None = None,
metadata: dict | None = None,
uri: str | None = None,
storage_options: dict[str, str] | None = None,
) -> Document | list[Document]:
from haiku.rag.client.documents import create_document_from_source
return await create_document_from_source(self, source, title, metadata, uri=uri)
return await create_document_from_source(
self,
source,
title,
metadata,
uri=uri,
storage_options=storage_options,
)
async def update_document(
self,

144
haiku_rag_slim/haiku/rag/client/documents.py
View file

@ -197,6 +197,7 @@ async def create_document_from_source(
title: str | None = None,
metadata: dict | None = None,
uri: str | None = None,
storage_options: dict[str, str] | None = None,
) -> Document | list[Document]:
"""Create or update document(s) from a file path, directory, or URL.
@ -221,6 +222,15 @@ async def create_document_from_source(
return await _create_or_update_document_from_url(
client, source_str, title=title, metadata=metadata, uri=uri
)
elif parsed_url.scheme == "s3":
return await _create_or_update_document_from_s3(
client,
source_str,
storage_options=storage_options,
title=title,
metadata=metadata,
uri=uri,
)
elif parsed_url.scheme == "file":
source_path = Path(parsed_url.path)
else:
@ -441,6 +451,138 @@ async def _create_or_update_document_from_url(
)
async def _create_or_update_document_from_s3(
client: "HaikuRAG",
url: str,
*,
storage_options: dict[str, str] | None = None,
title: str | None = None,
metadata: dict | None = None,
uri: str | None = None,
) -> Document:
"""Create or update a document from an s3:// URL.
Two-stage change detection:
- HeadObject ETag matches stored metadata["etag"] → skip GetObject and re-chunk.
- ETag differs but content MD5 matches → refresh etag only, no re-chunk.
- Otherwise download, convert, chunk, embed.
``uri`` overrides the s3:// URL as the stored document identifier.
"""
from haiku.rag.client.processing import get_extension_from_content_type_or_url
from haiku.rag.embeddings import embed_chunks
from haiku.rag.s3 import make_s3_session
metadata = metadata or {}
stored_uri = uri if uri is not None else url
parsed = urlparse(url)
bucket = parsed.netloc
key = parsed.path.lstrip("/")
if not bucket or not key:
raise ValueError(f"Invalid S3 URI: {url}")
converter = get_converter(client._config)
supported_extensions = converter.supported_extensions
session, client_kwargs = make_s3_session(storage_options)
async with session.client("s3", **client_kwargs) as s3:
head = await s3.head_object(Bucket=bucket, Key=key)
etag = head["ETag"].strip('"')
content_type = (head.get("ContentType") or "").lower()
if not content_type:
content_type = "application/octet-stream"
existing_doc = await client.get_document_by_uri(stored_uri)
if existing_doc and existing_doc.metadata.get("etag") == etag:
updated = False
if title is not None and title != existing_doc.title:
existing_doc.title = title
updated = True
merged_metadata = {**(existing_doc.metadata or {}), **metadata}
if merged_metadata != existing_doc.metadata:
existing_doc.metadata = merged_metadata
updated = True
if updated:
return await client.document_repository.update(existing_doc)
return existing_doc
file_extension = get_extension_from_content_type_or_url(url, content_type)
if file_extension not in supported_extensions:
raise ValueError(
f"Unsupported content type/extension: {content_type}/{file_extension}"
)
get_resp = await s3.get_object(Bucket=bucket, Key=key)
body = await get_resp["Body"].read()
md5_hash = hashlib.md5(body, usedforsecurity=False).hexdigest()
with tempfile.NamedTemporaryFile(
mode="wb", suffix=file_extension, delete=False
) as temp_file:
temp_file.write(body)
temp_file.flush()
temp_path = Path(temp_file.name)
metadata.update({"contentType": content_type, "md5": md5_hash, "etag": etag})
if existing_doc and existing_doc.metadata.get("md5") == md5_hash:
temp_path.unlink(missing_ok=True)
merged_metadata = {**(existing_doc.metadata or {}), **metadata}
updated = False
if merged_metadata != existing_doc.metadata:
existing_doc.metadata = merged_metadata
updated = True
if title is not None and title != existing_doc.title:
existing_doc.title = title
updated = True
if updated:
return await client.document_repository.update(existing_doc)
return existing_doc
try:
docling_document = await client.convert(temp_path)
chunks = await client.chunk(docling_document)
embedded_chunks = await embed_chunks(chunks, client._config)
finally:
temp_path.unlink(missing_ok=True)
stored_content = docling_document.export_to_markdown()
if existing_doc:
existing_doc.content = stored_content
existing_doc.metadata = metadata
existing_doc.set_docling(docling_document)
if title is not None:
existing_doc.title = title
elif existing_doc.title is None:
existing_doc.title = await resolve_title(
client._config, docling_document, stored_content
)
return await _update_document_with_chunks(
client, existing_doc, embedded_chunks, docling_document
)
else:
if title is None:
title = await resolve_title(
client._config, docling_document, stored_content
)
document = Document(
content=stored_content,
uri=stored_uri,
title=title,
metadata=metadata,
)
document.set_docling(docling_document)
return await _store_document_with_chunks(
client, document, embedded_chunks, docling_document
)
async def update_document(
client: "HaikuRAG",
document_id: str,
@ -525,7 +667,7 @@ def check_source_accessible(uri: str) -> bool:
try:
if parsed_url.scheme == "file":
return Path(parsed_url.path).exists()
elif parsed_url.scheme in ("http", "https"):
elif parsed_url.scheme in ("http", "https", "s3"):
return True
return False
except Exception:

2
haiku_rag_slim/haiku/rag/config/__init__.py
View file

@ -18,6 +18,7 @@ from haiku.rag.config.models import (
QAConfig,
RerankingConfig,
ResearchConfig,
S3MonitorEntry,
StorageConfig,
)
@ -37,6 +38,7 @@ __all__ = [
"QAConfig",
"RerankingConfig",
"ResearchConfig",
"S3MonitorEntry",
"StorageConfig",
"find_config_file",
"generate_default_config",

10
haiku_rag_slim/haiku/rag/config/models.py
View file

@ -51,11 +51,21 @@ class StorageConfig(BaseModel):
vacuum_retention_seconds: int = 86400
class S3MonitorEntry(BaseModel):
uri: str
storage_options: dict[str, str] = Field(default_factory=dict)
poll_interval: int = 300
ignore_patterns: list[str] = []
include_patterns: list[str] = []
delete_orphans: bool = False
class MonitorConfig(BaseModel):
directories: list[Path] = []
ignore_patterns: list[str] = []
include_patterns: list[str] = []
delete_orphans: bool = False
s3: list[S3MonitorEntry] = []
class LanceDBConfig(BaseModel):

42
haiku_rag_slim/haiku/rag/s3.py Normal file
View file

@ -0,0 +1,42 @@
from typing import Any
def make_s3_session(
storage_options: dict[str, str] | None,
) -> tuple[Any, dict[str, Any]]:
"""Build an aioboto3 Session and S3 client kwargs from LanceDB-style options.
Accepts the same dict shape as `LanceDBConfig.storage_options` so users can
copy-paste their LanceDB credentials. Recognized keys: aws_access_key_id,
aws_secret_access_key, aws_session_token, region (or region_name), endpoint
(or endpoint_url), allow_http. Empty/missing keys fall back to the boto3
default credential chain (environment variables, IAM role, AWS profile).
"""
try:
import aioboto3 # type: ignore[import-not-found] # ty: ignore[unresolved-import]
except ImportError as e:
raise ImportError(
"aioboto3 is required for s3:// sources. "
"Install with: pip install haiku.rag-slim[s3]"
) from e
storage_options = storage_options or {}
session_kwargs: dict[str, Any] = {}
client_kwargs: dict[str, Any] = {}
for key in ("aws_access_key_id", "aws_secret_access_key", "aws_session_token"):
if key in storage_options:
session_kwargs[key] = storage_options[key]
region = storage_options.get("region") or storage_options.get("region_name")
if region:
session_kwargs["region_name"] = region
endpoint = storage_options.get("endpoint") or storage_options.get("endpoint_url")
if endpoint:
client_kwargs["endpoint_url"] = endpoint
if str(storage_options.get("allow_http", "")).lower() == "true":
client_kwargs["use_ssl"] = False
return aioboto3.Session(**session_kwargs), client_kwargs

2
haiku_rag_slim/pyproject.toml
View file

@ -43,6 +43,8 @@ dependencies = [
[project.optional-dependencies]
# Document processing
docling = ["docling>=2.91.0", "opencv-python-headless>=4.13.0.92"]
# S3 / object-storage monitoring
s3 = ["aioboto3>=13"]
# Embedding providers
voyageai = ["pydantic-ai-slim[voyageai]"]
# Rerankers

9
pyproject.toml
View file

@ -38,6 +38,7 @@ haiku-rag = "haiku.rag.cli:cli"
[project.optional-dependencies]
tui = ["textual>=1.0.0"]
s3 = ["haiku.rag-slim[s3]==0.45.0"]
[build-system]
requires = ["hatchling"]
@ -57,6 +58,14 @@ exclude = [
[tool.hatch.build.targets.wheel]
only-include = ["/README.md"]
[tool.uv]
conflicts = [
[
{ package = "haiku.rag-slim", extra = "s3" },
{ package = "pydantic-ai-slim", extra = "bedrock" },
],
]
[tool.uv.workspace]
members = ["haiku_rag_slim", "evaluations"]

261
tests/test_s3_source.py Normal file
View file

@ -0,0 +1,261 @@
import sys
from unittest.mock import AsyncMock, MagicMock
import pytest
from haiku.rag.client import HaikuRAG
@pytest.fixture
def fake_aioboto3(monkeypatch):
"""Install a fake aioboto3 module in sys.modules.
Returns the module itself; callers configure `Session` to control behavior.
"""
fake = MagicMock()
monkeypatch.setitem(sys.modules, "aioboto3", fake)
return fake
@pytest.fixture
def fake_s3_client(fake_aioboto3):
"""Configure fake_aioboto3.Session to return a controllable S3 client.
Returns a MagicMock representing the S3 client (head_object, get_object).
Tests override its return values per-case.
"""
s3_client = MagicMock()
s3_client.head_object = AsyncMock()
s3_client.get_object = AsyncMock()
client_ctx = AsyncMock()
client_ctx.__aenter__.return_value = s3_client
client_ctx.__aexit__.return_value = None
session = MagicMock()
session.client.return_value = client_ctx
fake_aioboto3.Session.return_value = session
return s3_client
def _streaming_body(data: bytes) -> AsyncMock:
body = AsyncMock()
body.read.return_value = data
return body
def test_make_s3_session_translates_lancedb_keys(fake_aioboto3):
from haiku.rag.s3 import make_s3_session
storage_options = {
"endpoint": "http://seaweed:8333",
"region": "us-east-1",
"aws_access_key_id": "AKIA",
"aws_secret_access_key": "secret",
"allow_http": "true",
}
session, client_kwargs = make_s3_session(storage_options)
fake_aioboto3.Session.assert_called_once_with(
aws_access_key_id="AKIA",
aws_secret_access_key="secret",
region_name="us-east-1",
)
assert client_kwargs == {
"endpoint_url": "http://seaweed:8333",
"use_ssl": False,
}
assert session is fake_aioboto3.Session.return_value
def test_make_s3_session_accepts_native_aliases(fake_aioboto3):
from haiku.rag.s3 import make_s3_session
_, client_kwargs = make_s3_session(
{"region_name": "eu-west-1", "endpoint_url": "https://s3"}
)
fake_aioboto3.Session.assert_called_once_with(region_name="eu-west-1")
assert client_kwargs == {"endpoint_url": "https://s3"}
def test_make_s3_session_no_options_uses_default_chain(fake_aioboto3):
from haiku.rag.s3 import make_s3_session
session, client_kwargs = make_s3_session(None)
fake_aioboto3.Session.assert_called_once_with()
assert client_kwargs == {}
def test_make_s3_session_allow_http_only_when_truthy(fake_aioboto3):
from haiku.rag.s3 import make_s3_session
_, ck = make_s3_session({"allow_http": "false"})
assert "use_ssl" not in ck
_, ck = make_s3_session({"allow_http": "True"})
assert ck["use_ssl"] is False
def test_make_s3_session_missing_aioboto3_raises_actionable_error(monkeypatch):
monkeypatch.setitem(sys.modules, "aioboto3", None)
from haiku.rag.s3 import make_s3_session
with pytest.raises(ImportError, match=r"haiku\.rag-slim\[s3\]"):
make_s3_session({})
@pytest.mark.asyncio
async def test_create_document_from_s3_new(fake_s3_client, temp_db_path):
text = b"S3 hosted content"
fake_s3_client.head_object.return_value = {
"ETag": '"abc123"',
"ContentType": "text/plain",
}
fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)}
async with HaikuRAG(temp_db_path, create=True) as client:
doc = await client.create_document_from_source("s3://my-bucket/folder/file.txt")
assert doc.uri == "s3://my-bucket/folder/file.txt"
assert doc.metadata["etag"] == "abc123" # quotes stripped
assert doc.metadata["md5"] # real content MD5
assert doc.metadata["md5"] != "abc123"
fake_s3_client.head_object.assert_awaited_once()
fake_s3_client.get_object.assert_awaited_once()
@pytest.mark.asyncio
async def test_create_document_from_s3_skips_when_etag_unchanged(
fake_s3_client, temp_db_path
):
text = b"S3 hosted content"
fake_s3_client.head_object.return_value = {
"ETag": '"abc123"',
"ContentType": "text/plain",
}
fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)}
async with HaikuRAG(temp_db_path, create=True) as client:
first = await client.create_document_from_source("s3://my-bucket/file.txt")
# Second call with the same ETag must not GetObject.
fake_s3_client.get_object.reset_mock()
# Re-arm body so a stray call would still produce something readable.
fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)}
second = await client.create_document_from_source("s3://my-bucket/file.txt")
assert second.id == first.id
assert fake_s3_client.head_object.await_count == 2
fake_s3_client.get_object.assert_not_awaited()
@pytest.mark.asyncio
async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk(
fake_s3_client, temp_db_path
):
"""Multipart re-upload of same content: etag changes, MD5 doesn't.
Expected: GetObject runs to verify, but no re-chunk; only metadata.etag updates.
"""
text = b"S3 hosted content"
fake_s3_client.head_object.return_value = {
"ETag": '"abc123"',
"ContentType": "text/plain",
}
fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)}
async with HaikuRAG(temp_db_path, create=True) as client:
first = await client.create_document_from_source("s3://my-bucket/file.txt")
original_md5 = first.metadata["md5"]
original_updated_at = first.updated_at
# Same bytes, different ETag (simulating a multipart re-upload).
fake_s3_client.head_object.return_value = {
"ETag": '"def456-2"', # multipart-style ETag
"ContentType": "text/plain",
}
fake_s3_client.get_object.return_value = {"Body": _streaming_body(text)}
second = await client.create_document_from_source("s3://my-bucket/file.txt")
assert second.id == first.id
assert second.metadata["md5"] == original_md5 # MD5 unchanged
assert second.metadata["etag"] == "def456-2" # ETag refreshed
assert second.updated_at >= original_updated_at
# GetObject ran once (initial create) plus once more for the etag-changed compare.
assert fake_s3_client.get_object.await_count == 2
@pytest.mark.asyncio
async def test_create_document_from_s3_etag_changed_md5_changed_rechunks(
fake_s3_client, temp_db_path
):
fake_s3_client.head_object.return_value = {
"ETag": '"abc123"',
"ContentType": "text/plain",
}
fake_s3_client.get_object.return_value = {"Body": _streaming_body(b"original text")}
async with HaikuRAG(temp_db_path, create=True) as client:
first = await client.create_document_from_source("s3://my-bucket/file.txt")
fake_s3_client.head_object.return_value = {
"ETag": '"new999"',
"ContentType": "text/plain",
}
fake_s3_client.get_object.return_value = {
"Body": _streaming_body(b"different text now")
}
second = await client.create_document_from_source("s3://my-bucket/file.txt")
assert second.id == first.id
assert second.metadata["md5"] != first.metadata["md5"]
assert second.metadata["etag"] == "new999"
assert "different text now" in second.content
@pytest.mark.asyncio
async def test_create_document_from_s3_rejects_invalid_uri(
fake_s3_client, temp_db_path
):
async with HaikuRAG(temp_db_path, create=True) as client:
with pytest.raises(ValueError, match="Invalid S3 URI"):
await client.create_document_from_source("s3://only-bucket-no-key")
@pytest.mark.asyncio
async def test_create_document_from_s3_passes_storage_options(
fake_aioboto3, fake_s3_client, temp_db_path
):
fake_s3_client.head_object.return_value = {
"ETag": '"abc"',
"ContentType": "text/plain",
}
fake_s3_client.get_object.return_value = {"Body": _streaming_body(b"hello")}
async with HaikuRAG(temp_db_path, create=True) as client:
await client.create_document_from_source(
"s3://bucket/key.txt",
storage_options={
"endpoint": "http://seaweed:8333",
"region": "us-east-1",
"allow_http": "true",
"aws_access_key_id": "AKIA",
"aws_secret_access_key": "secret",
},
)
fake_aioboto3.Session.assert_called_with(
aws_access_key_id="AKIA",
aws_secret_access_key="secret",
region_name="us-east-1",
)
fake_aioboto3.Session.return_value.client.assert_called_with(
"s3", endpoint_url="http://seaweed:8333", use_ssl=False
)

82
uv.lock
View file

@ -872,14 +872,14 @@ wheels = [
[[package]]
name = "docling"
version = "2.92.0"
version = "2.93.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "docling-slim", extra = ["standard"] },
]
sdist = { url = "https://files.pythonhosted.org/packages/e9/87/343d558da23e0626ba027d2e27da25f1c9022964807bfe80778205550a65/docling-2.92.0.tar.gz", hash = "sha256:e8e393ce1a9520c6a61acc67977a6c2d7c2588d98c7446a935b01f9877af9f93", size = 8727, upload-time = "2026-04-29T07:41:25.438Z" }
sdist = { url = "https://files.pythonhosted.org/packages/df/e7/41a7ef58935b914332a7924e0a9d817d50214201aeca7507944404229161/docling-2.93.0.tar.gz", hash = "sha256:c8b4455466d9a6314c27cf84debfdf491532298f65364ec3449acaad517bfea6", size = 8726, upload-time = "2026-05-07T11:55:38.745Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b1/ad/e3c776dac9639ba7a9fff4c3fe3687258b6eb651d83c4885fad71b3b6aa4/docling-2.92.0-py3-none-any.whl", hash = "sha256:bd85e102e34bb5a90d3be5c2179855fedbdeb9f45318dfba340bf54f1b3268b2", size = 4829, upload-time = "2026-04-29T07:41:23.992Z" },
{ url = "https://files.pythonhosted.org/packages/47/6f/3befa6e4cd42f3a9cf284165d18ca1039ab676ab36da327242aa701e6b63/docling-2.93.0-py3-none-any.whl", hash = "sha256:30a2dc2db733d6a24095e624cc133ce67e9edc46e778f982cf719e137bcee200", size = 4827, upload-time = "2026-05-07T11:55:37.457Z" },
]
[[package]]
@ -969,7 +969,7 @@ wheels = [
[[package]]
name = "docling-slim"
version = "2.92.0"
version = "2.93.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "certifi" },
@ -981,9 +981,9 @@ dependencies = [
{ name = "requests" },
{ name = "tqdm" },
]
sdist = { url = "https://files.pythonhosted.org/packages/3c/45/ff4d565ccf69694b2ab3b0fcc7cd403243b8650c735bb1a42a48ba82bcaf/docling_slim-2.92.0.tar.gz", hash = "sha256:f54a2159a46cf00f4738888594c5a81372048d8a0d1a15dd279a7390641a04fa", size = 387431, upload-time = "2026-04-29T07:40:05.913Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c4/51/2ee874abcd62b990f0a86abec2e3b87b4cc00731b675ed446fefff6199d9/docling_slim-2.93.0.tar.gz", hash = "sha256:2962f4fc5bdf9dd6d67d6f36f09334f7187039985c0fd4b2d4b1d375e4799157", size = 390036, upload-time = "2026-05-07T11:54:14.562Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/91/12/bdf2579d85e43e5944a146c1c4eab708103784a043fb12d793b524c1cd02/docling_slim-2.92.0-py3-none-any.whl", hash = "sha256:e6b7ea5b955c5c47e9bc36f828268b2b78c32f544842036d43d529746783e380", size = 503467, upload-time = "2026-04-29T07:40:03.714Z" },
{ url = "https://files.pythonhosted.org/packages/f3/3f/f2195f79a62fd6cd10c768c04c758c9d6bd0c804a46175d0f3f10a784fca/docling_slim-2.93.0-py3-none-any.whl", hash = "sha256:98e3db67f7976f051f132e6a6e04f73e7fcd4017877eddf9e849e0969c39fbe7", size = 506046, upload-time = "2026-05-07T11:54:10.884Z" },
]
[package.optional-dependencies]
@ -1445,6 +1445,9 @@ dependencies = [
]
[package.optional-dependencies]
s3 = [
{ name = "haiku-rag-slim", extra = ["s3"] },
]
tui = [
{ name = "textual" },
]
@ -1455,6 +1458,7 @@ dev = [
{ name = "haiku-rag-evals" },
{ name = "mkdocs" },
{ name = "mkdocs-material" },
{ name = "obstore" },
{ name = "pre-commit" },
{ name = "pydantic-ai-slim", extra = ["anthropic", "bedrock", "google", "groq"] },
{ name = "pytest" },
@ -1469,9 +1473,10 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "tui"], editable = "haiku_rag_slim" },
{ name = "haiku-rag-slim", extras = ["s3"], marker = "extra == 's3'", editable = "haiku_rag_slim" },
{ name = "textual", marker = "extra == 'tui'", specifier = ">=1.0.0" },
]
provides-extras = ["tui"]
provides-extras = ["tui", "s3"]
[package.metadata.requires-dev]
dev = [
@ -1479,6 +1484,7 @@ dev = [
{ name = "haiku-rag-evals", editable = "evaluations" },
{ name = "mkdocs", specifier = ">=1.6.1" },
{ name = "mkdocs-material", specifier = ">=9.7.6" },
{ name = "obstore", specifier = ">=0.9,<0.10" },
{ name = "pre-commit", specifier = ">=4.5.1" },
{ name = "pydantic-ai-slim", extras = ["anthropic"] },
{ name = "pydantic-ai-slim", extras = ["bedrock"] },
@ -1571,6 +1577,9 @@ mistral = [
mxbai = [
{ name = "mxbai-rerank" },
]
s3 = [
{ name = "obstore" },
]
tui = [
{ name = "textual" },
{ name = "textual-image" },
@ -1596,6 +1605,7 @@ requires-dist = [
{ name = "jsonpatch", specifier = ">=1.33" },
{ name = "lancedb", specifier = "==0.30.2" },
{ name = "mxbai-rerank", marker = "extra == 'mxbai'", specifier = ">=0.1.6" },
{ name = "obstore", marker = "extra == 's3'", specifier = ">=0.9,<0.10" },
{ name = "opencv-python-headless", marker = "extra == 'docling'", specifier = ">=4.13.0.92" },
{ name = "pathspec", specifier = ">=1.0.4" },
{ name = "pydantic", specifier = ">=2.12.5" },
@ -1620,7 +1630,7 @@ requires-dist = [
{ name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a11" },
{ name = "zstandard", marker = "python_full_version < '3.14'", specifier = ">=0.23.0" },
]
provides-extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"]
provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"]
[[package]]
name = "haiku-skills"
@ -2878,6 +2888,56 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" },
]
[[package]]
name = "obstore"
version = "0.9.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/d3/7a/3a37b0bf0da898478029fcc511a0d2a7252689b1f29e46db7ae74a219c74/obstore-0.9.4.tar.gz", hash = "sha256:e2b93f1372c59da2c7e74122fc6dc4b713d84fd4528b5b500ef7f548425496b5", size = 124167, upload-time = "2026-04-22T19:51:05.261Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ac/25/4449a0066796b91e282d7604a66387bba399b14752598c748ea9557c4c32/obstore-0.9.4-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0d17cd04e7f22960050a85f8daa6e274d693e8fb3b97b81eeaa293c6f9e62eb4", size = 4090743, upload-time = "2026-04-22T19:49:26.461Z" },
{ url = "https://files.pythonhosted.org/packages/93/91/639fe5f5644593b9f4bea66f8f29c7bfd4de3b3381fb74b4f7df678f505f/obstore-0.9.4-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4beec92710fb8826fb357baf28fb79a91ee07dcdfe73777207aa762164aaa35", size = 3876313, upload-time = "2026-04-22T19:49:28.107Z" },
{ url = "https://files.pythonhosted.org/packages/ce/71/d6675f845ebe1e3927f2dce6a2a4d5a393359274762ee00c5e6855d5f468/obstore-0.9.4-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d523c8c365ab60afb8d232614a00a92bea439a9f5c55b92486c23a47af038a1e", size = 4029950, upload-time = "2026-04-22T19:49:30.279Z" },
{ url = "https://files.pythonhosted.org/packages/0e/3a/5915a173f5c6a95f9ec186a7e29b0ce6a23bd9b04c2b0b29a351dbe2baf6/obstore-0.9.4-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee0483619088337ee365cb344fceee337e2670ec4de2a1da92ac7f6b2220f18e", size = 4129455, upload-time = "2026-04-22T19:49:31.934Z" },
{ url = "https://files.pythonhosted.org/packages/b5/a9/63c31d2d436c06c4d39ed5cb154fe54202b303854532ec09537c4ce0755b/obstore-0.9.4-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:83da348bf0a7dd84e5839c0cd54d79dcd08e0729c394e566f73a605b93b9e998", size = 4416727, upload-time = "2026-04-22T19:49:34.016Z" },
{ url = "https://files.pythonhosted.org/packages/7f/fa/23c5c6db02be0e13abcbe01c1ca94c5f7876e8c58e74cb9ac2b57b068866/obstore-0.9.4-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f282a17200bcc37b8d7a1d02a146ed41812eb6e76fd0a4c9a154f02da1b8031f", size = 4311520, upload-time = "2026-04-22T19:49:35.905Z" },
{ url = "https://files.pythonhosted.org/packages/86/f0/49f6b02dab9c05e3fd79d6129e4d9e7e9874d6e5e05369ca3b3b80a48aaa/obstore-0.9.4-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d29dcfceaa0a205ded2263d29a2a3aa206819d549e0325c1f2106f79e2658584", size = 4220536, upload-time = "2026-04-22T19:49:38.343Z" },
{ url = "https://files.pythonhosted.org/packages/50/ab/d0bfd6d68422e7d8f2204d91736c7e62767e0576ad749da442a71e7773b2/obstore-0.9.4-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:caecb912723ab8e9da8da26def249d66da4318959df2bafc0a55af64f3255902", size = 4105099, upload-time = "2026-04-22T19:49:40.384Z" },
{ url = "https://files.pythonhosted.org/packages/66/3b/f595d0ee354f9daa69438991f8818602f34bc59498c8468456a02d45fb27/obstore-0.9.4-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c1c06fec8837595a2829b5f7536d0d01e940ce10b07ad2a8594fec1cfd0b7d5", size = 4294206, upload-time = "2026-04-22T19:49:42.016Z" },
{ url = "https://files.pythonhosted.org/packages/60/54/3c5af2d59258aaa9e5bef05320658ea6e9b1f3897a3a977bf7f54a0b6ec1/obstore-0.9.4-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c132795a789ec5ade31bf4d5b55ed321fb41d9749e9145520bf19063e1da5f7b", size = 4265047, upload-time = "2026-04-22T19:49:43.983Z" },
{ url = "https://files.pythonhosted.org/packages/fb/af/a8ba1feb81b9833b253147839da40405ec6bfa51feb3abfe909c800208a5/obstore-0.9.4-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:c6e342360a5d0ae71486bc5f8311778aa144ec1a905c23593f8ef57b5bceae24", size = 4255361, upload-time = "2026-04-22T19:49:45.864Z" },
{ url = "https://files.pythonhosted.org/packages/15/f7/3ccc0288111e057f8ba3d99bee14f95d9e9bb00acaf6e9700e0eb4cd82c3/obstore-0.9.4-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aeb6f7e7e862550f5020a10692ef6f02d5ba4912dba08942eb59bb7d73f93fe0", size = 4439378, upload-time = "2026-04-22T19:49:47.581Z" },
{ url = "https://files.pythonhosted.org/packages/c4/b1/3ac8b5772743c60064f3c7e02d27f346dbb58feaa99a49ee09798d1cfb00/obstore-0.9.4-cp311-abi3-win_amd64.whl", hash = "sha256:a58ef942292841f99d69ac11d19d05544c835447c8c09dacbfb7409c6374c4a1", size = 4191594, upload-time = "2026-04-22T19:49:49.308Z" },
{ url = "https://files.pythonhosted.org/packages/9d/81/8f6b6509f8df603261cdb5ddb521c49891457775669c6ad857812bf4a7c1/obstore-0.9.4-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:fff17f59390ed307afcd1fb18c56076c1f911dd9f5c2636b7d7133c4d07f8c3f", size = 4071300, upload-time = "2026-04-22T19:49:51.386Z" },
{ url = "https://files.pythonhosted.org/packages/ab/fe/0c74ddf3ab9b24ef356925bfb613bc7846f869220361a784b63f754d8563/obstore-0.9.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4527c4c7889f1bd1f1952017d74774870e14e199d6b50b9e72f291f9498d898c", size = 3870593, upload-time = "2026-04-22T19:49:53.481Z" },
{ url = "https://files.pythonhosted.org/packages/73/fa/260ec94f9a7b4f4c8afbdd016710bed0736615488d3ac0c5620f9179bfcd/obstore-0.9.4-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a57c2016e3e569de35050f95c679ffe61813c4e3cb6d6028c4c3f57231021eb4", size = 4023990, upload-time = "2026-04-22T19:49:55.644Z" },
{ url = "https://files.pythonhosted.org/packages/8d/84/5b8e2b9607fb93c96a39a4cfa6d37bd3049ebf7265d0e9f8afa938bf32fe/obstore-0.9.4-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd5327cee4fb3578b51beb1c92915cc3a05ffe794be40f50bd68d27e97d78c5c", size = 4119971, upload-time = "2026-04-22T19:49:57.745Z" },
{ url = "https://files.pythonhosted.org/packages/3c/2b/e6c093acb7e62009d5b1678d82839903287c29d4a6e1dfbea8fbf41313d5/obstore-0.9.4-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12b1e6105eafe02d8973dbeb2d274eeac2271c67f1126ffa16f18ddea8dd5443", size = 4407147, upload-time = "2026-04-22T19:49:59.928Z" },
{ url = "https://files.pythonhosted.org/packages/ce/3d/5c93a9adee8f045b89d5f21b337f53667499db770bda129f805723ab14e4/obstore-0.9.4-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b0d378248fda4e36652808d73eaaeb7e67154427e6c724248c9b0b9b03e70a6", size = 4312215, upload-time = "2026-04-22T19:50:01.534Z" },
{ url = "https://files.pythonhosted.org/packages/9a/de/507f60b4e6a8c0cad9f93a51a7b28132c9db49e20aadbcd542fa2abc57c4/obstore-0.9.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a78fb77c346abd2bcdfa071d7166be2bdc38c28573ae5a230746df6158a5593e", size = 4216936, upload-time = "2026-04-22T19:50:03.244Z" },
{ url = "https://files.pythonhosted.org/packages/7b/ff/612bd5f8258349bfe9e8c349d184b5ea3333038d4cce0d003eefafb2160c/obstore-0.9.4-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:f4e5a6dfe6877fb599868d560d6fcf4d7416cadbdf3bd947254b53830c2f11c0", size = 4105091, upload-time = "2026-04-22T19:50:05.038Z" },
{ url = "https://files.pythonhosted.org/packages/e3/73/b083b99e7bc0b529bee7b4437cafd7cc7d9f59c10995a48b6c26447fdf7f/obstore-0.9.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f8114a2b84268c991232d89b105d9239299b6afb56e4941a61c09f3a89033022", size = 4292570, upload-time = "2026-04-22T19:50:06.823Z" },
{ url = "https://files.pythonhosted.org/packages/8b/cd/3c4555f98db9a49432bc0afa68bfc33dd47bdfa3699c915b4b0e887577e3/obstore-0.9.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:9d7b959f5f74532a142fb449c0bef5814dfe3fa5c43c31ac4284a15221a75aaf", size = 4261946, upload-time = "2026-04-22T19:50:08.789Z" },
{ url = "https://files.pythonhosted.org/packages/96/f8/bdc66df3d0dfdcfb3931a585a7fb3b74336619baf6d3540b1425b424232b/obstore-0.9.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a8e9101fc2659dd938e7ae06512075bc0a8f02ab28d2ee438d6fca8b4f3bdfba", size = 4245595, upload-time = "2026-04-22T19:50:10.765Z" },
{ url = "https://files.pythonhosted.org/packages/d7/22/1aa58ea676293e5b888391c8433ff6ab8f66622aae30427287f9daac6d46/obstore-0.9.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:538384255545b5c575497fcab26389c8f01707402b6ddcdd73b769b66311635d", size = 4436599, upload-time = "2026-04-22T19:50:12.585Z" },
{ url = "https://files.pythonhosted.org/packages/1d/9e/b52f2c97be27952d488cf1980af0c635f9947003e5744e3e1dc6252f0040/obstore-0.9.4-cp313-cp313t-win_amd64.whl", hash = "sha256:eef1c772657bb1293adad0d671ca1ff1e1dcae84ec4dfbf1a34e47c2a1f134ac", size = 4180463, upload-time = "2026-04-22T19:50:14.288Z" },
{ url = "https://files.pythonhosted.org/packages/19/76/c53583f95c6811057abd3116756dca46785318d564a0e99c207cbb2d8938/obstore-0.9.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e009e7437770c85beae4c32cb79f662f0a9922676ef127e943d107a5c082d38d", size = 4071302, upload-time = "2026-04-22T19:50:15.967Z" },
{ url = "https://files.pythonhosted.org/packages/2f/23/ac3b9c05a09b3d5f178ed6f288c5d6913df8f7386059590194e0fee65d15/obstore-0.9.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ac5f3ad314bd4592fe484b79c229518be7bb5f6218bed33c20742026d5caf860", size = 3870813, upload-time = "2026-04-22T19:50:17.62Z" },
{ url = "https://files.pythonhosted.org/packages/f7/ec/c3458e0f24d2d1a4f185f541905b07e51c91b3fec589b1600c77d511e585/obstore-0.9.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:db79d5ebc4177360565ffcec4abd49930cf052cdbeb94e3a3ece2e2d08f087d0", size = 4024237, upload-time = "2026-04-22T19:50:19.81Z" },
{ url = "https://files.pythonhosted.org/packages/a7/eb/6cf468a200e491fdc6c04075e2fbbac1707bbecd243f0f56ae1e75d052ed/obstore-0.9.4-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05b565d89c3115fb74385852dd628e12f6645a1bba97523dceae016b538a3f33", size = 4119635, upload-time = "2026-04-22T19:50:21.605Z" },
{ url = "https://files.pythonhosted.org/packages/81/fb/b44d002767fa5af95ab4ca8e16c3a9057fc11f13de03f498b99adf0c4e50/obstore-0.9.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dfc4fc98403d8fbb316eb04257c8122b6f1dda37e80869491fdacf60a815e4c", size = 4406906, upload-time = "2026-04-22T19:50:23.654Z" },
{ url = "https://files.pythonhosted.org/packages/4b/18/9a75ad5082cd581c4a55f0e62bedf4b030a8b53824976fc1f030eff225b3/obstore-0.9.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c69af620fd3d06a8cfb62d25faf1adb6ccc97cc572f47ee04dddcde5a5e5444e", size = 4311826, upload-time = "2026-04-22T19:50:25.458Z" },
{ url = "https://files.pythonhosted.org/packages/8d/03/b0f945b31f40364a7ed4dbc5677abc66331fcf478732f4d643e17e56bb13/obstore-0.9.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa78c0230e0b9d49b25ed18980e1751331ddfe05782d6ce97579a9ccda8229ea", size = 4217086, upload-time = "2026-04-22T19:50:27.266Z" },
{ url = "https://files.pythonhosted.org/packages/1a/26/bdd85264c806802086f21d73cc7c95a5baca5feeeac4bce8acb97142163f/obstore-0.9.4-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:c828719f0bb310a9cf0e0f08cb62a0b8cc550138617cb03ac897900aec9d3d47", size = 4105560, upload-time = "2026-04-22T19:50:29.324Z" },
{ url = "https://files.pythonhosted.org/packages/6a/36/4a4a6a398e5f145edd1886388ebe5e6f6bbaf74950a5dea1a6ceae63e6b5/obstore-0.9.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:49a0455519f284b6bc2e0694298114926aff1d1f3d5d344e9163e03b446826cc", size = 4292582, upload-time = "2026-04-22T19:50:31.028Z" },
{ url = "https://files.pythonhosted.org/packages/39/4c/9caa197cd2eba726e9a5285db34027049b9527a23e1a7e08479678ad6a4a/obstore-0.9.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf437309fc0fe852591ae50405300490229f876ea06574651fd753ca3fd23f25", size = 4261613, upload-time = "2026-04-22T19:50:32.868Z" },
{ url = "https://files.pythonhosted.org/packages/b8/94/a3fbe6fb3ee1c57fd4943ddbb21848eea3925b77e0789614c857d86b795e/obstore-0.9.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d83dbd20b6a5d42e35794ef64046de39040854829ec4f1eb2f6dfb54df48cc3d", size = 4245638, upload-time = "2026-04-22T19:50:35.009Z" },
{ url = "https://files.pythonhosted.org/packages/56/a7/d18e168f318327d63512dfa7cf3b5e89ed9bfba6d6a8917ad7d4700b8657/obstore-0.9.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5a0c337f37f30a2d66555d69bf3abd840457a279c57ede93bd02e014721ed364", size = 4437226, upload-time = "2026-04-22T19:50:36.635Z" },
{ url = "https://files.pythonhosted.org/packages/f4/ce/66aadd155db1e273c6ec2236c0fb904666d10c2e3b791b40624c272e586c/obstore-0.9.4-cp314-cp314t-win_amd64.whl", hash = "sha256:24e37a1c713c95a964e119f8ef879415a495432162e74e80ed29d645aeeca114", size = 4180746, upload-time = "2026-04-22T19:50:38.396Z" },
]
[[package]]
name = "omegaconf"
version = "2.3.0"
@ -3835,16 +3895,16 @@ wheels = [
[[package]]
name = "pydantic-settings"
version = "2.14.0"
version = "2.14.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic" },
{ name = "python-dotenv" },
{ name = "typing-inspection" },
]
sdist = { url = "https://files.pythonhosted.org/packages/42/98/c8345dccdc31de4228c039a98f6467a941e39558da41c1744fbe29fa5666/pydantic_settings-2.14.0.tar.gz", hash = "sha256:24285fd4b0e0c06507dd9fdfd331ee23794305352aaec8fc4eb92d4047aeb67d", size = 235709, upload-time = "2026-04-20T13:37:40.293Z" }
sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/01/dd/bebff3040138f00ae8a102d426b27349b9a49acc310fcae7f92112d867e3/pydantic_settings-2.14.0-py3-none-any.whl", hash = "sha256:fc8d5d692eb7092e43c8647c1c35a3ecd00e040fcf02ed86f4cb5458ca62182e", size = 60940, upload-time = "2026-04-20T13:37:38.586Z" },
{ url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" },
]
[[package]]