import hashlib import mimetypes from collections.abc import AsyncIterator from datetime import UTC, datetime from urllib.parse import urlparse from haiku.rag.client.exceptions import UnsupportedSourceError from haiku.rag.ingester.sources.base import ( FetchResult, RevisionSnapshot, SourceEvent, SourceEventKind, check_file_size, ) from haiku.rag.ingester.sources.filter import ( FileFilter, _default_supported_extensions, ) def _parse_s3_uri(uri: str) -> tuple[str, str]: parsed = urlparse(uri) if parsed.scheme != "s3" or not parsed.netloc: raise UnsupportedSourceError(f"Invalid S3 URI: {uri}") return parsed.netloc, parsed.path.lstrip("/") def _parse_s3_object_uri(uri: str) -> tuple[str, str]: """Like _parse_s3_uri but rejects bucket-only URIs (no key).""" bucket, key = _parse_s3_uri(uri) if not key: raise UnsupportedSourceError(f"Invalid S3 URI: {uri}") return bucket, key class S3Source: def __init__( self, *, uri: str, storage_options: dict[str, str] | None = None, ignore_patterns: list[str] | None = None, include_patterns: list[str] | None = None, supported_extensions: list[str] | None = None, source_id: str | None = None, max_file_size: int | None = None, ) -> None: self.bucket, self.prefix = _parse_s3_uri(uri) # uri_prefix is the canonical "everything I own" — used by supports() # to scope dispatch, and by discover() to build per-key URIs. self.uri_prefix = f"s3://{self.bucket}/{self.prefix}" self.source_id = source_id or f"s3:{self.bucket}/{self.prefix}" self.storage_options = dict(storage_options or {}) self.supported_extensions = ( list(supported_extensions) if supported_extensions is not None else _default_supported_extensions() ) self.filter = FileFilter( ignore_patterns=ignore_patterns, include_patterns=include_patterns, supported_extensions=self.supported_extensions, ) self._max_file_size = max_file_size def supports(self, uri: str) -> bool: return uri.startswith(self.uri_prefix) async def aclose(self) -> None: # pragma: no cover - no resources to release pass async def head(self, uri: str) -> str | None: import obstore # type: ignore[import-not-found] from haiku.rag.s3 import make_s3_store bucket, key = _parse_s3_object_uri(uri) store = make_s3_store(bucket, self.storage_options) head = await obstore.head_async(store, key) return (head.get("e_tag") or "").strip('"').strip() or None async def fetch(self, uri: str) -> FetchResult: import obstore # type: ignore[import-not-found] from haiku.rag.s3 import make_s3_store bucket, key = _parse_s3_object_uri(uri) store = make_s3_store(bucket, self.storage_options) head = await obstore.head_async(store, key) etag = (head.get("e_tag") or "").strip('"').strip() or None size = head.get("size") or head.get("content_length") if size is not None: check_file_size(int(size), self._max_file_size, uri) resp = await obstore.get_async(store, key) body = await resp.bytes_async() # obstore returns a Bytes view; convert to plain bytes so the rest of # the pipeline (hashing, tempfile write) doesn't have to know. body = bytes(body) content_type, _ = mimetypes.guess_type(key) if not content_type: content_type = "application/octet-stream" return FetchResult( uri=uri, body=body, content_type=content_type, content_hash=hashlib.md5(body, usedforsecurity=False).hexdigest(), revision=etag, ) async def discover( self, since: RevisionSnapshot | None = None, *, known_uris: set[str] | None = None, ) -> AsyncIterator[SourceEvent]: import obstore # type: ignore[import-not-found] from haiku.rag.s3 import make_s3_store snapshot: dict[str, str] = dict(since) if since else {} known = known_uris or set() now = datetime.now(UTC) seen: set[str] = set() store = make_s3_store(self.bucket, self.storage_options) async for batch in obstore.list(store, prefix=self.prefix or None): for obj in batch: key = obj["path"] if not self.filter.include_file(key): continue uri = f"s3://{self.bucket}/{key}" seen.add(uri) revision = (obj.get("e_tag") or "").strip('"').strip() or None if revision is not None and snapshot.get(uri) == revision: kind = SourceEventKind.UNCHANGED elif revision is None and uri in known: kind = SourceEventKind.UNCHANGED else: kind = SourceEventKind.UPSERT yield SourceEvent( source_id=self.source_id, uri=uri, kind=kind, revision=revision, discovered_at=now, ) # URIs previously known to this source that no longer appear under # the prefix have been deleted upstream — emit DELETE so the poller # cleans up. for uri in known - seen: yield SourceEvent( source_id=self.source_id, uri=uri, kind=SourceEventKind.DELETE, revision=None, discovered_at=now, )