ab19f78507
haiku.rag.ingester.sources was never ingester-only: one-shot client ingestion resolves adapters through it (create_document_from_source), and convert() now fetches through HTTPSource, so the core client imported into the ingester package to reach them. Move the package to haiku.rag.sources and update every import. No shims: haiku.rag.ingester.sources is gone. The haiku.rag.sources plugin entry-point group is unchanged, so third-party source packages need no edit — the group name now matches the module path it always implied. Source unit tests move to tests/sources/. test_source_plugins.py stays in tests/ingester/: it drives a PeriodicPoller against the job repo, so it is plugin wiring through ingester machinery rather than a source test.
48 lines
1.5 KiB
Python
48 lines
1.5 KiB
Python
from importlib.metadata import entry_points
|
|
from typing import Any, Protocol, runtime_checkable
|
|
|
|
from haiku.rag.sources.base import Source
|
|
|
|
|
|
@runtime_checkable
|
|
class SourceFactory(Protocol):
|
|
"""Builds a Source from a plugin source's config. A package registers a
|
|
factory under the ``haiku.rag.sources`` entry-point group; the ingester
|
|
calls it with the source id, the config's opaque ``options`` (which the
|
|
plugin validates itself), and the ambient extension/size limits."""
|
|
|
|
def __call__(
|
|
self,
|
|
*,
|
|
source_id: str,
|
|
options: dict[str, Any],
|
|
supported_extensions: list[str] | None,
|
|
max_file_size: int | None,
|
|
) -> Source: ...
|
|
|
|
|
|
@runtime_checkable
|
|
class LoadableEntryPoint(Protocol):
|
|
"""The slice of ``importlib.metadata.EntryPoint`` the factory loader needs:
|
|
a deferred ``load()`` returning the source factory."""
|
|
|
|
def load(self) -> SourceFactory: ...
|
|
|
|
|
|
ENTRY_POINT_GROUP = "haiku.rag.sources"
|
|
|
|
|
|
def load_source_factories() -> dict[str, LoadableEntryPoint]:
|
|
"""Discover registered source-factory entry points, keyed by name. The
|
|
entry points are not imported here; ``build_source`` loads only the one a
|
|
source references, so an unused plugin with a broken import does not fail
|
|
the ingester at startup."""
|
|
return {ep.name: ep for ep in entry_points(group=ENTRY_POINT_GROUP)}
|
|
|
|
|
|
__all__ = [
|
|
"ENTRY_POINT_GROUP",
|
|
"LoadableEntryPoint",
|
|
"SourceFactory",
|
|
"load_source_factories",
|
|
]
|