da6cdfbc51
urlparse().path keeps the leading slash in front of a Windows drive, so file:///C:/docs/a.pdf read as \C:\docs\a.pdf and the ingester reported "File does not exist" for every file it discovered. url2pathname is the stdlib conversion that strips it, per platform. Four sites each decided both "is this local" and "what path is this": FSSource._uri_to_path and supports, resolve_adhoc_fetcher, create_document_from_source and check_source_accessible, and convert. is_local_uri and uri_to_path in haiku.rag.uri own those two decisions now, which closes two more cases of the same root cause. A bare C:\docs\a.pdf parses with scheme "c", so add-src raised "No source adapter for URI scheme 'c'" and convert silently treated the path as raw text. And convert and check_source_accessible never percent-decoded at all, so a file named a[b] c.md read as missing on Linux and macOS too. A file URI's host is reattached after conversion rather than passed to url2pathname, which as of 3.14 rejects a non-local authority off Windows. file:////server/share is the empty-authority spelling of a UNC path, its host being the first path segment, so that host is normalised into the authority before conversion. Output is identical on 3.12, 3.13 and 3.14. The ad-hoc FS fetcher roots at the path's own anchor rather than "/", which on Windows is only the current drive. test_uri.py runs on ubuntu, macos and windows across 3.13 and 3.14 without the project installed: --noconftest because the repo conftest imports dependencies that job does not need, and -o addopts= to drop the repository's -n auto. The Windows legs are what cover the drive conversion. Fixes #574.
68 lines
2.4 KiB
Python
68 lines
2.4 KiB
Python
from collections.abc import Iterable
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
from haiku.rag.client.exceptions import UnsupportedSourceError
|
|
from haiku.rag.sources.base import Source
|
|
from haiku.rag.sources.fs import FSSource
|
|
from haiku.rag.sources.http import HTTPSource
|
|
from haiku.rag.sources.s3 import S3Source
|
|
from haiku.rag.uri import is_local_uri, uri_to_path
|
|
|
|
|
|
def resolve_configured_source(
|
|
uri: str,
|
|
source_id: str,
|
|
sources: Iterable[Source] | None,
|
|
) -> Source:
|
|
"""Strict lookup: return the configured source with this id, or raise.
|
|
|
|
Worker jobs carry source_id from when they were enqueued. Falling back
|
|
to an ad-hoc fetcher would silently drop credentials when a source has
|
|
been renamed or removed from config — better to raise and let the job
|
|
DLQ so the misconfiguration surfaces.
|
|
"""
|
|
for src in sources or ():
|
|
if src.source_id == source_id:
|
|
if not src.supports(uri):
|
|
raise UnsupportedSourceError(
|
|
f"Source {source_id!r} doesn't support URI {uri!r}"
|
|
)
|
|
return src
|
|
raise UnsupportedSourceError(
|
|
f"No configured source with id {source_id!r} for URI {uri!r}"
|
|
)
|
|
|
|
|
|
def resolve_adhoc_fetcher(
|
|
uri: str,
|
|
*,
|
|
sources: Iterable[Source] | None = None,
|
|
storage_options: dict[str, str] | None = None,
|
|
) -> Source:
|
|
"""Best-effort lookup for one-shot fetches (e.g. ``add-src <uri>``).
|
|
|
|
Configured ``sources`` win when one matches; otherwise a scheme-based
|
|
adapter is built so any URI can be fetched without configuration.
|
|
"""
|
|
if sources:
|
|
for src in sources:
|
|
if src.supports(uri):
|
|
return src
|
|
|
|
if is_local_uri(uri):
|
|
# Root only matters for discover(); fetch() needs an absolute path
|
|
# that already encodes the location, so the path's own anchor is
|
|
# enough. On Windows "/" is only the current drive.
|
|
return FSSource(root=Path(uri_to_path(uri).anchor or "/"))
|
|
|
|
scheme = urlparse(uri).scheme
|
|
if scheme in ("http", "https"):
|
|
return HTTPSource(source_id="http:adhoc")
|
|
if scheme == "s3":
|
|
bucket = urlparse(uri).netloc
|
|
if not bucket:
|
|
raise UnsupportedSourceError(f"Invalid S3 URI: {uri}")
|
|
return S3Source(uri=f"s3://{bucket}/", storage_options=storage_options)
|
|
|
|
raise UnsupportedSourceError(f"No source adapter for URI scheme {scheme!r}: {uri}")
|