Handle WebDAV as source
This commit is contained in:
parent
13cbadeb6f
commit
65c309e12b
10 changed files with 756 additions and 5 deletions
|
|
@ -3,7 +3,7 @@
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- New `haiku-ingester` service for continuous document ingestion: persistent SQLite job queue, async worker pool with retries and a dead-letter queue, FS/HTTP/S3 source adapters with per-source circuit breakers, and a FastAPI control plane (`/health`, `/jobs`, `/sources`, `/dlq`). Configured under `ingester:` in `haiku.rag.yaml`. Shipped behind the `[ingester]` extra, with Logfire spans (`ingester.poller.sweep` → `ingester.job` → `document.{fetch,convert,chunk,embed,store}`) for traceable ingestion. See [docs/ingester.md](docs/ingester.md).
|
- New `haiku-ingester` service for continuous document ingestion: persistent SQLite job queue, async worker pool with retries and a dead-letter queue, FS/HTTP/S3/WebDAV source adapters with per-source circuit breakers, and a FastAPI control plane (`/health`, `/jobs`, `/sources`, `/dlq`). Configured under `ingester:` in `haiku.rag.yaml`. Shipped behind the `[ingester]` extra, with Logfire spans (`ingester.poller.sweep` → `ingester.job` → `document.{fetch,convert,chunk,embed,store}`) for traceable ingestion. See [docs/ingester.md](docs/ingester.md).
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -109,6 +109,39 @@ response from a configured URL triggers a delete event; other failure
|
||||||
statuses fall through to UPSERT-with-no-revision so the worker can
|
statuses fall through to UPSERT-with-no-revision so the worker can
|
||||||
GET and decide.
|
GET and decide.
|
||||||
|
|
||||||
|
### WebDAV
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
ingester:
|
||||||
|
sources:
|
||||||
|
- type: webdav
|
||||||
|
id: nextcloud
|
||||||
|
base_url: https://nextcloud.example.com/remote.php/dav/files/alice/Documents/
|
||||||
|
username: alice
|
||||||
|
password: ${NEXTCLOUD_APP_PASSWORD}
|
||||||
|
ignore_patterns: ["**/Trash/**"]
|
||||||
|
poll_interval_s: 600
|
||||||
|
```
|
||||||
|
|
||||||
|
Each sweep issues one `PROPFIND` with `Depth: infinity` against
|
||||||
|
`base_url` and parses the multistatus response. Files (non-collection
|
||||||
|
resources) are emitted as UPSERT / UNCHANGED based on the `getetag`
|
||||||
|
property (falling back to `getlastmodified` if the server omits it);
|
||||||
|
URIs that were in the previous snapshot but no longer appear under the
|
||||||
|
collection are emitted as DELETE.
|
||||||
|
|
||||||
|
Fetches are plain HTTP GETs — any WebDAV server already supports them.
|
||||||
|
|
||||||
|
Bearer-token auth can replace HTTP Basic via the standard `headers` map:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- type: webdav
|
||||||
|
id: kdrive
|
||||||
|
base_url: https://kdrive.infomaniak.com/app/drive/123/
|
||||||
|
headers:
|
||||||
|
Authorization: Bearer ${KDRIVE_TOKEN}
|
||||||
|
```
|
||||||
|
|
||||||
## Workers and retry
|
## Workers and retry
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ from haiku.rag.config.models import (
|
||||||
S3SourceConfig,
|
S3SourceConfig,
|
||||||
SourceConfig,
|
SourceConfig,
|
||||||
StorageConfig,
|
StorageConfig,
|
||||||
|
WebDAVSourceConfig,
|
||||||
WorkerConfig,
|
WorkerConfig,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -53,6 +54,7 @@ __all__ = [
|
||||||
"S3SourceConfig",
|
"S3SourceConfig",
|
||||||
"SourceConfig",
|
"SourceConfig",
|
||||||
"StorageConfig",
|
"StorageConfig",
|
||||||
|
"WebDAVSourceConfig",
|
||||||
"WorkerConfig",
|
"WorkerConfig",
|
||||||
"find_config_file",
|
"find_config_file",
|
||||||
"generate_default_config",
|
"generate_default_config",
|
||||||
|
|
|
||||||
|
|
@ -323,8 +323,21 @@ class S3SourceConfig(_SourceBase):
|
||||||
include_patterns: list[str] = []
|
include_patterns: list[str] = []
|
||||||
|
|
||||||
|
|
||||||
|
class WebDAVSourceConfig(_SourceBase):
|
||||||
|
"""A WebDAV collection (Nextcloud, ownCloud, Apache mod_dav, etc.). Files
|
||||||
|
are discovered via PROPFIND on `base_url`; fetch is plain HTTP GET."""
|
||||||
|
|
||||||
|
type: Literal["webdav"]
|
||||||
|
base_url: str
|
||||||
|
username: str | None = None
|
||||||
|
password: str | None = None
|
||||||
|
headers: dict[str, str] = Field(default_factory=dict)
|
||||||
|
ignore_patterns: list[str] = []
|
||||||
|
include_patterns: list[str] = []
|
||||||
|
|
||||||
|
|
||||||
SourceConfig = Annotated[
|
SourceConfig = Annotated[
|
||||||
FSSourceConfig | HTTPSourceConfig | S3SourceConfig,
|
FSSourceConfig | HTTPSourceConfig | S3SourceConfig | WebDAVSourceConfig,
|
||||||
Field(discriminator="type"),
|
Field(discriminator="type"),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,15 @@ from haiku.rag.config import (
|
||||||
HTTPSourceConfig,
|
HTTPSourceConfig,
|
||||||
S3SourceConfig,
|
S3SourceConfig,
|
||||||
SourceConfig,
|
SourceConfig,
|
||||||
|
WebDAVSourceConfig,
|
||||||
|
)
|
||||||
|
from haiku.rag.ingester.sources import (
|
||||||
|
FSSource,
|
||||||
|
HTTPSource,
|
||||||
|
S3Source,
|
||||||
|
Source,
|
||||||
|
WebDAVSource,
|
||||||
)
|
)
|
||||||
from haiku.rag.ingester.sources import FSSource, HTTPSource, S3Source, Source
|
|
||||||
|
|
||||||
|
|
||||||
def build_source(
|
def build_source(
|
||||||
|
|
@ -16,7 +23,7 @@ def build_source(
|
||||||
|
|
||||||
Source IDs auto-derive from the target when the config didn't supply one,
|
Source IDs auto-derive from the target when the config didn't supply one,
|
||||||
matching the conventions in the adapters themselves (fs:<root>,
|
matching the conventions in the adapters themselves (fs:<root>,
|
||||||
s3:<bucket>/<prefix>, http:<id>).
|
s3:<bucket>/<prefix>, http:<id>, webdav:<id>).
|
||||||
"""
|
"""
|
||||||
if isinstance(cfg, FSSourceConfig):
|
if isinstance(cfg, FSSourceConfig):
|
||||||
return FSSource(
|
return FSSource(
|
||||||
|
|
@ -39,4 +46,17 @@ def build_source(
|
||||||
supported_extensions=supported_extensions,
|
supported_extensions=supported_extensions,
|
||||||
source_id=cfg.id,
|
source_id=cfg.id,
|
||||||
)
|
)
|
||||||
|
if isinstance(cfg, WebDAVSourceConfig):
|
||||||
|
if cfg.id is None:
|
||||||
|
raise ValueError("WebDAVSourceConfig.id is required")
|
||||||
|
return WebDAVSource(
|
||||||
|
source_id=cfg.id,
|
||||||
|
base_url=cfg.base_url,
|
||||||
|
username=cfg.username,
|
||||||
|
password=cfg.password,
|
||||||
|
headers=cfg.headers,
|
||||||
|
ignore_patterns=cfg.ignore_patterns or None,
|
||||||
|
include_patterns=cfg.include_patterns or None,
|
||||||
|
supported_extensions=supported_extensions,
|
||||||
|
)
|
||||||
raise TypeError(f"Unsupported source config: {type(cfg).__name__}")
|
raise TypeError(f"Unsupported source config: {type(cfg).__name__}")
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ from haiku.rag.ingester.sources.fs import FSSource
|
||||||
from haiku.rag.ingester.sources.http import HTTPSource
|
from haiku.rag.ingester.sources.http import HTTPSource
|
||||||
from haiku.rag.ingester.sources.registry import resolve_fetcher
|
from haiku.rag.ingester.sources.registry import resolve_fetcher
|
||||||
from haiku.rag.ingester.sources.s3 import S3Source
|
from haiku.rag.ingester.sources.s3 import S3Source
|
||||||
|
from haiku.rag.ingester.sources.webdav import WebDAVSource
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"FetchResult",
|
"FetchResult",
|
||||||
|
|
@ -21,5 +22,6 @@ __all__ = [
|
||||||
"Source",
|
"Source",
|
||||||
"SourceEvent",
|
"SourceEvent",
|
||||||
"SourceEventKind",
|
"SourceEventKind",
|
||||||
|
"WebDAVSource",
|
||||||
"resolve_fetcher",
|
"resolve_fetcher",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
292
haiku_rag_slim/haiku/rag/ingester/sources/webdav.py
Normal file
292
haiku_rag_slim/haiku/rag/ingester/sources/webdav.py
Normal file
|
|
@ -0,0 +1,292 @@
|
||||||
|
import hashlib
|
||||||
|
from collections.abc import AsyncIterator
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from urllib.parse import unquote, urljoin, urlparse
|
||||||
|
from xml.etree.ElementTree import Element, fromstring
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from haiku.rag.ingester.sources.base import (
|
||||||
|
FetchResult,
|
||||||
|
RevisionSnapshot,
|
||||||
|
SourceEvent,
|
||||||
|
SourceEventKind,
|
||||||
|
)
|
||||||
|
from haiku.rag.ingester.sources.filter import (
|
||||||
|
FileFilter,
|
||||||
|
_default_supported_extensions,
|
||||||
|
)
|
||||||
|
|
||||||
|
# WebDAV PROPFIND uses XML with the DAV: namespace. Element tags arrive as
|
||||||
|
# Clark-notation strings like "{DAV:}response", so we precompile them once.
|
||||||
|
_DAV_NS = "DAV:"
|
||||||
|
_TAG_RESPONSE = f"{{{_DAV_NS}}}response"
|
||||||
|
_TAG_HREF = f"{{{_DAV_NS}}}href"
|
||||||
|
_TAG_PROPSTAT = f"{{{_DAV_NS}}}propstat"
|
||||||
|
_TAG_PROP = f"{{{_DAV_NS}}}prop"
|
||||||
|
_TAG_STATUS = f"{{{_DAV_NS}}}status"
|
||||||
|
_TAG_RESOURCETYPE = f"{{{_DAV_NS}}}resourcetype"
|
||||||
|
_TAG_COLLECTION = f"{{{_DAV_NS}}}collection"
|
||||||
|
_TAG_GETETAG = f"{{{_DAV_NS}}}getetag"
|
||||||
|
_TAG_GETLASTMODIFIED = f"{{{_DAV_NS}}}getlastmodified"
|
||||||
|
_TAG_GETCONTENTTYPE = f"{{{_DAV_NS}}}getcontenttype"
|
||||||
|
|
||||||
|
|
||||||
|
_PROPFIND_BODY = b"""<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<propfind xmlns="DAV:">
|
||||||
|
<prop>
|
||||||
|
<resourcetype/>
|
||||||
|
<getetag/>
|
||||||
|
<getlastmodified/>
|
||||||
|
<getcontenttype/>
|
||||||
|
</prop>
|
||||||
|
</propfind>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class _PropfindEntry:
|
||||||
|
"""One <response> element decoded into the fields the source actually
|
||||||
|
uses. `is_collection` separates folders from files; revision is ETag
|
||||||
|
when present, otherwise the Last-Modified header value."""
|
||||||
|
|
||||||
|
__slots__ = ("href", "is_collection", "revision", "content_type")
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
href: str,
|
||||||
|
*,
|
||||||
|
is_collection: bool,
|
||||||
|
revision: str | None,
|
||||||
|
content_type: str | None,
|
||||||
|
) -> None:
|
||||||
|
self.href = href
|
||||||
|
self.is_collection = is_collection
|
||||||
|
self.revision = revision
|
||||||
|
self.content_type = content_type
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_etag(value: str | None) -> str | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
cleaned = value.strip().strip('"').strip("W/").strip().strip('"')
|
||||||
|
return cleaned or None
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_from_response(response: Element) -> _PropfindEntry | None:
|
||||||
|
"""Decode a single <response>. Returns None if the prop block is missing
|
||||||
|
or the entry didn't return HTTP 200 (e.g. 404 for a known-bad path)."""
|
||||||
|
href_el = response.find(_TAG_HREF)
|
||||||
|
if href_el is None or not href_el.text:
|
||||||
|
return None
|
||||||
|
href = href_el.text
|
||||||
|
|
||||||
|
is_collection = False
|
||||||
|
revision: str | None = None
|
||||||
|
last_modified: str | None = None
|
||||||
|
content_type: str | None = None
|
||||||
|
ok = False
|
||||||
|
|
||||||
|
for propstat in response.findall(_TAG_PROPSTAT):
|
||||||
|
status_el = propstat.find(_TAG_STATUS)
|
||||||
|
if status_el is None or not status_el.text:
|
||||||
|
continue
|
||||||
|
if " 200 " not in status_el.text:
|
||||||
|
continue
|
||||||
|
ok = True
|
||||||
|
prop = propstat.find(_TAG_PROP)
|
||||||
|
if prop is None:
|
||||||
|
continue
|
||||||
|
resourcetype = prop.find(_TAG_RESOURCETYPE)
|
||||||
|
if resourcetype is not None and resourcetype.find(_TAG_COLLECTION) is not None:
|
||||||
|
is_collection = True
|
||||||
|
etag_el = prop.find(_TAG_GETETAG)
|
||||||
|
if etag_el is not None and etag_el.text:
|
||||||
|
revision = _strip_etag(etag_el.text)
|
||||||
|
lm_el = prop.find(_TAG_GETLASTMODIFIED)
|
||||||
|
if lm_el is not None and lm_el.text:
|
||||||
|
last_modified = lm_el.text.strip() or None
|
||||||
|
ct_el = prop.find(_TAG_GETCONTENTTYPE)
|
||||||
|
if ct_el is not None and ct_el.text:
|
||||||
|
content_type = ct_el.text.split(";")[0].strip().lower() or None
|
||||||
|
|
||||||
|
if not ok:
|
||||||
|
return None
|
||||||
|
# Prefer ETag — stronger validator. Fall back to Last-Modified so revision
|
||||||
|
# detection still works against servers that don't return ETags on PROPFIND.
|
||||||
|
return _PropfindEntry(
|
||||||
|
href=href,
|
||||||
|
is_collection=is_collection,
|
||||||
|
revision=revision or last_modified,
|
||||||
|
content_type=content_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_multistatus(body: bytes) -> list[_PropfindEntry]:
|
||||||
|
"""Top-level multistatus parser. Raises ValueError on garbage XML so the
|
||||||
|
poller's circuit breaker can record a failure."""
|
||||||
|
try:
|
||||||
|
root = fromstring(body)
|
||||||
|
except Exception as exc: # ParseError + any defensive surprise
|
||||||
|
raise ValueError(f"Invalid PROPFIND response XML: {exc}") from exc
|
||||||
|
return [
|
||||||
|
entry
|
||||||
|
for response in root.findall(_TAG_RESPONSE)
|
||||||
|
if (entry := _entry_from_response(response)) is not None
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_href(href: str, base_url: str) -> str:
|
||||||
|
"""PROPFIND href values can be either absolute URLs or server-relative
|
||||||
|
paths. Resolve to absolute against base_url either way, then URL-decode
|
||||||
|
the path so the URI we store matches what a user would type."""
|
||||||
|
absolute = urljoin(base_url, href)
|
||||||
|
parsed = urlparse(absolute)
|
||||||
|
decoded_path = unquote(parsed.path)
|
||||||
|
rebuilt = parsed._replace(path=decoded_path)
|
||||||
|
return rebuilt.geturl()
|
||||||
|
|
||||||
|
|
||||||
|
class WebDAVSource:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_id: str,
|
||||||
|
base_url: str,
|
||||||
|
username: str | None = None,
|
||||||
|
password: str | None = None,
|
||||||
|
headers: dict[str, str] | None = None,
|
||||||
|
ignore_patterns: list[str] | None = None,
|
||||||
|
include_patterns: list[str] | None = None,
|
||||||
|
supported_extensions: list[str] | None = None,
|
||||||
|
transport: httpx.AsyncBaseTransport | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.source_id = source_id
|
||||||
|
# Trailing slash matters: urljoin treats path-without-slash as a sibling
|
||||||
|
# link, so "https://srv/dav" joined with "subdir/x" gives ".../x".
|
||||||
|
self.base_url = base_url if base_url.endswith("/") else base_url + "/"
|
||||||
|
self.username = username
|
||||||
|
self.password = password
|
||||||
|
self.headers = dict(headers or {})
|
||||||
|
self.supported_extensions = (
|
||||||
|
list(supported_extensions)
|
||||||
|
if supported_extensions is not None
|
||||||
|
else _default_supported_extensions()
|
||||||
|
)
|
||||||
|
self.filter = FileFilter(
|
||||||
|
ignore_patterns=ignore_patterns,
|
||||||
|
include_patterns=include_patterns,
|
||||||
|
supported_extensions=self.supported_extensions,
|
||||||
|
)
|
||||||
|
# transport is for testing — production callers leave it None.
|
||||||
|
self._transport = transport
|
||||||
|
|
||||||
|
def supports(self, uri: str) -> bool:
|
||||||
|
return uri.startswith(self.base_url)
|
||||||
|
|
||||||
|
def _client(self) -> httpx.AsyncClient:
|
||||||
|
auth = (
|
||||||
|
(self.username, self.password)
|
||||||
|
if self.username is not None and self.password is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
return httpx.AsyncClient(
|
||||||
|
auth=auth, headers=self.headers, transport=self._transport
|
||||||
|
)
|
||||||
|
|
||||||
|
async def head(self, uri: str) -> str | None:
|
||||||
|
async with self._client() as http:
|
||||||
|
response = await http.request(
|
||||||
|
"PROPFIND",
|
||||||
|
uri,
|
||||||
|
headers={"Depth": "0", "Content-Type": "application/xml"},
|
||||||
|
content=_PROPFIND_BODY,
|
||||||
|
)
|
||||||
|
if response.is_error:
|
||||||
|
return None
|
||||||
|
entries = _parse_multistatus(response.content)
|
||||||
|
if not entries:
|
||||||
|
return None
|
||||||
|
return entries[0].revision
|
||||||
|
|
||||||
|
async def fetch(self, uri: str) -> FetchResult:
|
||||||
|
async with self._client() as http:
|
||||||
|
response = await http.get(uri)
|
||||||
|
response.raise_for_status()
|
||||||
|
body = response.content
|
||||||
|
content_type = (
|
||||||
|
response.headers.get("content-type", "application/octet-stream")
|
||||||
|
.split(";")[0]
|
||||||
|
.strip()
|
||||||
|
.lower()
|
||||||
|
)
|
||||||
|
# ETag from the GET response is the freshest revision; fall back to
|
||||||
|
# Last-Modified, matching HTTPSource's preference order.
|
||||||
|
revision = (
|
||||||
|
_strip_etag(response.headers.get("etag"))
|
||||||
|
or (response.headers.get("last-modified") or "").strip()
|
||||||
|
or None
|
||||||
|
)
|
||||||
|
extra: dict[str, str] = {}
|
||||||
|
last_modified = (response.headers.get("last-modified") or "").strip()
|
||||||
|
if last_modified:
|
||||||
|
extra["last_modified"] = last_modified
|
||||||
|
return FetchResult(
|
||||||
|
uri=uri,
|
||||||
|
body=body,
|
||||||
|
content_type=content_type,
|
||||||
|
content_hash=hashlib.md5(body, usedforsecurity=False).hexdigest(),
|
||||||
|
revision=revision,
|
||||||
|
extra_metadata=extra,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def discover(
|
||||||
|
self, since: RevisionSnapshot | None = None
|
||||||
|
) -> AsyncIterator[SourceEvent]:
|
||||||
|
snapshot: dict[str, str] = dict(since) if since else {}
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
seen: set[str] = set()
|
||||||
|
|
||||||
|
async with self._client() as http:
|
||||||
|
response = await http.request(
|
||||||
|
"PROPFIND",
|
||||||
|
self.base_url,
|
||||||
|
headers={"Depth": "infinity", "Content-Type": "application/xml"},
|
||||||
|
content=_PROPFIND_BODY,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
entries = _parse_multistatus(response.content)
|
||||||
|
|
||||||
|
for entry in entries:
|
||||||
|
if entry.is_collection:
|
||||||
|
continue
|
||||||
|
uri = _resolve_href(entry.href, self.base_url)
|
||||||
|
# The base URL itself sometimes appears as a non-collection on
|
||||||
|
# broken servers; skip anything that's not strictly under it.
|
||||||
|
if uri == self.base_url.rstrip("/") or not uri.startswith(self.base_url):
|
||||||
|
continue
|
||||||
|
if not self.filter.include_file(uri):
|
||||||
|
continue
|
||||||
|
seen.add(uri)
|
||||||
|
revision = entry.revision
|
||||||
|
if revision is not None and snapshot.get(uri) == revision:
|
||||||
|
kind = SourceEventKind.UNCHANGED
|
||||||
|
else:
|
||||||
|
kind = SourceEventKind.UPSERT
|
||||||
|
yield SourceEvent(
|
||||||
|
source_id=self.source_id,
|
||||||
|
uri=uri,
|
||||||
|
kind=kind,
|
||||||
|
revision=revision,
|
||||||
|
discovered_at=now,
|
||||||
|
)
|
||||||
|
|
||||||
|
for uri in snapshot:
|
||||||
|
if uri in seen:
|
||||||
|
continue
|
||||||
|
yield SourceEvent(
|
||||||
|
source_id=self.source_id,
|
||||||
|
uri=uri,
|
||||||
|
kind=SourceEventKind.DELETE,
|
||||||
|
revision=None,
|
||||||
|
discovered_at=now,
|
||||||
|
)
|
||||||
|
|
@ -48,6 +48,27 @@ def test_discriminator_picks_s3_source():
|
||||||
assert cfg.sources[0].uri == "s3://bucket/prefix/"
|
assert cfg.sources[0].uri == "s3://bucket/prefix/"
|
||||||
|
|
||||||
|
|
||||||
|
def test_discriminator_picks_webdav_source():
|
||||||
|
cfg = IngesterConfig.model_validate(
|
||||||
|
{
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"type": "webdav",
|
||||||
|
"id": "nc",
|
||||||
|
"base_url": "https://nc.example.com/dav/",
|
||||||
|
"username": "alice",
|
||||||
|
"password": "hunter2",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
from haiku.rag.config import WebDAVSourceConfig
|
||||||
|
|
||||||
|
assert isinstance(cfg.sources[0], WebDAVSourceConfig)
|
||||||
|
assert cfg.sources[0].base_url == "https://nc.example.com/dav/"
|
||||||
|
assert cfg.sources[0].username == "alice"
|
||||||
|
|
||||||
|
|
||||||
def test_discriminator_rejects_unknown_type():
|
def test_discriminator_rejects_unknown_type():
|
||||||
with pytest.raises(ValidationError):
|
with pytest.raises(ValidationError):
|
||||||
IngesterConfig.model_validate({"sources": [{"type": "ftp", "uri": "x"}]})
|
IngesterConfig.model_validate({"sources": [{"type": "ftp", "uri": "x"}]})
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ from haiku.rag.config import (
|
||||||
FSSourceConfig,
|
FSSourceConfig,
|
||||||
HTTPSourceConfig,
|
HTTPSourceConfig,
|
||||||
S3SourceConfig,
|
S3SourceConfig,
|
||||||
|
WebDAVSourceConfig,
|
||||||
)
|
)
|
||||||
from haiku.rag.ingester.pollers.circuit_breaker import CircuitBreaker
|
from haiku.rag.ingester.pollers.circuit_breaker import CircuitBreaker
|
||||||
from haiku.rag.ingester.pollers.manager import PollerManager
|
from haiku.rag.ingester.pollers.manager import PollerManager
|
||||||
|
|
@ -278,6 +279,9 @@ async def test_manager_builds_pollers_per_source(tmp_path, jobs, sync):
|
||||||
FSSourceConfig(type="fs", root=tmp_path),
|
FSSourceConfig(type="fs", root=tmp_path),
|
||||||
S3SourceConfig(type="s3", uri="s3://bucket/"),
|
S3SourceConfig(type="s3", uri="s3://bucket/"),
|
||||||
HTTPSourceConfig(type="http", id="urls", urls=[]),
|
HTTPSourceConfig(type="http", id="urls", urls=[]),
|
||||||
|
WebDAVSourceConfig(
|
||||||
|
type="webdav", id="nc", base_url="https://nc.example.com/dav/"
|
||||||
|
),
|
||||||
]
|
]
|
||||||
manager = PollerManager(
|
manager = PollerManager(
|
||||||
configs=configs,
|
configs=configs,
|
||||||
|
|
@ -285,11 +289,12 @@ async def test_manager_builds_pollers_per_source(tmp_path, jobs, sync):
|
||||||
sync_repo=sync,
|
sync_repo=sync,
|
||||||
)
|
)
|
||||||
built = manager.build_pollers()
|
built = manager.build_pollers()
|
||||||
assert len(built) == 3
|
assert len(built) == 4
|
||||||
assert {p.source_id for p in built} == {
|
assert {p.source_id for p in built} == {
|
||||||
f"fs:{tmp_path.resolve()}",
|
f"fs:{tmp_path.resolve()}",
|
||||||
"s3:bucket/",
|
"s3:bucket/",
|
||||||
"urls",
|
"urls",
|
||||||
|
"nc",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
363
tests/ingester/test_webdav_source.py
Normal file
363
tests/ingester/test_webdav_source.py
Normal file
|
|
@ -0,0 +1,363 @@
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from haiku.rag.ingester.sources.base import SourceEventKind
|
||||||
|
from haiku.rag.ingester.sources.webdav import WebDAVSource
|
||||||
|
|
||||||
|
|
||||||
|
def _transport(handler) -> httpx.MockTransport:
|
||||||
|
return httpx.MockTransport(handler)
|
||||||
|
|
||||||
|
|
||||||
|
def _multistatus(*entries: dict) -> bytes:
|
||||||
|
"""Build a <multistatus> response. Each entry is a dict like
|
||||||
|
{'href': '/dav/x.md', 'collection': False, 'etag': '"abc"',
|
||||||
|
'last_modified': 'Wed, ...', 'content_type': 'text/markdown'}."""
|
||||||
|
body = ['<?xml version="1.0" encoding="utf-8"?>', '<d:multistatus xmlns:d="DAV:">']
|
||||||
|
for e in entries:
|
||||||
|
body.append(" <d:response>")
|
||||||
|
body.append(f" <d:href>{e['href']}</d:href>")
|
||||||
|
body.append(" <d:propstat>")
|
||||||
|
body.append(" <d:status>HTTP/1.1 200 OK</d:status>")
|
||||||
|
body.append(" <d:prop>")
|
||||||
|
body.append(" <d:resourcetype>")
|
||||||
|
if e.get("collection"):
|
||||||
|
body.append(" <d:collection/>")
|
||||||
|
body.append(" </d:resourcetype>")
|
||||||
|
if "etag" in e:
|
||||||
|
body.append(f" <d:getetag>{e['etag']}</d:getetag>")
|
||||||
|
if "last_modified" in e:
|
||||||
|
body.append(
|
||||||
|
f" <d:getlastmodified>{e['last_modified']}</d:getlastmodified>"
|
||||||
|
)
|
||||||
|
if "content_type" in e:
|
||||||
|
body.append(
|
||||||
|
f" <d:getcontenttype>{e['content_type']}</d:getcontenttype>"
|
||||||
|
)
|
||||||
|
body.append(" </d:prop>")
|
||||||
|
body.append(" </d:propstat>")
|
||||||
|
body.append(" </d:response>")
|
||||||
|
body.append("</d:multistatus>")
|
||||||
|
return "\n".join(body).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def test_supports_uri_under_base_url():
|
||||||
|
src = WebDAVSource(source_id="nc", base_url="https://nc.example.com/dav/")
|
||||||
|
assert src.supports("https://nc.example.com/dav/file.md")
|
||||||
|
assert src.supports("https://nc.example.com/dav/sub/file.md")
|
||||||
|
assert not src.supports("https://nc.example.com/other/file.md")
|
||||||
|
assert not src.supports("https://other.example.com/dav/file.md")
|
||||||
|
|
||||||
|
|
||||||
|
def test_base_url_trailing_slash_normalised():
|
||||||
|
"""base_url without trailing slash mustn't break urljoin during discovery."""
|
||||||
|
src = WebDAVSource(source_id="nc", base_url="https://nc.example.com/dav")
|
||||||
|
assert src.base_url == "https://nc.example.com/dav/"
|
||||||
|
assert src.supports("https://nc.example.com/dav/x.md")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_returns_bytes_md5_revision_and_content_type():
|
||||||
|
body = b"hello dav"
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
assert request.method == "GET"
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
content=body,
|
||||||
|
headers={
|
||||||
|
"content-type": "text/markdown; charset=utf-8",
|
||||||
|
"etag": '"rev-1"',
|
||||||
|
"last-modified": "Wed, 21 Oct 2025 07:28:00 GMT",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
result = await src.fetch("https://nc.example.com/dav/a.md")
|
||||||
|
assert result.body == body
|
||||||
|
assert result.content_hash == hashlib.md5(body, usedforsecurity=False).hexdigest()
|
||||||
|
assert result.content_type == "text/markdown"
|
||||||
|
assert result.revision == "rev-1"
|
||||||
|
assert result.extra_metadata == {"last_modified": "Wed, 21 Oct 2025 07:28:00 GMT"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_falls_back_to_last_modified_when_no_etag():
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
content=b"x",
|
||||||
|
headers={
|
||||||
|
"content-type": "text/plain",
|
||||||
|
"last-modified": "Wed, 21 Oct 2025 07:28:00 GMT",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
result = await src.fetch("https://nc.example.com/dav/a.txt")
|
||||||
|
assert result.revision == "Wed, 21 Oct 2025 07:28:00 GMT"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_head_returns_etag_from_propfind_depth_zero():
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
assert request.method == "PROPFIND"
|
||||||
|
assert request.headers["Depth"] == "0"
|
||||||
|
return httpx.Response(
|
||||||
|
207,
|
||||||
|
content=_multistatus(
|
||||||
|
{"href": "/dav/a.md", "etag": '"rev-9"'},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
assert await src.head("https://nc.example.com/dav/a.md") == "rev-9"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_head_returns_none_on_404():
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(404)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
assert await src.head("https://nc.example.com/dav/missing.md") is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discover_yields_upserts_and_skips_collections_and_unsupported():
|
||||||
|
multistatus = _multistatus(
|
||||||
|
{"href": "/dav/", "collection": True},
|
||||||
|
{"href": "/dav/sub/", "collection": True},
|
||||||
|
{"href": "/dav/a.md", "etag": '"rev-a"', "content_type": "text/markdown"},
|
||||||
|
{"href": "/dav/sub/b.txt", "etag": '"rev-b"', "content_type": "text/plain"},
|
||||||
|
{"href": "/dav/skip.log", "etag": '"rev-log"', "content_type": "text/plain"},
|
||||||
|
)
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
assert request.method == "PROPFIND"
|
||||||
|
assert request.headers["Depth"] == "infinity"
|
||||||
|
return httpx.Response(207, content=multistatus)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
|
||||||
|
events = [event async for event in src.discover()]
|
||||||
|
by_uri = {e.uri: e for e in events}
|
||||||
|
assert set(by_uri) == {
|
||||||
|
"https://nc.example.com/dav/a.md",
|
||||||
|
"https://nc.example.com/dav/sub/b.txt",
|
||||||
|
}
|
||||||
|
assert all(e.kind is SourceEventKind.UPSERT for e in events)
|
||||||
|
assert by_uri["https://nc.example.com/dav/a.md"].revision == "rev-a"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discover_emits_unchanged_when_snapshot_matches():
|
||||||
|
multistatus = _multistatus(
|
||||||
|
{"href": "/dav/", "collection": True},
|
||||||
|
{"href": "/dav/a.md", "etag": '"rev-a"', "content_type": "text/markdown"},
|
||||||
|
)
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(207, content=multistatus)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
snapshot = {"https://nc.example.com/dav/a.md": "rev-a"}
|
||||||
|
events = [event async for event in src.discover(since=snapshot)]
|
||||||
|
assert [e.kind for e in events] == [SourceEventKind.UNCHANGED]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discover_emits_delete_for_files_no_longer_listed():
|
||||||
|
multistatus = _multistatus(
|
||||||
|
{"href": "/dav/", "collection": True},
|
||||||
|
{"href": "/dav/a.md", "etag": '"rev-a"', "content_type": "text/markdown"},
|
||||||
|
)
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(207, content=multistatus)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
snapshot = {
|
||||||
|
"https://nc.example.com/dav/a.md": "rev-a",
|
||||||
|
"https://nc.example.com/dav/gone.md": "rev-old",
|
||||||
|
}
|
||||||
|
events = [event async for event in src.discover(since=snapshot)]
|
||||||
|
kinds = {e.uri: e.kind for e in events}
|
||||||
|
assert kinds == {
|
||||||
|
"https://nc.example.com/dav/a.md": SourceEventKind.UNCHANGED,
|
||||||
|
"https://nc.example.com/dav/gone.md": SourceEventKind.DELETE,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discover_uses_last_modified_when_etag_absent():
|
||||||
|
multistatus = _multistatus(
|
||||||
|
{
|
||||||
|
"href": "/dav/a.md",
|
||||||
|
"last_modified": "Wed, 21 Oct 2025 07:28:00 GMT",
|
||||||
|
"content_type": "text/markdown",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(207, content=multistatus)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
events = [event async for event in src.discover()]
|
||||||
|
assert events[0].revision == "Wed, 21 Oct 2025 07:28:00 GMT"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discover_raises_on_malformed_xml():
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(207, content=b"<not></valid xml")
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="Invalid PROPFIND"):
|
||||||
|
[event async for event in src.discover()]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discover_propagates_http_error():
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(401)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
with pytest.raises(httpx.HTTPStatusError):
|
||||||
|
[event async for event in src.discover()]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_basic_auth_sent_when_credentials_configured():
|
||||||
|
seen_auth: list[str | None] = []
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
seen_auth.append(request.headers.get("authorization"))
|
||||||
|
return httpx.Response(
|
||||||
|
207,
|
||||||
|
content=_multistatus({"href": "/dav/", "collection": True}),
|
||||||
|
)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
username="alice",
|
||||||
|
password="hunter2",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
[event async for event in src.discover()]
|
||||||
|
assert seen_auth and seen_auth[0] is not None
|
||||||
|
assert seen_auth[0].startswith("Basic ")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_custom_headers_forwarded():
|
||||||
|
seen: list[str | None] = []
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
seen.append(request.headers.get("authorization"))
|
||||||
|
return httpx.Response(
|
||||||
|
207,
|
||||||
|
content=_multistatus({"href": "/dav/", "collection": True}),
|
||||||
|
)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
headers={"Authorization": "Bearer tok-123"},
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
[event async for event in src.discover()]
|
||||||
|
assert seen == ["Bearer tok-123"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discover_resolves_absolute_href():
|
||||||
|
"""Some servers return absolute URLs in href, others return server paths.
|
||||||
|
Both must produce the same stored URI."""
|
||||||
|
multistatus = _multistatus(
|
||||||
|
{
|
||||||
|
"href": "https://nc.example.com/dav/a.md",
|
||||||
|
"etag": '"rev"',
|
||||||
|
"content_type": "text/markdown",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(207, content=multistatus)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
events = [event async for event in src.discover()]
|
||||||
|
assert [e.uri for e in events] == ["https://nc.example.com/dav/a.md"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discover_url_decodes_href_path():
|
||||||
|
"""PROPFIND hrefs are percent-encoded per RFC 3986. We unquote them so
|
||||||
|
the stored URI matches what a user types in `add-src`."""
|
||||||
|
multistatus = _multistatus(
|
||||||
|
{
|
||||||
|
"href": "/dav/my%20docs/Hello%20World.md",
|
||||||
|
"etag": '"rev"',
|
||||||
|
"content_type": "text/markdown",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(207, content=multistatus)
|
||||||
|
|
||||||
|
src = WebDAVSource(
|
||||||
|
source_id="nc",
|
||||||
|
base_url="https://nc.example.com/dav/",
|
||||||
|
transport=_transport(handler),
|
||||||
|
)
|
||||||
|
events = [event async for event in src.discover()]
|
||||||
|
assert [e.uri for e in events] == [
|
||||||
|
"https://nc.example.com/dav/my docs/Hello World.md"
|
||||||
|
]
|
||||||
Loading…
Reference in a new issue