Handle WebDAV as source
This commit is contained in:
parent
13cbadeb6f
commit
65c309e12b
10 changed files with 756 additions and 5 deletions
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
### Added
|
||||
|
||||
- New `haiku-ingester` service for continuous document ingestion: persistent SQLite job queue, async worker pool with retries and a dead-letter queue, FS/HTTP/S3 source adapters with per-source circuit breakers, and a FastAPI control plane (`/health`, `/jobs`, `/sources`, `/dlq`). Configured under `ingester:` in `haiku.rag.yaml`. Shipped behind the `[ingester]` extra, with Logfire spans (`ingester.poller.sweep` → `ingester.job` → `document.{fetch,convert,chunk,embed,store}`) for traceable ingestion. See [docs/ingester.md](docs/ingester.md).
|
||||
- New `haiku-ingester` service for continuous document ingestion: persistent SQLite job queue, async worker pool with retries and a dead-letter queue, FS/HTTP/S3/WebDAV source adapters with per-source circuit breakers, and a FastAPI control plane (`/health`, `/jobs`, `/sources`, `/dlq`). Configured under `ingester:` in `haiku.rag.yaml`. Shipped behind the `[ingester]` extra, with Logfire spans (`ingester.poller.sweep` → `ingester.job` → `document.{fetch,convert,chunk,embed,store}`) for traceable ingestion. See [docs/ingester.md](docs/ingester.md).
|
||||
|
||||
### Removed
|
||||
|
||||
|
|
|
|||
|
|
@ -109,6 +109,39 @@ response from a configured URL triggers a delete event; other failure
|
|||
statuses fall through to UPSERT-with-no-revision so the worker can
|
||||
GET and decide.
|
||||
|
||||
### WebDAV
|
||||
|
||||
```yaml
|
||||
ingester:
|
||||
sources:
|
||||
- type: webdav
|
||||
id: nextcloud
|
||||
base_url: https://nextcloud.example.com/remote.php/dav/files/alice/Documents/
|
||||
username: alice
|
||||
password: ${NEXTCLOUD_APP_PASSWORD}
|
||||
ignore_patterns: ["**/Trash/**"]
|
||||
poll_interval_s: 600
|
||||
```
|
||||
|
||||
Each sweep issues one `PROPFIND` with `Depth: infinity` against
|
||||
`base_url` and parses the multistatus response. Files (non-collection
|
||||
resources) are emitted as UPSERT / UNCHANGED based on the `getetag`
|
||||
property (falling back to `getlastmodified` if the server omits it);
|
||||
URIs that were in the previous snapshot but no longer appear under the
|
||||
collection are emitted as DELETE.
|
||||
|
||||
Fetches are plain HTTP GETs — any WebDAV server already supports them.
|
||||
|
||||
Bearer-token auth can replace HTTP Basic via the standard `headers` map:
|
||||
|
||||
```yaml
|
||||
- type: webdav
|
||||
id: kdrive
|
||||
base_url: https://kdrive.infomaniak.com/app/drive/123/
|
||||
headers:
|
||||
Authorization: Bearer ${KDRIVE_TOKEN}
|
||||
```
|
||||
|
||||
## Workers and retry
|
||||
|
||||
```yaml
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from haiku.rag.config.models import (
|
|||
S3SourceConfig,
|
||||
SourceConfig,
|
||||
StorageConfig,
|
||||
WebDAVSourceConfig,
|
||||
WorkerConfig,
|
||||
)
|
||||
|
||||
|
|
@ -53,6 +54,7 @@ __all__ = [
|
|||
"S3SourceConfig",
|
||||
"SourceConfig",
|
||||
"StorageConfig",
|
||||
"WebDAVSourceConfig",
|
||||
"WorkerConfig",
|
||||
"find_config_file",
|
||||
"generate_default_config",
|
||||
|
|
|
|||
|
|
@ -323,8 +323,21 @@ class S3SourceConfig(_SourceBase):
|
|||
include_patterns: list[str] = []
|
||||
|
||||
|
||||
class WebDAVSourceConfig(_SourceBase):
|
||||
"""A WebDAV collection (Nextcloud, ownCloud, Apache mod_dav, etc.). Files
|
||||
are discovered via PROPFIND on `base_url`; fetch is plain HTTP GET."""
|
||||
|
||||
type: Literal["webdav"]
|
||||
base_url: str
|
||||
username: str | None = None
|
||||
password: str | None = None
|
||||
headers: dict[str, str] = Field(default_factory=dict)
|
||||
ignore_patterns: list[str] = []
|
||||
include_patterns: list[str] = []
|
||||
|
||||
|
||||
SourceConfig = Annotated[
|
||||
FSSourceConfig | HTTPSourceConfig | S3SourceConfig,
|
||||
FSSourceConfig | HTTPSourceConfig | S3SourceConfig | WebDAVSourceConfig,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,15 @@ from haiku.rag.config import (
|
|||
HTTPSourceConfig,
|
||||
S3SourceConfig,
|
||||
SourceConfig,
|
||||
WebDAVSourceConfig,
|
||||
)
|
||||
from haiku.rag.ingester.sources import (
|
||||
FSSource,
|
||||
HTTPSource,
|
||||
S3Source,
|
||||
Source,
|
||||
WebDAVSource,
|
||||
)
|
||||
from haiku.rag.ingester.sources import FSSource, HTTPSource, S3Source, Source
|
||||
|
||||
|
||||
def build_source(
|
||||
|
|
@ -16,7 +23,7 @@ def build_source(
|
|||
|
||||
Source IDs auto-derive from the target when the config didn't supply one,
|
||||
matching the conventions in the adapters themselves (fs:<root>,
|
||||
s3:<bucket>/<prefix>, http:<id>).
|
||||
s3:<bucket>/<prefix>, http:<id>, webdav:<id>).
|
||||
"""
|
||||
if isinstance(cfg, FSSourceConfig):
|
||||
return FSSource(
|
||||
|
|
@ -39,4 +46,17 @@ def build_source(
|
|||
supported_extensions=supported_extensions,
|
||||
source_id=cfg.id,
|
||||
)
|
||||
if isinstance(cfg, WebDAVSourceConfig):
|
||||
if cfg.id is None:
|
||||
raise ValueError("WebDAVSourceConfig.id is required")
|
||||
return WebDAVSource(
|
||||
source_id=cfg.id,
|
||||
base_url=cfg.base_url,
|
||||
username=cfg.username,
|
||||
password=cfg.password,
|
||||
headers=cfg.headers,
|
||||
ignore_patterns=cfg.ignore_patterns or None,
|
||||
include_patterns=cfg.include_patterns or None,
|
||||
supported_extensions=supported_extensions,
|
||||
)
|
||||
raise TypeError(f"Unsupported source config: {type(cfg).__name__}")
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from haiku.rag.ingester.sources.fs import FSSource
|
|||
from haiku.rag.ingester.sources.http import HTTPSource
|
||||
from haiku.rag.ingester.sources.registry import resolve_fetcher
|
||||
from haiku.rag.ingester.sources.s3 import S3Source
|
||||
from haiku.rag.ingester.sources.webdav import WebDAVSource
|
||||
|
||||
__all__ = [
|
||||
"FetchResult",
|
||||
|
|
@ -21,5 +22,6 @@ __all__ = [
|
|||
"Source",
|
||||
"SourceEvent",
|
||||
"SourceEventKind",
|
||||
"WebDAVSource",
|
||||
"resolve_fetcher",
|
||||
]
|
||||
|
|
|
|||
292
haiku_rag_slim/haiku/rag/ingester/sources/webdav.py
Normal file
292
haiku_rag_slim/haiku/rag/ingester/sources/webdav.py
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
import hashlib
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import unquote, urljoin, urlparse
|
||||
from xml.etree.ElementTree import Element, fromstring
|
||||
|
||||
import httpx
|
||||
|
||||
from haiku.rag.ingester.sources.base import (
|
||||
FetchResult,
|
||||
RevisionSnapshot,
|
||||
SourceEvent,
|
||||
SourceEventKind,
|
||||
)
|
||||
from haiku.rag.ingester.sources.filter import (
|
||||
FileFilter,
|
||||
_default_supported_extensions,
|
||||
)
|
||||
|
||||
# WebDAV PROPFIND uses XML with the DAV: namespace. Element tags arrive as
|
||||
# Clark-notation strings like "{DAV:}response", so we precompile them once.
|
||||
_DAV_NS = "DAV:"
|
||||
_TAG_RESPONSE = f"{{{_DAV_NS}}}response"
|
||||
_TAG_HREF = f"{{{_DAV_NS}}}href"
|
||||
_TAG_PROPSTAT = f"{{{_DAV_NS}}}propstat"
|
||||
_TAG_PROP = f"{{{_DAV_NS}}}prop"
|
||||
_TAG_STATUS = f"{{{_DAV_NS}}}status"
|
||||
_TAG_RESOURCETYPE = f"{{{_DAV_NS}}}resourcetype"
|
||||
_TAG_COLLECTION = f"{{{_DAV_NS}}}collection"
|
||||
_TAG_GETETAG = f"{{{_DAV_NS}}}getetag"
|
||||
_TAG_GETLASTMODIFIED = f"{{{_DAV_NS}}}getlastmodified"
|
||||
_TAG_GETCONTENTTYPE = f"{{{_DAV_NS}}}getcontenttype"
|
||||
|
||||
|
||||
_PROPFIND_BODY = b"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<propfind xmlns="DAV:">
|
||||
<prop>
|
||||
<resourcetype/>
|
||||
<getetag/>
|
||||
<getlastmodified/>
|
||||
<getcontenttype/>
|
||||
</prop>
|
||||
</propfind>
|
||||
"""
|
||||
|
||||
|
||||
class _PropfindEntry:
|
||||
"""One <response> element decoded into the fields the source actually
|
||||
uses. `is_collection` separates folders from files; revision is ETag
|
||||
when present, otherwise the Last-Modified header value."""
|
||||
|
||||
__slots__ = ("href", "is_collection", "revision", "content_type")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
href: str,
|
||||
*,
|
||||
is_collection: bool,
|
||||
revision: str | None,
|
||||
content_type: str | None,
|
||||
) -> None:
|
||||
self.href = href
|
||||
self.is_collection = is_collection
|
||||
self.revision = revision
|
||||
self.content_type = content_type
|
||||
|
||||
|
||||
def _strip_etag(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
cleaned = value.strip().strip('"').strip("W/").strip().strip('"')
|
||||
return cleaned or None
|
||||
|
||||
|
||||
def _entry_from_response(response: Element) -> _PropfindEntry | None:
|
||||
"""Decode a single <response>. Returns None if the prop block is missing
|
||||
or the entry didn't return HTTP 200 (e.g. 404 for a known-bad path)."""
|
||||
href_el = response.find(_TAG_HREF)
|
||||
if href_el is None or not href_el.text:
|
||||
return None
|
||||
href = href_el.text
|
||||
|
||||
is_collection = False
|
||||
revision: str | None = None
|
||||
last_modified: str | None = None
|
||||
content_type: str | None = None
|
||||
ok = False
|
||||
|
||||
for propstat in response.findall(_TAG_PROPSTAT):
|
||||
status_el = propstat.find(_TAG_STATUS)
|
||||
if status_el is None or not status_el.text:
|
||||
continue
|
||||
if " 200 " not in status_el.text:
|
||||
continue
|
||||
ok = True
|
||||
prop = propstat.find(_TAG_PROP)
|
||||
if prop is None:
|
||||
continue
|
||||
resourcetype = prop.find(_TAG_RESOURCETYPE)
|
||||
if resourcetype is not None and resourcetype.find(_TAG_COLLECTION) is not None:
|
||||
is_collection = True
|
||||
etag_el = prop.find(_TAG_GETETAG)
|
||||
if etag_el is not None and etag_el.text:
|
||||
revision = _strip_etag(etag_el.text)
|
||||
lm_el = prop.find(_TAG_GETLASTMODIFIED)
|
||||
if lm_el is not None and lm_el.text:
|
||||
last_modified = lm_el.text.strip() or None
|
||||
ct_el = prop.find(_TAG_GETCONTENTTYPE)
|
||||
if ct_el is not None and ct_el.text:
|
||||
content_type = ct_el.text.split(";")[0].strip().lower() or None
|
||||
|
||||
if not ok:
|
||||
return None
|
||||
# Prefer ETag — stronger validator. Fall back to Last-Modified so revision
|
||||
# detection still works against servers that don't return ETags on PROPFIND.
|
||||
return _PropfindEntry(
|
||||
href=href,
|
||||
is_collection=is_collection,
|
||||
revision=revision or last_modified,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
|
||||
def _parse_multistatus(body: bytes) -> list[_PropfindEntry]:
|
||||
"""Top-level multistatus parser. Raises ValueError on garbage XML so the
|
||||
poller's circuit breaker can record a failure."""
|
||||
try:
|
||||
root = fromstring(body)
|
||||
except Exception as exc: # ParseError + any defensive surprise
|
||||
raise ValueError(f"Invalid PROPFIND response XML: {exc}") from exc
|
||||
return [
|
||||
entry
|
||||
for response in root.findall(_TAG_RESPONSE)
|
||||
if (entry := _entry_from_response(response)) is not None
|
||||
]
|
||||
|
||||
|
||||
def _resolve_href(href: str, base_url: str) -> str:
|
||||
"""PROPFIND href values can be either absolute URLs or server-relative
|
||||
paths. Resolve to absolute against base_url either way, then URL-decode
|
||||
the path so the URI we store matches what a user would type."""
|
||||
absolute = urljoin(base_url, href)
|
||||
parsed = urlparse(absolute)
|
||||
decoded_path = unquote(parsed.path)
|
||||
rebuilt = parsed._replace(path=decoded_path)
|
||||
return rebuilt.geturl()
|
||||
|
||||
|
||||
class WebDAVSource:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
source_id: str,
|
||||
base_url: str,
|
||||
username: str | None = None,
|
||||
password: str | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
ignore_patterns: list[str] | None = None,
|
||||
include_patterns: list[str] | None = None,
|
||||
supported_extensions: list[str] | None = None,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
) -> None:
|
||||
self.source_id = source_id
|
||||
# Trailing slash matters: urljoin treats path-without-slash as a sibling
|
||||
# link, so "https://srv/dav" joined with "subdir/x" gives ".../x".
|
||||
self.base_url = base_url if base_url.endswith("/") else base_url + "/"
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.headers = dict(headers or {})
|
||||
self.supported_extensions = (
|
||||
list(supported_extensions)
|
||||
if supported_extensions is not None
|
||||
else _default_supported_extensions()
|
||||
)
|
||||
self.filter = FileFilter(
|
||||
ignore_patterns=ignore_patterns,
|
||||
include_patterns=include_patterns,
|
||||
supported_extensions=self.supported_extensions,
|
||||
)
|
||||
# transport is for testing — production callers leave it None.
|
||||
self._transport = transport
|
||||
|
||||
def supports(self, uri: str) -> bool:
|
||||
return uri.startswith(self.base_url)
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
auth = (
|
||||
(self.username, self.password)
|
||||
if self.username is not None and self.password is not None
|
||||
else None
|
||||
)
|
||||
return httpx.AsyncClient(
|
||||
auth=auth, headers=self.headers, transport=self._transport
|
||||
)
|
||||
|
||||
async def head(self, uri: str) -> str | None:
|
||||
async with self._client() as http:
|
||||
response = await http.request(
|
||||
"PROPFIND",
|
||||
uri,
|
||||
headers={"Depth": "0", "Content-Type": "application/xml"},
|
||||
content=_PROPFIND_BODY,
|
||||
)
|
||||
if response.is_error:
|
||||
return None
|
||||
entries = _parse_multistatus(response.content)
|
||||
if not entries:
|
||||
return None
|
||||
return entries[0].revision
|
||||
|
||||
async def fetch(self, uri: str) -> FetchResult:
|
||||
async with self._client() as http:
|
||||
response = await http.get(uri)
|
||||
response.raise_for_status()
|
||||
body = response.content
|
||||
content_type = (
|
||||
response.headers.get("content-type", "application/octet-stream")
|
||||
.split(";")[0]
|
||||
.strip()
|
||||
.lower()
|
||||
)
|
||||
# ETag from the GET response is the freshest revision; fall back to
|
||||
# Last-Modified, matching HTTPSource's preference order.
|
||||
revision = (
|
||||
_strip_etag(response.headers.get("etag"))
|
||||
or (response.headers.get("last-modified") or "").strip()
|
||||
or None
|
||||
)
|
||||
extra: dict[str, str] = {}
|
||||
last_modified = (response.headers.get("last-modified") or "").strip()
|
||||
if last_modified:
|
||||
extra["last_modified"] = last_modified
|
||||
return FetchResult(
|
||||
uri=uri,
|
||||
body=body,
|
||||
content_type=content_type,
|
||||
content_hash=hashlib.md5(body, usedforsecurity=False).hexdigest(),
|
||||
revision=revision,
|
||||
extra_metadata=extra,
|
||||
)
|
||||
|
||||
async def discover(
|
||||
self, since: RevisionSnapshot | None = None
|
||||
) -> AsyncIterator[SourceEvent]:
|
||||
snapshot: dict[str, str] = dict(since) if since else {}
|
||||
now = datetime.now(UTC)
|
||||
seen: set[str] = set()
|
||||
|
||||
async with self._client() as http:
|
||||
response = await http.request(
|
||||
"PROPFIND",
|
||||
self.base_url,
|
||||
headers={"Depth": "infinity", "Content-Type": "application/xml"},
|
||||
content=_PROPFIND_BODY,
|
||||
)
|
||||
response.raise_for_status()
|
||||
entries = _parse_multistatus(response.content)
|
||||
|
||||
for entry in entries:
|
||||
if entry.is_collection:
|
||||
continue
|
||||
uri = _resolve_href(entry.href, self.base_url)
|
||||
# The base URL itself sometimes appears as a non-collection on
|
||||
# broken servers; skip anything that's not strictly under it.
|
||||
if uri == self.base_url.rstrip("/") or not uri.startswith(self.base_url):
|
||||
continue
|
||||
if not self.filter.include_file(uri):
|
||||
continue
|
||||
seen.add(uri)
|
||||
revision = entry.revision
|
||||
if revision is not None and snapshot.get(uri) == revision:
|
||||
kind = SourceEventKind.UNCHANGED
|
||||
else:
|
||||
kind = SourceEventKind.UPSERT
|
||||
yield SourceEvent(
|
||||
source_id=self.source_id,
|
||||
uri=uri,
|
||||
kind=kind,
|
||||
revision=revision,
|
||||
discovered_at=now,
|
||||
)
|
||||
|
||||
for uri in snapshot:
|
||||
if uri in seen:
|
||||
continue
|
||||
yield SourceEvent(
|
||||
source_id=self.source_id,
|
||||
uri=uri,
|
||||
kind=SourceEventKind.DELETE,
|
||||
revision=None,
|
||||
discovered_at=now,
|
||||
)
|
||||
|
|
@ -48,6 +48,27 @@ def test_discriminator_picks_s3_source():
|
|||
assert cfg.sources[0].uri == "s3://bucket/prefix/"
|
||||
|
||||
|
||||
def test_discriminator_picks_webdav_source():
|
||||
cfg = IngesterConfig.model_validate(
|
||||
{
|
||||
"sources": [
|
||||
{
|
||||
"type": "webdav",
|
||||
"id": "nc",
|
||||
"base_url": "https://nc.example.com/dav/",
|
||||
"username": "alice",
|
||||
"password": "hunter2",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
from haiku.rag.config import WebDAVSourceConfig
|
||||
|
||||
assert isinstance(cfg.sources[0], WebDAVSourceConfig)
|
||||
assert cfg.sources[0].base_url == "https://nc.example.com/dav/"
|
||||
assert cfg.sources[0].username == "alice"
|
||||
|
||||
|
||||
def test_discriminator_rejects_unknown_type():
|
||||
with pytest.raises(ValidationError):
|
||||
IngesterConfig.model_validate({"sources": [{"type": "ftp", "uri": "x"}]})
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from haiku.rag.config import (
|
|||
FSSourceConfig,
|
||||
HTTPSourceConfig,
|
||||
S3SourceConfig,
|
||||
WebDAVSourceConfig,
|
||||
)
|
||||
from haiku.rag.ingester.pollers.circuit_breaker import CircuitBreaker
|
||||
from haiku.rag.ingester.pollers.manager import PollerManager
|
||||
|
|
@ -278,6 +279,9 @@ async def test_manager_builds_pollers_per_source(tmp_path, jobs, sync):
|
|||
FSSourceConfig(type="fs", root=tmp_path),
|
||||
S3SourceConfig(type="s3", uri="s3://bucket/"),
|
||||
HTTPSourceConfig(type="http", id="urls", urls=[]),
|
||||
WebDAVSourceConfig(
|
||||
type="webdav", id="nc", base_url="https://nc.example.com/dav/"
|
||||
),
|
||||
]
|
||||
manager = PollerManager(
|
||||
configs=configs,
|
||||
|
|
@ -285,11 +289,12 @@ async def test_manager_builds_pollers_per_source(tmp_path, jobs, sync):
|
|||
sync_repo=sync,
|
||||
)
|
||||
built = manager.build_pollers()
|
||||
assert len(built) == 3
|
||||
assert len(built) == 4
|
||||
assert {p.source_id for p in built} == {
|
||||
f"fs:{tmp_path.resolve()}",
|
||||
"s3:bucket/",
|
||||
"urls",
|
||||
"nc",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
363
tests/ingester/test_webdav_source.py
Normal file
363
tests/ingester/test_webdav_source.py
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
import hashlib
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from haiku.rag.ingester.sources.base import SourceEventKind
|
||||
from haiku.rag.ingester.sources.webdav import WebDAVSource
|
||||
|
||||
|
||||
def _transport(handler) -> httpx.MockTransport:
|
||||
return httpx.MockTransport(handler)
|
||||
|
||||
|
||||
def _multistatus(*entries: dict) -> bytes:
|
||||
"""Build a <multistatus> response. Each entry is a dict like
|
||||
{'href': '/dav/x.md', 'collection': False, 'etag': '"abc"',
|
||||
'last_modified': 'Wed, ...', 'content_type': 'text/markdown'}."""
|
||||
body = ['<?xml version="1.0" encoding="utf-8"?>', '<d:multistatus xmlns:d="DAV:">']
|
||||
for e in entries:
|
||||
body.append(" <d:response>")
|
||||
body.append(f" <d:href>{e['href']}</d:href>")
|
||||
body.append(" <d:propstat>")
|
||||
body.append(" <d:status>HTTP/1.1 200 OK</d:status>")
|
||||
body.append(" <d:prop>")
|
||||
body.append(" <d:resourcetype>")
|
||||
if e.get("collection"):
|
||||
body.append(" <d:collection/>")
|
||||
body.append(" </d:resourcetype>")
|
||||
if "etag" in e:
|
||||
body.append(f" <d:getetag>{e['etag']}</d:getetag>")
|
||||
if "last_modified" in e:
|
||||
body.append(
|
||||
f" <d:getlastmodified>{e['last_modified']}</d:getlastmodified>"
|
||||
)
|
||||
if "content_type" in e:
|
||||
body.append(
|
||||
f" <d:getcontenttype>{e['content_type']}</d:getcontenttype>"
|
||||
)
|
||||
body.append(" </d:prop>")
|
||||
body.append(" </d:propstat>")
|
||||
body.append(" </d:response>")
|
||||
body.append("</d:multistatus>")
|
||||
return "\n".join(body).encode()
|
||||
|
||||
|
||||
def test_supports_uri_under_base_url():
|
||||
src = WebDAVSource(source_id="nc", base_url="https://nc.example.com/dav/")
|
||||
assert src.supports("https://nc.example.com/dav/file.md")
|
||||
assert src.supports("https://nc.example.com/dav/sub/file.md")
|
||||
assert not src.supports("https://nc.example.com/other/file.md")
|
||||
assert not src.supports("https://other.example.com/dav/file.md")
|
||||
|
||||
|
||||
def test_base_url_trailing_slash_normalised():
|
||||
"""base_url without trailing slash mustn't break urljoin during discovery."""
|
||||
src = WebDAVSource(source_id="nc", base_url="https://nc.example.com/dav")
|
||||
assert src.base_url == "https://nc.example.com/dav/"
|
||||
assert src.supports("https://nc.example.com/dav/x.md")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_returns_bytes_md5_revision_and_content_type():
|
||||
body = b"hello dav"
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.method == "GET"
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=body,
|
||||
headers={
|
||||
"content-type": "text/markdown; charset=utf-8",
|
||||
"etag": '"rev-1"',
|
||||
"last-modified": "Wed, 21 Oct 2025 07:28:00 GMT",
|
||||
},
|
||||
)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
result = await src.fetch("https://nc.example.com/dav/a.md")
|
||||
assert result.body == body
|
||||
assert result.content_hash == hashlib.md5(body, usedforsecurity=False).hexdigest()
|
||||
assert result.content_type == "text/markdown"
|
||||
assert result.revision == "rev-1"
|
||||
assert result.extra_metadata == {"last_modified": "Wed, 21 Oct 2025 07:28:00 GMT"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_falls_back_to_last_modified_when_no_etag():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=b"x",
|
||||
headers={
|
||||
"content-type": "text/plain",
|
||||
"last-modified": "Wed, 21 Oct 2025 07:28:00 GMT",
|
||||
},
|
||||
)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
result = await src.fetch("https://nc.example.com/dav/a.txt")
|
||||
assert result.revision == "Wed, 21 Oct 2025 07:28:00 GMT"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_head_returns_etag_from_propfind_depth_zero():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.method == "PROPFIND"
|
||||
assert request.headers["Depth"] == "0"
|
||||
return httpx.Response(
|
||||
207,
|
||||
content=_multistatus(
|
||||
{"href": "/dav/a.md", "etag": '"rev-9"'},
|
||||
),
|
||||
)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
assert await src.head("https://nc.example.com/dav/a.md") == "rev-9"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_head_returns_none_on_404():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(404)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
assert await src.head("https://nc.example.com/dav/missing.md") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_yields_upserts_and_skips_collections_and_unsupported():
|
||||
multistatus = _multistatus(
|
||||
{"href": "/dav/", "collection": True},
|
||||
{"href": "/dav/sub/", "collection": True},
|
||||
{"href": "/dav/a.md", "etag": '"rev-a"', "content_type": "text/markdown"},
|
||||
{"href": "/dav/sub/b.txt", "etag": '"rev-b"', "content_type": "text/plain"},
|
||||
{"href": "/dav/skip.log", "etag": '"rev-log"', "content_type": "text/plain"},
|
||||
)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.method == "PROPFIND"
|
||||
assert request.headers["Depth"] == "infinity"
|
||||
return httpx.Response(207, content=multistatus)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
|
||||
events = [event async for event in src.discover()]
|
||||
by_uri = {e.uri: e for e in events}
|
||||
assert set(by_uri) == {
|
||||
"https://nc.example.com/dav/a.md",
|
||||
"https://nc.example.com/dav/sub/b.txt",
|
||||
}
|
||||
assert all(e.kind is SourceEventKind.UPSERT for e in events)
|
||||
assert by_uri["https://nc.example.com/dav/a.md"].revision == "rev-a"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_emits_unchanged_when_snapshot_matches():
|
||||
multistatus = _multistatus(
|
||||
{"href": "/dav/", "collection": True},
|
||||
{"href": "/dav/a.md", "etag": '"rev-a"', "content_type": "text/markdown"},
|
||||
)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(207, content=multistatus)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
snapshot = {"https://nc.example.com/dav/a.md": "rev-a"}
|
||||
events = [event async for event in src.discover(since=snapshot)]
|
||||
assert [e.kind for e in events] == [SourceEventKind.UNCHANGED]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_emits_delete_for_files_no_longer_listed():
|
||||
multistatus = _multistatus(
|
||||
{"href": "/dav/", "collection": True},
|
||||
{"href": "/dav/a.md", "etag": '"rev-a"', "content_type": "text/markdown"},
|
||||
)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(207, content=multistatus)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
snapshot = {
|
||||
"https://nc.example.com/dav/a.md": "rev-a",
|
||||
"https://nc.example.com/dav/gone.md": "rev-old",
|
||||
}
|
||||
events = [event async for event in src.discover(since=snapshot)]
|
||||
kinds = {e.uri: e.kind for e in events}
|
||||
assert kinds == {
|
||||
"https://nc.example.com/dav/a.md": SourceEventKind.UNCHANGED,
|
||||
"https://nc.example.com/dav/gone.md": SourceEventKind.DELETE,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_uses_last_modified_when_etag_absent():
|
||||
multistatus = _multistatus(
|
||||
{
|
||||
"href": "/dav/a.md",
|
||||
"last_modified": "Wed, 21 Oct 2025 07:28:00 GMT",
|
||||
"content_type": "text/markdown",
|
||||
},
|
||||
)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(207, content=multistatus)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
events = [event async for event in src.discover()]
|
||||
assert events[0].revision == "Wed, 21 Oct 2025 07:28:00 GMT"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_raises_on_malformed_xml():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(207, content=b"<not></valid xml")
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
with pytest.raises(ValueError, match="Invalid PROPFIND"):
|
||||
[event async for event in src.discover()]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_propagates_http_error():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(401)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
[event async for event in src.discover()]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_auth_sent_when_credentials_configured():
|
||||
seen_auth: list[str | None] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen_auth.append(request.headers.get("authorization"))
|
||||
return httpx.Response(
|
||||
207,
|
||||
content=_multistatus({"href": "/dav/", "collection": True}),
|
||||
)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
username="alice",
|
||||
password="hunter2",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
[event async for event in src.discover()]
|
||||
assert seen_auth and seen_auth[0] is not None
|
||||
assert seen_auth[0].startswith("Basic ")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_headers_forwarded():
|
||||
seen: list[str | None] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request.headers.get("authorization"))
|
||||
return httpx.Response(
|
||||
207,
|
||||
content=_multistatus({"href": "/dav/", "collection": True}),
|
||||
)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
headers={"Authorization": "Bearer tok-123"},
|
||||
transport=_transport(handler),
|
||||
)
|
||||
[event async for event in src.discover()]
|
||||
assert seen == ["Bearer tok-123"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_resolves_absolute_href():
|
||||
"""Some servers return absolute URLs in href, others return server paths.
|
||||
Both must produce the same stored URI."""
|
||||
multistatus = _multistatus(
|
||||
{
|
||||
"href": "https://nc.example.com/dav/a.md",
|
||||
"etag": '"rev"',
|
||||
"content_type": "text/markdown",
|
||||
}
|
||||
)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(207, content=multistatus)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
events = [event async for event in src.discover()]
|
||||
assert [e.uri for e in events] == ["https://nc.example.com/dav/a.md"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_url_decodes_href_path():
|
||||
"""PROPFIND hrefs are percent-encoded per RFC 3986. We unquote them so
|
||||
the stored URI matches what a user types in `add-src`."""
|
||||
multistatus = _multistatus(
|
||||
{
|
||||
"href": "/dav/my%20docs/Hello%20World.md",
|
||||
"etag": '"rev"',
|
||||
"content_type": "text/markdown",
|
||||
}
|
||||
)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(207, content=multistatus)
|
||||
|
||||
src = WebDAVSource(
|
||||
source_id="nc",
|
||||
base_url="https://nc.example.com/dav/",
|
||||
transport=_transport(handler),
|
||||
)
|
||||
events = [event async for event in src.discover()]
|
||||
assert [e.uri for e in events] == [
|
||||
"https://nc.example.com/dav/my docs/Hello World.md"
|
||||
]
|
||||
Loading…
Reference in a new issue