Narrow HTTP discover() exception catch to TransportError only
The bare `except Exception` in HTTPSource.discover() silently swallowed all errors from HEAD requests — including configuration errors (bad auth, invalid headers) and programming errors (TypeError, AttributeError) — treating them identically to network failures by emitting UPSERT with no revision. Narrow the catch to httpx.TransportError (the umbrella for ConnectError, TimeoutException, etc.) and add a debug log. Other exceptions now propagate to the poller's circuit breaker where they surface as failures instead of being silently retried forever.
This commit is contained in:
parent
64f2b7b7d2
commit
61ea22527a
2 changed files with 27 additions and 1 deletions
|
|
@ -1,4 +1,5 @@
|
|||
import hashlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import urlparse
|
||||
|
|
@ -12,6 +13,8 @@ from haiku.rag.ingester.sources.base import (
|
|||
SourceEventKind,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _extract_revision(headers: httpx.Headers) -> tuple[str | None, dict[str, str]]:
|
||||
"""Return (canonical_revision, extras). ETag is the stronger validator
|
||||
|
|
@ -103,7 +106,12 @@ class HTTPSource:
|
|||
for url in self.urls:
|
||||
try:
|
||||
head = await self._http.head(url)
|
||||
except Exception:
|
||||
except httpx.TransportError as exc:
|
||||
logger.debug(
|
||||
"HEAD %s failed (%s); emitting UPSERT with no revision",
|
||||
url,
|
||||
exc,
|
||||
)
|
||||
yield SourceEvent(
|
||||
source_id=self.source_id,
|
||||
uri=url,
|
||||
|
|
|
|||
|
|
@ -324,3 +324,21 @@ async def test_discover_emits_delete_for_removed_url_with_no_revision_tracked():
|
|||
]
|
||||
by_uri = {e.uri: e for e in events}
|
||||
assert by_uri["https://example.com/no-etag.md"].kind is SourceEventKind.DELETE
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_propagates_non_transport_errors():
|
||||
"""Programming errors (TypeError, etc.) should propagate instead of
|
||||
being silently swallowed as UPSERT events."""
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise TypeError("unexpected bug")
|
||||
|
||||
src = HTTPSource(
|
||||
source_id="x",
|
||||
urls=["https://example.com/a.md"],
|
||||
transport=httpx.MockTransport(handler),
|
||||
)
|
||||
with pytest.raises(TypeError, match="unexpected bug"):
|
||||
async for _ in src.discover():
|
||||
pass
|
||||
|
|
|
|||
Loading…
Reference in a new issue