Narrow HTTP discover() exception catch to TransportError only

The bare `except Exception` in HTTPSource.discover() silently
swallowed all errors from HEAD requests — including configuration
errors (bad auth, invalid headers) and programming errors (TypeError,
AttributeError) — treating them identically to network failures by
emitting UPSERT with no revision.

Narrow the catch to httpx.TransportError (the umbrella for
ConnectError, TimeoutException, etc.) and add a debug log. Other
exceptions now propagate to the poller's circuit breaker where they
surface as failures instead of being silently retried forever.
This commit is contained in:
Chris McDonough 2026-06-01 08:05:42 -04:00 committed by Yiorgis Gozadinos
parent 64f2b7b7d2
commit 61ea22527a
No known key found for this signature in database
2 changed files with 27 additions and 1 deletions

View file

@ -1,4 +1,5 @@
import hashlib import hashlib
import logging
from collections.abc import AsyncIterator from collections.abc import AsyncIterator
from datetime import UTC, datetime from datetime import UTC, datetime
from urllib.parse import urlparse from urllib.parse import urlparse
@ -12,6 +13,8 @@ from haiku.rag.ingester.sources.base import (
SourceEventKind, SourceEventKind,
) )
logger = logging.getLogger(__name__)
def _extract_revision(headers: httpx.Headers) -> tuple[str | None, dict[str, str]]: def _extract_revision(headers: httpx.Headers) -> tuple[str | None, dict[str, str]]:
"""Return (canonical_revision, extras). ETag is the stronger validator """Return (canonical_revision, extras). ETag is the stronger validator
@ -103,7 +106,12 @@ class HTTPSource:
for url in self.urls: for url in self.urls:
try: try:
head = await self._http.head(url) head = await self._http.head(url)
except Exception: except httpx.TransportError as exc:
logger.debug(
"HEAD %s failed (%s); emitting UPSERT with no revision",
url,
exc,
)
yield SourceEvent( yield SourceEvent(
source_id=self.source_id, source_id=self.source_id,
uri=url, uri=url,

View file

@ -324,3 +324,21 @@ async def test_discover_emits_delete_for_removed_url_with_no_revision_tracked():
] ]
by_uri = {e.uri: e for e in events} by_uri = {e.uri: e for e in events}
assert by_uri["https://example.com/no-etag.md"].kind is SourceEventKind.DELETE assert by_uri["https://example.com/no-etag.md"].kind is SourceEventKind.DELETE
@pytest.mark.asyncio
async def test_discover_propagates_non_transport_errors():
"""Programming errors (TypeError, etc.) should propagate instead of
being silently swallowed as UPSERT events."""
def handler(request: httpx.Request) -> httpx.Response:
raise TypeError("unexpected bug")
src = HTTPSource(
source_id="x",
urls=["https://example.com/a.md"],
transport=httpx.MockTransport(handler),
)
with pytest.raises(TypeError, match="unexpected bug"):
async for _ in src.discover():
pass