Narrow HTTP discover() exception catch to TransportError only

The bare `except Exception` in HTTPSource.discover() silently
swallowed all errors from HEAD requests — including configuration
errors (bad auth, invalid headers) and programming errors (TypeError,
AttributeError) — treating them identically to network failures by
emitting UPSERT with no revision.

Narrow the catch to httpx.TransportError (the umbrella for
ConnectError, TimeoutException, etc.) and add a debug log. Other
exceptions now propagate to the poller's circuit breaker where they
surface as failures instead of being silently retried forever.
This commit is contained in:
Chris McDonough 2026-06-01 08:05:42 -04:00 committed by Yiorgis Gozadinos
parent 64f2b7b7d2
commit 61ea22527a
No known key found for this signature in database
2 changed files with 27 additions and 1 deletions

View file

@ -1,4 +1,5 @@
import hashlib
import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from urllib.parse import urlparse
@ -12,6 +13,8 @@ from haiku.rag.ingester.sources.base import (
SourceEventKind,
)
logger = logging.getLogger(__name__)
def _extract_revision(headers: httpx.Headers) -> tuple[str | None, dict[str, str]]:
"""Return (canonical_revision, extras). ETag is the stronger validator
@ -103,7 +106,12 @@ class HTTPSource:
for url in self.urls:
try:
head = await self._http.head(url)
except Exception:
except httpx.TransportError as exc:
logger.debug(
"HEAD %s failed (%s); emitting UPSERT with no revision",
url,
exc,
)
yield SourceEvent(
source_id=self.source_id,
uri=url,

View file

@ -324,3 +324,21 @@ async def test_discover_emits_delete_for_removed_url_with_no_revision_tracked():
]
by_uri = {e.uri: e for e in events}
assert by_uri["https://example.com/no-etag.md"].kind is SourceEventKind.DELETE
@pytest.mark.asyncio
async def test_discover_propagates_non_transport_errors():
"""Programming errors (TypeError, etc.) should propagate instead of
being silently swallowed as UPSERT events."""
def handler(request: httpx.Request) -> httpx.Response:
raise TypeError("unexpected bug")
src = HTTPSource(
source_id="x",
urls=["https://example.com/a.md"],
transport=httpx.MockTransport(handler),
)
with pytest.raises(TypeError, match="unexpected bug"):
async for _ in src.discover():
pass