diff --git a/haiku_rag_slim/haiku/rag/ingester/sources/http.py b/haiku_rag_slim/haiku/rag/ingester/sources/http.py index a0d9041b..ba77fa5d 100644 --- a/haiku_rag_slim/haiku/rag/ingester/sources/http.py +++ b/haiku_rag_slim/haiku/rag/ingester/sources/http.py @@ -86,9 +86,9 @@ class HTTPSource: self, since: RevisionSnapshot | None = None ) -> AsyncIterator[SourceEvent]: # HTTP has no listing concept — discover() only reports on what is - # currently configured in self.urls. Config drift (URIs that were - # configured before but aren't now) is not visible here; the poller - # layer detects that by diffing self.urls across sweeps. + # currently configured in self.urls. URLs that were previously in + # config but aren't now emit DELETE so the poller can clean up + # alongside the in-source 410 signal. # # 410 Gone is the one real source-side deletion signal: the origin # explicitly says "permanently gone". 404 and other failures are @@ -97,6 +97,7 @@ class HTTPSource: # via GET. snapshot: dict[str, str] = dict(since) if since else {} now = datetime.now(UTC) + configured = set(self.urls) async with self._client() as http: for url in self.urls: @@ -145,3 +146,18 @@ class HTTPSource: revision=revision, discovered_at=now, ) + + # Anything previously ingested by this source that's no longer in + # config (URL removed from `urls`) emits DELETE so delete_orphans + # can clean up. Without this, removing a URL from config leaves the + # document and sync_state indefinitely. + for url in snapshot: + if url in configured: + continue + yield SourceEvent( + source_id=self.source_id, + uri=url, + kind=SourceEventKind.DELETE, + revision=None, + discovered_at=now, + ) diff --git a/tests/ingester/test_http_source.py b/tests/ingester/test_http_source.py index 5c5e3a49..eb08a1f1 100644 --- a/tests/ingester/test_http_source.py +++ b/tests/ingester/test_http_source.py @@ -274,9 +274,11 @@ async def test_discover_treats_network_errors_as_upsert(): @pytest.mark.asyncio -async def test_discover_does_not_emit_delete_for_unconfigured_uri(): - """Config drift is the poller's job, not HTTPSource's. A URI that was in - the snapshot but is no longer configured must not appear as a DELETE.""" +async def test_discover_emits_delete_for_removed_url(): + """A URI that was previously in config (and therefore in the snapshot) + but is no longer configured emits DELETE so the poller can clean up + when delete_orphans=True. Mirrors what FS/S3/WebDAV already do for + items missing from a listing.""" transport = _transport( { ("HEAD", "https://example.com/a.md"): httpx.Response( @@ -290,5 +292,7 @@ async def test_discover_does_not_emit_delete_for_unconfigured_uri(): events = [ e async for e in src.discover(since={"https://example.com/gone.md": "old"}) ] - assert {e.uri for e in events} == {"https://example.com/a.md"} - assert all(e.kind is not SourceEventKind.DELETE for e in events) + by_uri = {e.uri: e for e in events} + assert by_uri["https://example.com/a.md"].kind is not SourceEventKind.DELETE + assert by_uri["https://example.com/gone.md"].kind is SourceEventKind.DELETE + assert by_uri["https://example.com/gone.md"].revision is None