Emit DELETE for HTTP URLs removed from config

haiku_rag_slim/haiku/rag/ingester/sources/http.py

@@ -86,9 +86,9 @@ class HTTPSource:
         self, since: RevisionSnapshot | None = None
     ) -> AsyncIterator[SourceEvent]:
         # HTTP has no listing concept — discover() only reports on what is
-        # currently configured in self.urls. Config drift (URIs that were
+        # currently configured in self.urls. URLs that were previously in
-        # configured before but aren't now) is not visible here; the poller
+        # config but aren't now emit DELETE so the poller can clean up
-        # layer detects that by diffing self.urls across sweeps.
+        # alongside the in-source 410 signal.
         #
         # 410 Gone is the one real source-side deletion signal: the origin
         # explicitly says "permanently gone". 404 and other failures are
@@ -97,6 +97,7 @@ class HTTPSource:
         # via GET.
         snapshot: dict[str, str] = dict(since) if since else {}
         now = datetime.now(UTC)
+        configured = set(self.urls)
 
         async with self._client() as http:
             for url in self.urls:
@@ -145,3 +146,18 @@ class HTTPSource:
                     revision=revision,
                     discovered_at=now,
                 )
+
+        # Anything previously ingested by this source that's no longer in
+        # config (URL removed from `urls`) emits DELETE so delete_orphans
+        # can clean up. Without this, removing a URL from config leaves the
+        # document and sync_state indefinitely.
+        for url in snapshot:
+            if url in configured:
+                continue
+            yield SourceEvent(
+                source_id=self.source_id,
+                uri=url,
+                kind=SourceEventKind.DELETE,
+                revision=None,
+                discovered_at=now,
+            )

tests/ingester/test_http_source.py

@@ -274,9 +274,11 @@ async def test_discover_treats_network_errors_as_upsert():
 
 
 @pytest.mark.asyncio
-async def test_discover_does_not_emit_delete_for_unconfigured_uri():
+async def test_discover_emits_delete_for_removed_url():
-    """Config drift is the poller's job, not HTTPSource's. A URI that was in
+    """A URI that was previously in config (and therefore in the snapshot)
-    the snapshot but is no longer configured must not appear as a DELETE."""
+    but is no longer configured emits DELETE so the poller can clean up
+    when delete_orphans=True. Mirrors what FS/S3/WebDAV already do for
+    items missing from a listing."""
     transport = _transport(
         {
             ("HEAD", "https://example.com/a.md"): httpx.Response(
@@ -290,5 +292,7 @@ async def test_discover_does_not_emit_delete_for_unconfigured_uri():
     events = [
         e async for e in src.discover(since={"https://example.com/gone.md": "old"})
     ]
-    assert {e.uri for e in events} == {"https://example.com/a.md"}
+    by_uri = {e.uri: e for e in events}
-    assert all(e.kind is not SourceEventKind.DELETE for e in events)
+    assert by_uri["https://example.com/a.md"].kind is not SourceEventKind.DELETE
+    assert by_uri["https://example.com/gone.md"].kind is SourceEventKind.DELETE
+    assert by_uri["https://example.com/gone.md"].revision is None