Introduce delete_orphans in monitor (default=false) to delete db entries that do not resolve in the filesystem

CHANGELOG.md

@@ -1,6 +1,10 @@
 # Changelog
 ## [Unreleased]
 
+### Added
+
+- **File Monitor**: Orphan deletion feature - automatically removes documents from database when source files are deleted (enabled via `monitor.delete_orphans` config option, default: false)
+
 ### Changed
 
 - **Configuration**: All CLI commands now properly support `--config` parameter for specifying custom configuration files
@@ -9,6 +13,10 @@
 - Updated CLI documentation to clarify global vs per-command options
 - **BREAKING**: Standardized configuration filename to `haiku.rag.yaml` in user directories (was incorrectly using `config.yaml`). Users with existing `config.yaml` in their user directory will need to rename it to `haiku.rag.yaml`
 
+### Fixed
+
+- **File Monitor**: Fixed incorrect "Updated document" logging for unchanged files - monitor now properly skips files when MD5 hash hasn't changed
+
 ### Removed
 
 - **BREAKING**: A2A (Agent-to-Agent) protocol support has been moved to a separate self-contained package in `examples/a2a-server/`. The A2A server is no longer part of the main haiku.rag package. Users who need A2A functionality can install and run it from the examples directory with `cd examples/a2a-server && uv sync`.

haiku_rag_slim/haiku/rag/config/models.py

@@ -15,6 +15,7 @@ class MonitorConfig(BaseModel):
     directories: list[Path] = []
     ignore_patterns: list[str] = []
     include_patterns: list[str] = []
+    delete_orphans: bool = False
 
 
 class LanceDBConfig(BaseModel):

haiku_rag_slim/haiku/rag/monitor.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -75,6 +76,7 @@ class FileWatcher:
         self.client = client
         self.ignore_patterns = config.monitor.ignore_patterns or None
         self.include_patterns = config.monitor.include_patterns or None
+        self.delete_orphans = config.monitor.delete_orphans
 
     async def observe(self):
         logger.info(f"Watching files in {self.paths}")
@@ -97,6 +99,11 @@ class FileWatcher:
         # Lazy import to avoid loading docling
         from haiku.rag.reader import FileReader
 
+        # Delete orphaned documents in background if enabled
+        if self.delete_orphans:
+            logger.info("Starting orphan cleanup in background")
+            asyncio.create_task(self._delete_orphans())
+
         # Create filter to apply same logic as observe()
         filter = FileFilter(
             ignore_patterns=self.ignore_patterns, include_patterns=self.include_patterns
@@ -133,6 +140,48 @@ class FileWatcher:
             logger.error(f"Failed to upsert document from {file}: {e}")
             return None
 
+    async def _delete_orphans(self):
+        """Delete documents whose source files no longer exist."""
+        try:
+            from urllib.parse import unquote, urlparse
+
+            # Create filter to apply same include/exclude logic
+            filter = FileFilter(
+                ignore_patterns=self.ignore_patterns,
+                include_patterns=self.include_patterns,
+            )
+
+            all_docs = await self.client.list_documents()
+
+            for doc in all_docs:
+                if not doc.uri or not doc.id:
+                    continue
+
+                # Only check file:// URIs
+                parsed = urlparse(doc.uri)
+                if parsed.scheme != "file":
+                    continue
+
+                # Convert URI to Path, decoding URL-encoded characters (like %20 for spaces)
+                file_path = Path(unquote(parsed.path))
+
+                # Check if file exists
+                if not file_path.exists():
+                    # Check if file is within monitored directories
+                    is_monitored = any(
+                        file_path.is_relative_to(monitored_path)
+                        for monitored_path in self.paths
+                    )
+
+                    # Check if file would have been included by filters
+                    if is_monitored and filter.include_file(str(file_path)):
+                        await self.client.delete_document(doc.id)
+                        logger.info(
+                            f"Deleted orphaned document {doc.id} for {file_path}"
+                        )
+        except Exception as e:
+            logger.error(f"Failed to delete orphaned documents: {e}")
+
     async def _delete_document(self, file: Path):
         try:
             uri = file.as_uri()

tests/test_monitor.py

@@ -246,8 +246,8 @@ async def test_file_watcher_with_include_patterns():
 
 
 @pytest.mark.asyncio
-async def test_file_watcher_skips_unchanged_document(caplog):
+async def test_file_watcher_skips_unchanged_document():
-    """Test FileWatcher skips document when content hasn't changed."""
+    """Test FileWatcher returns existing document when content hasn't changed."""
     from datetime import datetime
 
     with tempfile.TemporaryDirectory() as temp_dir:
@@ -273,13 +273,170 @@ async def test_file_watcher_skips_unchanged_document(caplog):
         test_config = AppConfig(monitor=MonitorConfig(directories=[temp_path]))
         watcher = FileWatcher(client=mock_client, config=test_config)
 
-        with caplog.at_level("INFO"):
+        result = await watcher._upsert_document(test_file)
-            result = await watcher._upsert_document(test_file)
 
         assert result is not None
         assert result.id == "1"
-        # Should log that document was skipped, not updated
+        # Verify timestamp hasn't changed (document wasn't updated)
-        assert any("Skipped" in record.message for record in caplog.records)
+        assert result.updated_at == now
-        assert not any(
+
-            "Updated document" in record.message for record in caplog.records
+
+@pytest.mark.asyncio
+async def test_file_watcher_deletes_orphans():
+    """Test FileWatcher deletes documents whose files no longer exist."""
+    import asyncio
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+        existing_file = temp_path / "exists.txt"
+        existing_file.write_text("Existing file")
+
+        # Create a document for a file that doesn't exist
+        orphan_uri = (temp_path / "deleted.txt").as_uri()
+
+        mock_client = AsyncMock(spec=HaikuRAG)
+        orphan_doc = Document(id="orphan-1", content="Orphaned content", uri=orphan_uri)
+        existing_doc = Document(
+            id="existing-1", content="Existing content", uri=existing_file.as_uri()
         )
+
+        # Mock list_documents to return both documents
+        mock_client.list_documents.return_value = [orphan_doc, existing_doc]
+        mock_client.get_document_by_uri.return_value = None
+        mock_client.create_document_from_source.return_value = existing_doc
+
+        test_config = AppConfig(
+            monitor=MonitorConfig(directories=[temp_path], delete_orphans=True)
+        )
+        watcher = FileWatcher(client=mock_client, config=test_config)
+
+        # Run refresh which should delete orphan and process existing file
+        await watcher.refresh()
+
+        # Give background task time to complete
+        await asyncio.sleep(0.1)
+
+        # Should have deleted the orphan document
+        mock_client.delete_document.assert_called_once_with("orphan-1")
+        # Should have processed the existing file
+        mock_client.create_document_from_source.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_file_watcher_skips_orphan_deletion_when_disabled():
+    """Test FileWatcher does not delete orphans when delete_orphans is False."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+
+        # Create a document for a file that doesn't exist
+        orphan_uri = (temp_path / "deleted.txt").as_uri()
+
+        mock_client = AsyncMock(spec=HaikuRAG)
+        orphan_doc = Document(id="orphan-1", content="Orphaned content", uri=orphan_uri)
+
+        # Mock list_documents to return orphan document
+        mock_client.list_documents.return_value = [orphan_doc]
+
+        test_config = AppConfig(
+            monitor=MonitorConfig(directories=[temp_path], delete_orphans=False)
+        )
+        watcher = FileWatcher(client=mock_client, config=test_config)
+
+        # Run refresh
+        await watcher.refresh()
+
+        # Should NOT have deleted the orphan document
+        mock_client.delete_document.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_file_watcher_orphan_deletion_respects_patterns():
+    """Test orphan deletion respects include/ignore patterns."""
+    import asyncio
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+
+        # Create documents for files that don't exist
+        ignored_orphan_uri = (temp_path / "draft.md").as_uri()
+        excluded_orphan_uri = (temp_path / "file.pdf").as_uri()
+        included_orphan_uri = (temp_path / "readme.md").as_uri()
+
+        mock_client = AsyncMock(spec=HaikuRAG)
+
+        ignored_doc = Document(
+            id="ignored-1", content="Ignored", uri=ignored_orphan_uri
+        )
+        excluded_doc = Document(
+            id="excluded-1", content="Excluded", uri=excluded_orphan_uri
+        )
+        included_doc = Document(
+            id="included-1", content="Included", uri=included_orphan_uri
+        )
+
+        # Mock list_documents to return all orphan documents
+        mock_client.list_documents.return_value = [
+            ignored_doc,
+            excluded_doc,
+            included_doc,
+        ]
+
+        # Config with patterns: only .md files, but exclude draft*
+        test_config = AppConfig(
+            monitor=MonitorConfig(
+                directories=[temp_path],
+                delete_orphans=True,
+                include_patterns=["*.md"],
+                ignore_patterns=["draft*"],
+            )
+        )
+        watcher = FileWatcher(client=mock_client, config=test_config)
+
+        # Run refresh
+        await watcher.refresh()
+
+        # Give background task time to complete
+        await asyncio.sleep(0.1)
+
+        # Should only delete the included orphan (readme.md)
+        # - draft.md matches ignore pattern -> NOT deleted
+        # - file.pdf doesn't match include pattern -> NOT deleted
+        # - readme.md matches include and not ignored -> DELETED
+        mock_client.delete_document.assert_called_once_with("included-1")
+
+
+@pytest.mark.asyncio
+async def test_file_watcher_orphan_handles_spaces_in_filenames():
+    """Test orphan deletion correctly handles files with spaces in names."""
+    import asyncio
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+        # Create a file with spaces that exists
+        existing_file = temp_path / "my file with spaces.txt"
+        existing_file.write_text("Existing file")
+
+        mock_client = AsyncMock(spec=HaikuRAG)
+        # Document with URI that has URL-encoded spaces (%20)
+        existing_doc = Document(
+            id="existing-1", content="Existing", uri=existing_file.as_uri()
+        )
+
+        # Mock list_documents to return document with encoded spaces
+        mock_client.list_documents.return_value = [existing_doc]
+        mock_client.get_document_by_uri.return_value = None
+        mock_client.create_document_from_source.return_value = existing_doc
+
+        test_config = AppConfig(
+            monitor=MonitorConfig(directories=[temp_path], delete_orphans=True)
+        )
+        watcher = FileWatcher(client=mock_client, config=test_config)
+
+        # Run refresh
+        await watcher.refresh()
+
+        # Give background task time to complete
+        await asyncio.sleep(0.1)
+
+        # Should NOT delete the document since file exists
+        mock_client.delete_document.assert_not_called()