Merge pull request #124 from ggozad/fix/add-src-monitor-filter

Apply configured include/exclude filters when using CLI's add-src to add a directory

docs/cli.md

@@ -45,6 +45,14 @@ haiku-rag add-src /mnt/data/doc1.pdf --title "Q3 Financial Report"
 haiku-rag add-src /mnt/data/doc1.pdf --meta source=manual --meta page_count=12 --meta published=true
 ```
 
+From directory (recursively adds all supported files):
+```bash
+haiku-rag add-src /path/to/documents/
+```
+
+!!! note
+    When adding a directory, the same content filters configured for [file monitoring](configuration.md#filtering-monitored-files) are applied. This means `ignore_patterns` and `include_patterns` from your configuration will be used to filter which files are added.
+
 !!! note
     As you add documents to `haiku.rag` the database keeps growing. By default, LanceDB supports versioning
     of your data. Create/update operations are atomic‑feeling: if anything fails during chunking or embedding,
@@ -205,7 +213,7 @@ Reduce disk usage by optimizing and pruning old table versions across all tables
 haiku-rag vacuum
 ```
 
-**Automatic Cleanup:** Vacuum runs automatically in the background after document operations. By default, it removes versions older than 60 seconds (`VACUUM_RETENTION_SECONDS`), preserving recent versions for concurrent connections. Manual vacuum can be useful for cleanup after bulk operations or to free disk space immediately.
+**Automatic Cleanup:** Vacuum runs automatically in the background after document operations. By default, it removes versions older than 60 seconds (configurable via `storage.vacuum_retention_seconds`), preserving recent versions for concurrent connections. Manual vacuum can be useful for cleanup after bulk operations or to free disk space immediately.
 
 ### Rebuild Database
 

docs/tutorial.md

@@ -149,7 +149,7 @@ logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
 logger.debug("AGI here we come")
 
-# Uses LanceDB database from Config.DEFAULT_DATA_DIR
+# Uses LanceDB database from default storage location
 async with HaikuRAG() as client:
     answer = await client.ask("What is the best programming language in the world?")
     print(answer)

src/haiku/rag/client.py

@@ -135,9 +135,6 @@ class HaikuRAG:
             ValueError: If the file/URL cannot be parsed or doesn't exist
             httpx.RequestError: If URL request fails
         """
-        # Lazy import to avoid loading docling
-        from haiku.rag.reader import FileReader
-
         # Normalize metadata
         metadata = metadata or {}
 
@@ -157,15 +154,17 @@ class HaikuRAG:
 
         # Handle directories
         if source_path.is_dir():
+            from haiku.rag.monitor import FileFilter
+
             documents = []
-            supported_extensions = set(FileReader.extensions)
-            for file_path in source_path.rglob("*"):
-                if (
-                    file_path.is_file()
-                    and file_path.suffix.lower() in supported_extensions
-                ):
+            filter = FileFilter(
+                ignore_patterns=self._config.monitor.ignore_patterns or None,
+                include_patterns=self._config.monitor.include_patterns or None,
+            )
+            for path in source_path.rglob("*"):
+                if path.is_file() and filter.include_file(str(path)):
                     doc = await self._create_document_from_file(
-                        file_path, title=None, metadata=metadata
+                        path, title=None, metadata=metadata
                     )
                     documents.append(doc)
             return documents

src/haiku/rag/monitor.py

@@ -40,6 +40,14 @@ class FileFilter(DefaultFilter):
         super().__init__()
 
     def __call__(self, change: Change, path: str) -> bool:
+        if not self.include_file(path):
+            return False
+
+        # Apply default watchfiles filter
+        return super().__call__(change, path)
+
+    def include_file(self, path: str) -> bool:
+        """Check if a file should be included based on filters."""
         # Check extension filter
         if not path.endswith(self.extensions):
             return False
@@ -54,8 +62,7 @@ class FileFilter(DefaultFilter):
             if self.ignore_spec.match_file(path):
                 return False
 
-        # Apply default watchfiles filter
-        return super().__call__(change, path)
+        return True
 
 
 class FileWatcher:

tests/test_client.py

@@ -210,6 +210,55 @@ async def test_client_create_document_from_directory(temp_db_path):
             assert not any("unsupported.xyz" in uri for uri in uris)
 
 
+@pytest.mark.asyncio
+async def test_client_create_document_from_directory_with_filters(
+    monkeypatch, temp_db_path
+):
+    """Test creating documents from a directory with ignore and include patterns."""
+    # Mock config to have ignore and include patterns
+    monkeypatch.setattr(
+        "haiku.rag.client.Config.monitor.ignore_patterns", ["**/ignore_me/**", "*.log"]
+    )
+    monkeypatch.setattr(
+        "haiku.rag.client.Config.monitor.include_patterns", ["**/include/**/*.txt"]
+    )
+
+    async with HaikuRAG(temp_db_path) as client:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            test_dir = Path(temp_dir) / "test_docs"
+            test_dir.mkdir()
+
+            # Create files in include directory - should be included
+            include_dir = test_dir / "include"
+            include_dir.mkdir()
+            (include_dir / "doc1.txt").write_text("Content of doc1")
+            (include_dir / "doc2.txt").write_text("Content of doc2")
+
+            # Create files outside include directory - should be excluded by include pattern
+            (test_dir / "doc3.txt").write_text("Content of doc3")
+
+            # Create files in ignore directory - should be excluded by ignore pattern
+            ignore_dir = test_dir / "ignore_me"
+            ignore_dir.mkdir()
+            (ignore_dir / "doc4.txt").write_text("Content of doc4")
+
+            # Create log file - should be excluded by ignore pattern
+            (test_dir / "debug.log").write_text("log content")
+
+            result = await client.create_document_from_source(test_dir)
+
+            assert isinstance(result, list)
+            # Should only include doc1.txt and doc2.txt from include directory
+            assert len(result) == 2
+
+            uris = [doc.uri for doc in result if doc.uri]
+            assert any("doc1.txt" in uri for uri in uris)
+            assert any("doc2.txt" in uri for uri in uris)
+            assert not any("doc3.txt" in uri for uri in uris)
+            assert not any("doc4.txt" in uri for uri in uris)
+            assert not any("debug.log" in uri for uri in uris)
+
+
 @pytest.mark.asyncio
 async def test_client_create_document_from_url(temp_db_path):
     """Test creating a document from a URL."""