diff --git a/docs/cli.md b/docs/cli.md index 2b2e88ec..0907d0d1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -45,6 +45,14 @@ haiku-rag add-src /mnt/data/doc1.pdf --title "Q3 Financial Report" haiku-rag add-src /mnt/data/doc1.pdf --meta source=manual --meta page_count=12 --meta published=true ``` +From directory (recursively adds all supported files): +```bash +haiku-rag add-src /path/to/documents/ +``` + +!!! note + When adding a directory, the same content filters configured for [file monitoring](configuration.md#filtering-monitored-files) are applied. This means `ignore_patterns` and `include_patterns` from your configuration will be used to filter which files are added. + !!! note As you add documents to `haiku.rag` the database keeps growing. By default, LanceDB supports versioning of your data. Create/update operations are atomic‑feeling: if anything fails during chunking or embedding, @@ -205,7 +213,7 @@ Reduce disk usage by optimizing and pruning old table versions across all tables haiku-rag vacuum ``` -**Automatic Cleanup:** Vacuum runs automatically in the background after document operations. By default, it removes versions older than 60 seconds (`VACUUM_RETENTION_SECONDS`), preserving recent versions for concurrent connections. Manual vacuum can be useful for cleanup after bulk operations or to free disk space immediately. +**Automatic Cleanup:** Vacuum runs automatically in the background after document operations. By default, it removes versions older than 60 seconds (configurable via `storage.vacuum_retention_seconds`), preserving recent versions for concurrent connections. Manual vacuum can be useful for cleanup after bulk operations or to free disk space immediately. ### Rebuild Database diff --git a/docs/tutorial.md b/docs/tutorial.md index 9f15dbbc..e9b26df3 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -149,7 +149,7 @@ logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger.debug("AGI here we come") -# Uses LanceDB database from Config.DEFAULT_DATA_DIR +# Uses LanceDB database from default storage location async with HaikuRAG() as client: answer = await client.ask("What is the best programming language in the world?") print(answer) diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py index b9d4a1b5..9a81f906 100644 --- a/src/haiku/rag/client.py +++ b/src/haiku/rag/client.py @@ -135,9 +135,6 @@ class HaikuRAG: ValueError: If the file/URL cannot be parsed or doesn't exist httpx.RequestError: If URL request fails """ - # Lazy import to avoid loading docling - from haiku.rag.reader import FileReader - # Normalize metadata metadata = metadata or {} @@ -157,15 +154,17 @@ class HaikuRAG: # Handle directories if source_path.is_dir(): + from haiku.rag.monitor import FileFilter + documents = [] - supported_extensions = set(FileReader.extensions) - for file_path in source_path.rglob("*"): - if ( - file_path.is_file() - and file_path.suffix.lower() in supported_extensions - ): + filter = FileFilter( + ignore_patterns=self._config.monitor.ignore_patterns or None, + include_patterns=self._config.monitor.include_patterns or None, + ) + for path in source_path.rglob("*"): + if path.is_file() and filter.include_file(str(path)): doc = await self._create_document_from_file( - file_path, title=None, metadata=metadata + path, title=None, metadata=metadata ) documents.append(doc) return documents diff --git a/src/haiku/rag/monitor.py b/src/haiku/rag/monitor.py index 324a75be..55547365 100644 --- a/src/haiku/rag/monitor.py +++ b/src/haiku/rag/monitor.py @@ -40,6 +40,14 @@ class FileFilter(DefaultFilter): super().__init__() def __call__(self, change: Change, path: str) -> bool: + if not self.include_file(path): + return False + + # Apply default watchfiles filter + return super().__call__(change, path) + + def include_file(self, path: str) -> bool: + """Check if a file should be included based on filters.""" # Check extension filter if not path.endswith(self.extensions): return False @@ -54,8 +62,7 @@ class FileFilter(DefaultFilter): if self.ignore_spec.match_file(path): return False - # Apply default watchfiles filter - return super().__call__(change, path) + return True class FileWatcher: diff --git a/tests/test_client.py b/tests/test_client.py index a7889912..99fa57f6 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -210,6 +210,55 @@ async def test_client_create_document_from_directory(temp_db_path): assert not any("unsupported.xyz" in uri for uri in uris) +@pytest.mark.asyncio +async def test_client_create_document_from_directory_with_filters( + monkeypatch, temp_db_path +): + """Test creating documents from a directory with ignore and include patterns.""" + # Mock config to have ignore and include patterns + monkeypatch.setattr( + "haiku.rag.client.Config.monitor.ignore_patterns", ["**/ignore_me/**", "*.log"] + ) + monkeypatch.setattr( + "haiku.rag.client.Config.monitor.include_patterns", ["**/include/**/*.txt"] + ) + + async with HaikuRAG(temp_db_path) as client: + with tempfile.TemporaryDirectory() as temp_dir: + test_dir = Path(temp_dir) / "test_docs" + test_dir.mkdir() + + # Create files in include directory - should be included + include_dir = test_dir / "include" + include_dir.mkdir() + (include_dir / "doc1.txt").write_text("Content of doc1") + (include_dir / "doc2.txt").write_text("Content of doc2") + + # Create files outside include directory - should be excluded by include pattern + (test_dir / "doc3.txt").write_text("Content of doc3") + + # Create files in ignore directory - should be excluded by ignore pattern + ignore_dir = test_dir / "ignore_me" + ignore_dir.mkdir() + (ignore_dir / "doc4.txt").write_text("Content of doc4") + + # Create log file - should be excluded by ignore pattern + (test_dir / "debug.log").write_text("log content") + + result = await client.create_document_from_source(test_dir) + + assert isinstance(result, list) + # Should only include doc1.txt and doc2.txt from include directory + assert len(result) == 2 + + uris = [doc.uri for doc in result if doc.uri] + assert any("doc1.txt" in uri for uri in uris) + assert any("doc2.txt" in uri for uri in uris) + assert not any("doc3.txt" in uri for uri in uris) + assert not any("doc4.txt" in uri for uri in uris) + assert not any("debug.log" in uri for uri in uris) + + @pytest.mark.asyncio async def test_client_create_document_from_url(temp_db_path): """Test creating a document from a URL."""