From cabb7748f577f5f0f2a746dfda390a3c8ba3d624 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 29 Oct 2025 10:27:16 +0200 Subject: [PATCH] Filtering based on pathspec --- src/haiku/rag/app.py | 4 +-- src/haiku/rag/monitor.py | 62 +++++++++++++++++++++++++++++++++++----- tests/test_monitor.py | 21 +++++++++++--- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/src/haiku/rag/app.py b/src/haiku/rag/app.py index ac789d13..e7609f8d 100644 --- a/src/haiku/rag/app.py +++ b/src/haiku/rag/app.py @@ -474,9 +474,7 @@ class HaikuRAGApp: # Start file monitor if enabled if enable_monitor: - monitor = FileWatcher( - paths=Config.storage.monitor_directories, client=client - ) + monitor = FileWatcher(client=client) monitor_task = asyncio.create_task(monitor.observe()) tasks.append(monitor_task) diff --git a/src/haiku/rag/monitor.py b/src/haiku/rag/monitor.py index 573484da..3eb51ae6 100644 --- a/src/haiku/rag/monitor.py +++ b/src/haiku/rag/monitor.py @@ -2,9 +2,12 @@ import logging from pathlib import Path from typing import TYPE_CHECKING +import pathspec +from pathspec.patterns.gitwildmatch import GitWildMatchPattern from watchfiles import Change, DefaultFilter, awatch from haiku.rag.client import HaikuRAG +from haiku.rag.config import AppConfig, Config from haiku.rag.store.models.document import Document if TYPE_CHECKING: @@ -14,25 +17,63 @@ logger = logging.getLogger(__name__) class FileFilter(DefaultFilter): - def __init__(self, *, ignore_paths: list[Path] | None = None) -> None: + def __init__( + self, + *, + ignore_patterns: list[str] | None = None, + include_patterns: list[str] | None = None, + ) -> None: # Lazy import to avoid loading docling from haiku.rag.reader import FileReader self.extensions = tuple(FileReader.extensions) - super().__init__(ignore_paths=ignore_paths) + self.ignore_spec = ( + pathspec.PathSpec.from_lines(GitWildMatchPattern, ignore_patterns) + if ignore_patterns + else None + ) + self.include_spec = ( + pathspec.PathSpec.from_lines(GitWildMatchPattern, include_patterns) + if include_patterns + else None + ) + super().__init__() def __call__(self, change: Change, path: str) -> bool: - return path.endswith(self.extensions) and super().__call__(change, path) + # Check extension filter + if not path.endswith(self.extensions): + return False + + # Apply include patterns if specified (whitelist mode) + if self.include_spec: + if not self.include_spec.match_file(path): + return False + + # Apply ignore patterns (blacklist mode) + if self.ignore_spec: + if self.ignore_spec.match_file(path): + return False + + # Apply default watchfiles filter + return super().__call__(change, path) class FileWatcher: - def __init__(self, paths: list[Path], client: HaikuRAG): - self.paths = paths + def __init__( + self, + client: HaikuRAG, + config: AppConfig = Config, + ): + self.paths = config.storage.monitor_directories self.client = client + self.ignore_patterns = config.storage.monitor_ignore_patterns or None + self.include_patterns = config.storage.monitor_include_patterns or None async def observe(self): logger.info(f"Watching files in {self.paths}") - filter = FileFilter() + filter = FileFilter( + ignore_patterns=self.ignore_patterns, include_patterns=self.include_patterns + ) await self.refresh() async for changes in awatch(*self.paths, watch_filter=filter): @@ -49,10 +90,17 @@ class FileWatcher: # Lazy import to avoid loading docling from haiku.rag.reader import FileReader + # Create filter to apply same logic as observe() + filter = FileFilter( + ignore_patterns=self.ignore_patterns, include_patterns=self.include_patterns + ) + for path in self.paths: for f in Path(path).rglob("**/*"): if f.is_file() and f.suffix in FileReader.extensions: - await self._upsert_document(f) + # Apply pattern filters + if filter(Change.added, str(f)): + await self._upsert_document(f) async def _upsert_document(self, file: Path) -> Document | None: try: diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 811a1b34..2059704e 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -5,6 +5,7 @@ from unittest.mock import AsyncMock import pytest from haiku.rag.client import HaikuRAG +from haiku.rag.config import AppConfig, StorageConfig from haiku.rag.monitor import FileWatcher from haiku.rag.store.models.document import Document @@ -22,7 +23,10 @@ async def test_file_watcher_upsert_document(): mock_client.create_document_from_source.return_value = mock_doc mock_client.get_document_by_uri.return_value = None # No existing document - watcher = FileWatcher(paths=[temp_path.parent], client=mock_client) + test_config = AppConfig( + storage=StorageConfig(monitor_directories=[temp_path.parent]) + ) + watcher = FileWatcher(client=mock_client, config=test_config) result = await watcher._upsert_document(temp_path) @@ -49,7 +53,10 @@ async def test_file_watcher_upsert_existing_document(): mock_client.get_document_by_uri.return_value = existing_doc mock_client.create_document_from_source.return_value = updated_doc - watcher = FileWatcher(paths=[temp_path.parent], client=mock_client) + test_config = AppConfig( + storage=StorageConfig(monitor_directories=[temp_path.parent]) + ) + watcher = FileWatcher(client=mock_client, config=test_config) result = await watcher._upsert_document(temp_path) @@ -69,7 +76,10 @@ async def test_file_watcher_delete_document(): mock_client.get_document_by_uri.return_value = existing_doc mock_client.delete_document.return_value = True - watcher = FileWatcher(paths=[temp_path.parent], client=mock_client) + test_config = AppConfig( + storage=StorageConfig(monitor_directories=[temp_path.parent]) + ) + watcher = FileWatcher(client=mock_client, config=test_config) await watcher._delete_document(temp_path) @@ -85,7 +95,10 @@ async def test_file_watcher_delete_nonexistent_document(): mock_client = AsyncMock(spec=HaikuRAG) mock_client.get_document_by_uri.return_value = None - watcher = FileWatcher(paths=[temp_path.parent], client=mock_client) + test_config = AppConfig( + storage=StorageConfig(monitor_directories=[temp_path.parent]) + ) + watcher = FileWatcher(client=mock_client, config=test_config) await watcher._delete_document(temp_path)