diff --git a/docs/configuration.md b/docs/configuration.md index a22c0428..d56d7c16 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -55,14 +55,16 @@ environment: production storage: data_dir: "" # Empty = use default platform location - monitor_directories: - - /path/to/documents - - /another/path - monitor_ignore_patterns: [] # Gitignore-style patterns to exclude - monitor_include_patterns: [] # Gitignore-style patterns to include disable_autocreate: false vacuum_retention_seconds: 60 +monitor: + directories: + - /path/to/documents + - /another/path + ignore_patterns: [] # Gitignore-style patterns to exclude + include_patterns: [] # Gitignore-style patterns to include + lancedb: uri: "" # Empty for local, or db://, s3://, az://, gs:// api_key: "" @@ -163,8 +165,8 @@ export OLLAMA_BASE_URL=http://localhost:11434 Set directories to monitor for automatic indexing: ```yaml -storage: - monitor_directories: +monitor: + directories: - /path/to/documents - /another_path/to/documents ``` @@ -174,19 +176,19 @@ storage: Use gitignore-style patterns to control which files are monitored: ```yaml -storage: - monitor_directories: +monitor: + directories: - /path/to/documents # Exclude specific files or directories - monitor_ignore_patterns: + ignore_patterns: - "*draft*" # Ignore files with "draft" in the name - "temp/" # Ignore temp directory - "**/archive/**" # Ignore all archive directories - "*.backup" # Ignore backup files # Only include specific files (whitelist mode) - monitor_include_patterns: + include_patterns: - "*.md" # Only markdown files - "*.pdf" # Only PDF files - "**/docs/**" # Only files in docs directories @@ -203,18 +205,20 @@ storage: ```yaml # Only monitor markdown documentation, but ignore drafts -monitor_include_patterns: - - "*.md" -monitor_ignore_patterns: - - "*draft*" - - "*WIP*" +monitor: + include_patterns: + - "*.md" + ignore_patterns: + - "*draft*" + - "*WIP*" # Monitor all supported files except in specific directories -monitor_ignore_patterns: - - "node_modules/" - - ".git/" - - "**/test/**" - - "**/temp/**" +monitor: + ignore_patterns: + - "node_modules/" + - ".git/" + - "**/test/**" + - "**/temp/**" ``` Patterns follow [gitignore syntax](https://git-scm.com/docs/gitignore#_pattern_format): @@ -589,8 +593,8 @@ This will read your current environment variables and generate a `haiku.rag.yaml When migrating from environment variables, list values like `MONITOR_DIRECTORIES` that were comma-separated (`/path1,/path2`) will be converted to proper YAML lists. In YAML, always use list syntax: ```yaml - storage: - monitor_directories: + monitor: + directories: - /path/to/dir1 - /path/to/dir2 ``` diff --git a/docs/server.md b/docs/server.md index ef79e7f9..aa7a472f 100644 --- a/docs/server.md +++ b/docs/server.md @@ -48,8 +48,8 @@ This will start file monitoring, MCP server on port 8001, and A2A server on port Configure directories to monitor in your `haiku.rag.yaml`: ```yaml -storage: - monitor_directories: +monitor: + directories: - /path/to/documents - /another/path ``` @@ -71,18 +71,18 @@ haiku-rag serve --monitor You can filter which files to monitor using gitignore-style patterns: ```yaml -storage: - monitor_directories: +monitor: + directories: - /path/to/documents # Ignore patterns (exclude files) - monitor_ignore_patterns: + ignore_patterns: - "*draft*" # Ignore draft files - "temp/" # Ignore temp directory - "**/archive/**" # Ignore archive directories # Include patterns (whitelist files) - monitor_include_patterns: + include_patterns: - "*.md" # Only markdown files - "**/docs/**" # Files in docs directories ``` diff --git a/src/haiku/rag/config/__init__.py b/src/haiku/rag/config/__init__.py index d8d672d2..0a01fba7 100644 --- a/src/haiku/rag/config/__init__.py +++ b/src/haiku/rag/config/__init__.py @@ -11,6 +11,7 @@ from haiku.rag.config.models import ( AppConfig, EmbeddingsConfig, LanceDBConfig, + MonitorConfig, OllamaConfig, ProcessingConfig, ProvidersConfig, @@ -25,6 +26,7 @@ __all__ = [ "Config", "AppConfig", "StorageConfig", + "MonitorConfig", "LanceDBConfig", "EmbeddingsConfig", "RerankingConfig", diff --git a/src/haiku/rag/config/loader.py b/src/haiku/rag/config/loader.py index 70e91b32..c200d2f7 100644 --- a/src/haiku/rag/config/loader.py +++ b/src/haiku/rag/config/loader.py @@ -50,10 +50,14 @@ def generate_default_config() -> dict: "environment": "production", "storage": { "data_dir": "", - "monitor_directories": [], "disable_autocreate": False, "vacuum_retention_seconds": 60, }, + "monitor": { + "directories": [], + "ignore_patterns": [], + "include_patterns": [], + }, "lancedb": {"uri": "", "api_key": "", "region": ""}, "embeddings": { "provider": "ollama", @@ -88,7 +92,7 @@ def load_config_from_env() -> dict: env_mappings = { "ENV": "environment", "DEFAULT_DATA_DIR": ("storage", "data_dir"), - "MONITOR_DIRECTORIES": ("storage", "monitor_directories"), + "MONITOR_DIRECTORIES": ("monitor", "directories"), "DISABLE_DB_AUTOCREATE": ("storage", "disable_autocreate"), "VACUUM_RETENTION_SECONDS": ("storage", "vacuum_retention_seconds"), "LANCEDB_URI": ("lancedb", "uri"), diff --git a/src/haiku/rag/config/models.py b/src/haiku/rag/config/models.py index 67bd3f71..4c654e7a 100644 --- a/src/haiku/rag/config/models.py +++ b/src/haiku/rag/config/models.py @@ -7,13 +7,16 @@ from haiku.rag.utils import get_default_data_dir class StorageConfig(BaseModel): data_dir: Path = Field(default_factory=get_default_data_dir) - monitor_directories: list[Path] = [] - monitor_ignore_patterns: list[str] = [] - monitor_include_patterns: list[str] = [] disable_autocreate: bool = False vacuum_retention_seconds: int = 60 +class MonitorConfig(BaseModel): + directories: list[Path] = [] + ignore_patterns: list[str] = [] + include_patterns: list[str] = [] + + class LanceDBConfig(BaseModel): uri: str = "" api_key: str = "" @@ -74,6 +77,7 @@ class A2AConfig(BaseModel): class AppConfig(BaseModel): environment: str = "production" storage: StorageConfig = Field(default_factory=StorageConfig) + monitor: MonitorConfig = Field(default_factory=MonitorConfig) lancedb: LanceDBConfig = Field(default_factory=LanceDBConfig) embeddings: EmbeddingsConfig = Field(default_factory=EmbeddingsConfig) reranking: RerankingConfig = Field(default_factory=RerankingConfig) diff --git a/src/haiku/rag/monitor.py b/src/haiku/rag/monitor.py index 3eb51ae6..324a75be 100644 --- a/src/haiku/rag/monitor.py +++ b/src/haiku/rag/monitor.py @@ -64,10 +64,10 @@ class FileWatcher: client: HaikuRAG, config: AppConfig = Config, ): - self.paths = config.storage.monitor_directories + self.paths = config.monitor.directories self.client = client - self.ignore_patterns = config.storage.monitor_ignore_patterns or None - self.include_patterns = config.storage.monitor_include_patterns or None + self.ignore_patterns = config.monitor.ignore_patterns or None + self.include_patterns = config.monitor.include_patterns or None async def observe(self): logger.info(f"Watching files in {self.paths}") diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 75f09955..75ea64ee 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -5,7 +5,7 @@ from unittest.mock import AsyncMock import pytest from haiku.rag.client import HaikuRAG -from haiku.rag.config import AppConfig, StorageConfig +from haiku.rag.config import AppConfig, MonitorConfig from haiku.rag.monitor import FileWatcher from haiku.rag.store.models.document import Document @@ -23,9 +23,7 @@ async def test_file_watcher_upsert_document(): mock_client.create_document_from_source.return_value = mock_doc mock_client.get_document_by_uri.return_value = None # No existing document - test_config = AppConfig( - storage=StorageConfig(monitor_directories=[temp_path.parent]) - ) + test_config = AppConfig(monitor=MonitorConfig(directories=[temp_path.parent])) watcher = FileWatcher(client=mock_client, config=test_config) result = await watcher._upsert_document(temp_path) @@ -53,9 +51,7 @@ async def test_file_watcher_upsert_existing_document(): mock_client.get_document_by_uri.return_value = existing_doc mock_client.create_document_from_source.return_value = updated_doc - test_config = AppConfig( - storage=StorageConfig(monitor_directories=[temp_path.parent]) - ) + test_config = AppConfig(monitor=MonitorConfig(directories=[temp_path.parent])) watcher = FileWatcher(client=mock_client, config=test_config) result = await watcher._upsert_document(temp_path) @@ -76,9 +72,7 @@ async def test_file_watcher_delete_document(): mock_client.get_document_by_uri.return_value = existing_doc mock_client.delete_document.return_value = True - test_config = AppConfig( - storage=StorageConfig(monitor_directories=[temp_path.parent]) - ) + test_config = AppConfig(monitor=MonitorConfig(directories=[temp_path.parent])) watcher = FileWatcher(client=mock_client, config=test_config) await watcher._delete_document(temp_path) @@ -95,9 +89,7 @@ async def test_file_watcher_delete_nonexistent_document(): mock_client = AsyncMock(spec=HaikuRAG) mock_client.get_document_by_uri.return_value = None - test_config = AppConfig( - storage=StorageConfig(monitor_directories=[temp_path.parent]) - ) + test_config = AppConfig(monitor=MonitorConfig(directories=[temp_path.parent])) watcher = FileWatcher(client=mock_client, config=test_config) await watcher._delete_document(temp_path) @@ -210,9 +202,7 @@ async def test_file_watcher_with_ignore_patterns(): mock_client.get_document_by_uri.return_value = None test_config = AppConfig( - storage=StorageConfig( - monitor_directories=[temp_path], monitor_ignore_patterns=["draft*"] - ) + monitor=MonitorConfig(directories=[temp_path], ignore_patterns=["draft*"]) ) watcher = FileWatcher(client=mock_client, config=test_config) @@ -243,9 +233,7 @@ async def test_file_watcher_with_include_patterns(): mock_client.get_document_by_uri.return_value = None test_config = AppConfig( - storage=StorageConfig( - monitor_directories=[temp_path], monitor_include_patterns=["*.md"] - ) + monitor=MonitorConfig(directories=[temp_path], include_patterns=["*.md"]) ) watcher = FileWatcher(client=mock_client, config=test_config) diff --git a/tests/test_settings.py b/tests/test_settings.py index e78105b0..d6e72d29 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -86,7 +86,9 @@ async def test_config_validation_on_db_load(temp_db_path): def test_monitor_filter_patterns_config(): """Test that monitor filter patterns are available in config.""" - assert hasattr(Config.storage, "monitor_ignore_patterns") - assert hasattr(Config.storage, "monitor_include_patterns") - assert isinstance(Config.storage.monitor_ignore_patterns, list) - assert isinstance(Config.storage.monitor_include_patterns, list) + assert hasattr(Config.monitor, "ignore_patterns") + assert hasattr(Config.monitor, "include_patterns") + assert hasattr(Config.monitor, "directories") + assert isinstance(Config.monitor.ignore_patterns, list) + assert isinstance(Config.monitor.include_patterns, list) + assert isinstance(Config.monitor.directories, list)