diff --git a/CHANGELOG.md b/CHANGELOG.md index a38a3543..2df0a506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # Changelog ## [Unreleased] +### Changed + +- **Configuration**: All CLI commands now properly support `--config` parameter for specifying custom configuration files +- Configuration loading consolidated across CLI, app, and client with consistent resolution order +- `HaikuRAGApp`, MCP server, and A2A server now accept `config` parameter for programmatic configuration +- Updated CLI documentation to clarify global vs per-command options +- **BREAKING**: Standardized configuration filename to `haiku.rag.yaml` in user directories (was incorrectly using `config.yaml`). Users with existing `config.yaml` in their user directory will need to rename it to `haiku.rag.yaml` + +### Removed + +- **BREAKING**: Removed deprecated `.env`-based configuration system. The `haiku-rag init-config --from-env` command and `load_config_from_env()` function have been removed. All configuration must now be done via YAML files. Environment variables for API keys (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) and service URLs (e.g., `OLLAMA_BASE_URL`) are still supported and can be set via `.env` files. + ## [0.14.1] - 2025-11-06 ### Added diff --git a/README.md b/README.md index eb0a429b..6591b030 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ Retrieval-Augmented Generation (RAG) library built on LanceDB. `haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported. -> **Note**: Configuration now uses YAML files instead of environment variables. If you're upgrading from an older version, run `haiku-rag init-config --from-env` to migrate your `.env` file to `haiku.rag.yaml`. See [Configuration](https://ggozad.github.io/haiku.rag/configuration/) for details. - ## Features - **Local LanceDB**: No external servers required, supports also LanceDB cloud storage, S3, Google Cloud & Azure diff --git a/docs/cli.md b/docs/cli.md index c1c650e4..f040aac8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,14 +3,20 @@ The `haiku-rag` CLI provides complete document management functionality. !!! note - All commands support: + Global options (must be specified before the command): + + - `--config` - Specify custom configuration file + - `--version` / `-v` - Show version and exit + + Per-command options: - `--db` - Specify custom database path - `-h` - Show help for specific command Example: ```bash - haiku-rag list --db /path/to/custom.db + haiku-rag --config /path/to/config.yaml list + haiku-rag --config /path/to/config.yaml list --db /path/to/custom.db haiku-rag add -h ``` diff --git a/docs/configuration.md b/docs/configuration.md index 8cb6c986..41f85975 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -15,15 +15,6 @@ haiku-rag init-config This creates a `haiku.rag.yaml` file in your current directory with all available settings. -!!! warning "Deprecation Notice" - Environment variable configuration via `.env` files is deprecated and will be removed in future versions. Please migrate to YAML configuration. - - To migrate from environment variables (`.env` file): - - ```bash - haiku-rag init-config --from-env - ``` - ## Configuration File Locations `haiku.rag` searches for configuration files in this order: @@ -31,9 +22,9 @@ This creates a `haiku.rag.yaml` file in your current directory with all availabl 1. Path specified via `--config` flag: `haiku-rag --config /path/to/config.yaml ` 2. `./haiku.rag.yaml` (current directory) 3. Platform-specific user directory: - - **Linux**: `~/.local/share/haiku.rag/config.yaml` - - **macOS**: `~/Library/Application Support/haiku.rag/config.yaml` - - **Windows**: `C:/Users//AppData/Roaming/haiku.rag/config.yaml` + - **Linux**: `~/.local/share/haiku.rag/haiku.rag.yaml` + - **macOS**: `~/Library/Application Support/haiku.rag/haiku.rag.yaml` + - **Windows**: `C:/Users//AppData/Roaming/haiku.rag/haiku.rag.yaml` ## Minimal Configuration @@ -219,14 +210,7 @@ embeddings: vector_dim: 1024 ``` -The Ollama base URL can be configured via environment variable or config file: - -```bash -# Via environment variable (recommended) -export OLLAMA_BASE_URL=http://localhost:11434 -``` - -Or in your config file: +The Ollama base URL can be configured in your config file or via environment variable: ```yaml providers: @@ -234,7 +218,16 @@ providers: base_url: http://localhost:11434 ``` -If neither is set, it defaults to `http://localhost:11434`. +Or via environment variable: + +```bash +export OLLAMA_BASE_URL=http://localhost:11434 +``` + +If not configured, it defaults to `http://localhost:11434`. + +!!! note + You can use a `.env` file in your project directory to set environment variables like `OLLAMA_BASE_URL` and API keys (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`). These will be automatically loaded when running `haiku-rag` commands. ### VoyageAI diff --git a/haiku_rag_slim/haiku/rag/a2a/__init__.py b/haiku_rag_slim/haiku/rag/a2a/__init__.py index 76e1ee36..76cd1e7c 100644 --- a/haiku_rag_slim/haiku/rag/a2a/__init__.py +++ b/haiku_rag_slim/haiku/rag/a2a/__init__.py @@ -5,7 +5,7 @@ from pathlib import Path import logfire from pydantic_ai import Agent, RunContext -from haiku.rag.config import Config +from haiku.rag.config import AppConfig, Config from haiku.rag.graph_common import get_model from .context import load_message_history, save_message_history @@ -42,6 +42,7 @@ __all__ = [ def create_a2a_app( db_path: Path, + config: AppConfig = Config, security_schemes: dict | None = None, security: list[dict[str, list[str]]] | None = None, ): @@ -49,6 +50,7 @@ def create_a2a_app( Args: db_path: Path to the LanceDB database + config: App configuration security_schemes: Optional security scheme definitions for the AgentCard security: Optional security requirements for the AgentCard @@ -57,12 +59,12 @@ def create_a2a_app( """ base_storage = InMemoryStorage() storage = LRUMemoryStorage( - storage=base_storage, max_contexts=Config.a2a.max_contexts + storage=base_storage, max_contexts=config.a2a.max_contexts ) broker = InMemoryBroker() # Create the agent with native search tool - model = get_model(Config.qa.provider, Config.qa.model) + model = get_model(config.qa.provider, config.qa.model) agent = Agent( model=model, deps_type=AgentDependencies, @@ -115,12 +117,13 @@ def create_a2a_app( broker=broker, db_path=db_path, agent=agent, # type: ignore + config=config, ) # Create FastA2A app with custom worker lifecycle @asynccontextmanager async def lifespan(app): - logger.info(f"Started A2A server (max contexts: {Config.a2a.max_contexts})") + logger.info(f"Started A2A server (max contexts: {config.a2a.max_contexts})") async with app.task_manager: async with worker.run(): yield diff --git a/haiku_rag_slim/haiku/rag/a2a/worker.py b/haiku_rag_slim/haiku/rag/a2a/worker.py index dbda76ef..e491128e 100644 --- a/haiku_rag_slim/haiku/rag/a2a/worker.py +++ b/haiku_rag_slim/haiku/rag/a2a/worker.py @@ -9,6 +9,7 @@ from haiku.rag.a2a.context import load_message_history, save_message_history from haiku.rag.a2a.models import AgentDependencies from haiku.rag.a2a.skills import extract_question_from_task from haiku.rag.client import HaikuRAG +from haiku.rag.config import AppConfig, Config try: from fasta2a import Worker # type: ignore @@ -37,10 +38,12 @@ class ConversationalWorker(Worker[list[Message]]): broker, db_path: Path, agent: "Agent[AgentDependencies, str]", + config: AppConfig = Config, ): super().__init__(storage=storage, broker=broker) self.db_path = db_path self.agent = agent + self.config = config async def run_task(self, params: TaskSendParams) -> None: task = await self.storage.load_task(params["id"]) @@ -62,7 +65,7 @@ class ConversationalWorker(Worker[list[Message]]): return try: - async with HaikuRAG(self.db_path) as client: + async with HaikuRAG(self.db_path, config=self.config) as client: context = await self.storage.load_context(task["context_id"]) or [] message_history = load_message_history(context) diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index 6aa2c178..fabd2d19 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -9,7 +9,7 @@ from rich.markdown import Markdown from rich.progress import Progress from haiku.rag.client import HaikuRAG -from haiku.rag.config import Config +from haiku.rag.config import AppConfig, Config from haiku.rag.mcp import create_mcp_server from haiku.rag.monitor import FileWatcher from haiku.rag.research.dependencies import ResearchContext @@ -23,8 +23,9 @@ logger = logging.getLogger(__name__) class HaikuRAGApp: - def __init__(self, db_path: Path): + def __init__(self, db_path: Path, config: AppConfig = Config): self.db_path = db_path + self.config = config self.console = Console() async def info(self): @@ -136,13 +137,13 @@ class HaikuRAGApp: ) async def list_documents(self, filter: str | None = None): - async with HaikuRAG(db_path=self.db_path) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: documents = await self.client.list_documents(filter=filter) for doc in documents: self._rich_print_document(doc, truncate=True) async def add_document_from_text(self, text: str, metadata: dict | None = None): - async with HaikuRAG(db_path=self.db_path) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: doc = await self.client.create_document(text, metadata=metadata) self._rich_print_document(doc, truncate=True) self.console.print( @@ -152,7 +153,7 @@ class HaikuRAGApp: async def add_document_from_source( self, source: str, title: str | None = None, metadata: dict | None = None ): - async with HaikuRAG(db_path=self.db_path) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: result = await self.client.create_document_from_source( source, title=title, metadata=metadata ) @@ -169,7 +170,7 @@ class HaikuRAGApp: ) async def get_document(self, doc_id: str): - async with HaikuRAG(db_path=self.db_path) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: doc = await self.client.get_document_by_id(doc_id) if doc is None: self.console.print(f"[red]Document with id {doc_id} not found.[/red]") @@ -177,7 +178,7 @@ class HaikuRAGApp: self._rich_print_document(doc, truncate=False) async def delete_document(self, doc_id: str): - async with HaikuRAG(db_path=self.db_path) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: deleted = await self.client.delete_document(doc_id) if deleted: self.console.print( @@ -189,7 +190,7 @@ class HaikuRAGApp: ) async def search(self, query: str, limit: int = 5, filter: str | None = None): - async with HaikuRAG(db_path=self.db_path) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: results = await self.client.search(query, limit=limit, filter=filter) if not results: self.console.print("[yellow]No results found.[/yellow]") @@ -212,21 +213,20 @@ class HaikuRAGApp: deep: Use deep QA mode (multi-step reasoning) verbose: Show verbose output """ - async with HaikuRAG(db_path=self.db_path) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: try: if deep: from rich.console import Console - from haiku.rag.config import Config from haiku.rag.qa.deep.dependencies import DeepQAContext from haiku.rag.qa.deep.graph import build_deep_qa_graph from haiku.rag.qa.deep.state import DeepQADeps, DeepQAState - graph = build_deep_qa_graph(config=Config) + graph = build_deep_qa_graph(config=self.config) context = DeepQAContext( original_question=question, use_citations=cite ) - state = DeepQAState.from_config(context=context, config=Config) + state = DeepQAState.from_config(context=context, config=self.config) deps = DeepQADeps( client=self.client, console=Console() if verbose else None ) @@ -254,18 +254,16 @@ class HaikuRAGApp: question: The research question verbose: Show verbose output """ - async with HaikuRAG(db_path=self.db_path) as client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as client: try: - from haiku.rag.config import Config - if verbose: self.console.print("[bold cyan]Starting research[/bold cyan]") self.console.print(f"[bold blue]Question:[/bold blue] {question}") self.console.print() - graph = build_research_graph(config=Config) + graph = build_research_graph(config=self.config) context = ResearchContext(original_question=question) - state = ResearchState.from_config(context=context, config=Config) + state = ResearchState.from_config(context=context, config=self.config) deps = ResearchDeps( client=client, console=self.console if verbose else None ) @@ -341,7 +339,9 @@ class HaikuRAGApp: self.console.print(f"[red]Error during research: {e}[/red]") async def rebuild(self): - async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client: + async with HaikuRAG( + db_path=self.db_path, config=self.config, skip_validation=True + ) as client: try: documents = await client.list_documents() total_docs = len(documents) @@ -369,7 +369,9 @@ class HaikuRAGApp: async def vacuum(self): """Run database maintenance: optimize and cleanup table history.""" try: - async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client: + async with HaikuRAG( + db_path=self.db_path, config=self.config, skip_validation=True + ) as client: await client.vacuum() self.console.print( "[bold green]Vacuum completed successfully.[/bold green]" @@ -383,7 +385,7 @@ class HaikuRAGApp: self.console.print() # Get all config fields dynamically - for field_name, field_value in Config.model_dump().items(): + for field_name, field_value in self.config.model_dump().items(): # Format the display value if isinstance(field_value, str) and ( "key" in field_name.lower() @@ -458,18 +460,18 @@ class HaikuRAGApp: a2a_port: int = 8000, ): """Start the server with selected services.""" - async with HaikuRAG(self.db_path) as client: + async with HaikuRAG(self.db_path, config=self.config) as client: tasks = [] # Start file monitor if enabled if enable_monitor: - monitor = FileWatcher(client=client) + monitor = FileWatcher(client=client, config=self.config) monitor_task = asyncio.create_task(monitor.observe()) tasks.append(monitor_task) # Start MCP server if enabled if enable_mcp: - server = create_mcp_server(self.db_path) + server = create_mcp_server(self.db_path, config=self.config) async def run_mcp(): if mcp_transport == "stdio": @@ -496,15 +498,15 @@ class HaikuRAGApp: logger.info(f"Starting A2A server on {a2a_host}:{a2a_port}") async def run_a2a(): - app = create_a2a_app(db_path=self.db_path) - config = uvicorn.Config( + app = create_a2a_app(db_path=self.db_path, config=self.config) + uvicorn_config = uvicorn.Config( app, host=a2a_host, port=a2a_port, log_level="warning", access_log=False, ) - server = uvicorn.Server(config) + server = uvicorn.Server(uvicorn_config) await server.serve() a2a_task = asyncio.create_task(run_a2a()) diff --git a/haiku_rag_slim/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py index edefec59..6274cb00 100644 --- a/haiku_rag_slim/haiku/rag/cli.py +++ b/haiku_rag_slim/haiku/rag/cli.py @@ -6,24 +6,41 @@ from pathlib import Path from typing import Any import typer +from dotenv import load_dotenv -# Load environment variables from .env file before importing Config -try: - from dotenv import load_dotenv - - load_dotenv() -except ImportError: - pass - -from haiku.rag.config import Config +from haiku.rag.app import HaikuRAGApp +from haiku.rag.config import ( + AppConfig, + find_config_file, + get_config, + load_yaml_config, + set_config, +) from haiku.rag.logging import configure_cli_logging from haiku.rag.utils import is_up_to_date +# Load environment variables from .env file for API keys and service URLs +load_dotenv() + cli = typer.Typer( context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True ) +def create_app(db: Path | None = None) -> HaikuRAGApp: + """Create HaikuRAGApp with loaded config and resolved database path. + + Args: + db: Optional database path. If None, uses path from config. + + Returns: + HaikuRAGApp instance with proper config and db path. + """ + config = get_config() + db_path = db if db else config.storage.data_dir / "haiku.rag.lancedb" + return HaikuRAGApp(db_path=db_path, config=config) + + async def check_version(): """Check if haiku.rag is up to date and show warning if not.""" up_to_date, current_version, latest_version = await is_up_to_date() @@ -57,14 +74,15 @@ def main( ), ): """haiku.rag CLI - Vector database RAG system""" - # Store config path in environment for config loader to use - if config: - import os - - os.environ["HAIKU_RAG_CONFIG_PATH"] = str(config.absolute()) + # Load config from --config, local folder, or default directory + config_path = find_config_file(cli_path=config) + if config_path: + yaml_data = load_yaml_config(config_path) + loaded_config = AppConfig.model_validate(yaml_data) + set_config(loaded_config) # Configure logging minimally for CLI context - if Config.environment == "development": + if get_config().environment == "development": # Lazy import logfire only in development try: import logfire # type: ignore @@ -87,8 +105,8 @@ def main( @cli.command("list", help="List all stored documents") def list_documents( - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), @@ -99,9 +117,7 @@ def list_documents( help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.list_documents(filter=filter)) @@ -140,15 +156,13 @@ def add_document_text( help="Metadata entries as KEY=VALUE (repeatable)", metavar="KEY=VALUE", ), - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) metadata = _parse_meta_options(meta) asyncio.run(app.add_document_from_text(text=text, metadata=metadata or None)) @@ -169,15 +183,13 @@ def add_document_src( help="Metadata entries as KEY=VALUE (repeatable)", metavar="KEY=VALUE", ), - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) metadata = _parse_meta_options(meta) asyncio.run( app.add_document_from_source( @@ -191,15 +203,13 @@ def get_document( doc_id: str = typer.Argument( help="The ID of the document to get", ), - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.get_document(doc_id=doc_id)) @@ -208,15 +218,13 @@ def delete_document( doc_id: str = typer.Argument( help="The ID of the document to delete", ), - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.delete_document(doc_id=doc_id)) @@ -241,15 +249,13 @@ def search( "-f", help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")", ), - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.search(query=query, limit=limit, filter=filter)) @@ -258,8 +264,8 @@ def ask( question: str = typer.Argument( help="The question to ask", ), - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), @@ -279,9 +285,7 @@ def ask( help="Show verbose progress output (only with --deep)", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.ask(question=question, cite=cite, deep=deep, verbose=verbose)) @@ -290,8 +294,8 @@ def research( question: str = typer.Argument( help="The research question to investigate", ), - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), @@ -301,17 +305,14 @@ def research( help="Show verbose progress output", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.research(question=question, verbose=verbose)) @cli.command("settings", help="Display current configuration settings") def settings(): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=Path()) # Don't need actual DB for settings + config = get_config() + app = HaikuRAGApp(db_path=Path(), config=config) app.show_settings() @@ -321,16 +322,11 @@ def init_config( Path("haiku.rag.yaml"), help="Output path for the config file", ), - from_env: bool = typer.Option( - False, - "--from-env", - help="Migrate settings from .env file", - ), ): - """Generate a YAML configuration file with defaults or from .env.""" + """Generate a YAML configuration file with defaults.""" import yaml - from haiku.rag.config.loader import generate_default_config, load_config_from_env + from haiku.rag.config.loader import generate_default_config if output.exists(): typer.echo( @@ -338,18 +334,7 @@ def init_config( ) raise typer.Exit(1) - if from_env: - # Load from environment variables (including .env if present) - from dotenv import load_dotenv - - load_dotenv() - config_data = load_config_from_env() - if not config_data: - typer.echo("Warning: No environment variables found to migrate.") - typer.echo("Generating default configuration instead.") - config_data = generate_default_config() - else: - config_data = generate_default_config() + config_data = generate_default_config() # Write YAML with comments with open(output, "w") as f: @@ -368,43 +353,37 @@ def init_config( help="Rebuild the database by deleting all chunks and re-indexing all documents", ) def rebuild( - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.rebuild()) @cli.command("vacuum", help="Optimize and clean up all tables to reduce disk usage") def vacuum( - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.vacuum()) @cli.command("info", help="Show read-only database info (no upgrades or writes)") def info( - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), ): - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) asyncio.run(app.info()) @@ -425,8 +404,8 @@ def download_models_cmd(): help="Start haiku.rag server. Use --monitor, --mcp, and/or --a2a to enable services.", ) def serve( - db: Path = typer.Option( - Config.storage.data_dir / "haiku.rag.lancedb", + db: Path | None = typer.Option( + None, "--db", help="Path to the LanceDB database file", ), @@ -478,9 +457,7 @@ def serve( typer.echo("Error: --stdio requires --mcp") raise typer.Exit(1) - from haiku.rag.app import HaikuRAGApp - - app = HaikuRAGApp(db_path=db) + app = create_app(db) transport = "stdio" if stdio else None diff --git a/haiku_rag_slim/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py index 0a01fba7..6e9d9aed 100644 --- a/haiku_rag_slim/haiku/rag/config/__init__.py +++ b/haiku_rag_slim/haiku/rag/config/__init__.py @@ -3,7 +3,6 @@ import os from haiku.rag.config.loader import ( find_config_file, generate_default_config, - load_config_from_env, load_yaml_config, ) from haiku.rag.config.models import ( @@ -40,13 +39,36 @@ __all__ = [ "find_config_file", "load_yaml_config", "generate_default_config", - "load_config_from_env", + "get_config", + "set_config", ] -# Load config from YAML file or use defaults -config_path = find_config_file(None) -if config_path: - yaml_data = load_yaml_config(config_path) - Config = AppConfig.model_validate(yaml_data) -else: - Config = AppConfig() +# Global config instance - initially loads from default locations +_config: AppConfig | None = None + + +def _load_default_config() -> AppConfig: + """Load config from default locations (used at import time).""" + config_path = find_config_file(None) + if config_path: + yaml_data = load_yaml_config(config_path) + return AppConfig.model_validate(yaml_data) + return AppConfig() + + +def set_config(config: AppConfig) -> None: + """Set the global config instance (used by CLI to override).""" + global _config + _config = config + + +def get_config() -> AppConfig: + """Get the current config instance.""" + global _config + if _config is None: + _config = _load_default_config() + return _config + + +# Legacy compatibility - Config is the default instance +Config = _load_default_config() diff --git a/haiku_rag_slim/haiku/rag/config/loader.py b/haiku_rag_slim/haiku/rag/config/loader.py index 0438e544..4b6f9500 100644 --- a/haiku_rag_slim/haiku/rag/config/loader.py +++ b/haiku_rag_slim/haiku/rag/config/loader.py @@ -32,7 +32,8 @@ def find_config_file(cli_path: Path | None = None) -> Path | None: # Use same directory as data storage for config from haiku.rag.utils import get_default_data_dir - user_config = get_default_data_dir() / "config.yaml" + data_dir = get_default_data_dir() + user_config = data_dir / "haiku.rag.yaml" if user_config.exists(): return user_config @@ -85,59 +86,3 @@ def generate_default_config() -> dict: }, "a2a": {"max_contexts": 1000}, } - - -def load_config_from_env() -> dict: - """Load current config from environment variables (for migration).""" - result = {} - - env_mappings = { - "ENV": "environment", - "DEFAULT_DATA_DIR": ("storage", "data_dir"), - "MONITOR_DIRECTORIES": ("monitor", "directories"), - "DISABLE_DB_AUTOCREATE": ("storage", "disable_autocreate"), - "VACUUM_RETENTION_SECONDS": ("storage", "vacuum_retention_seconds"), - "LANCEDB_URI": ("lancedb", "uri"), - "LANCEDB_API_KEY": ("lancedb", "api_key"), - "LANCEDB_REGION": ("lancedb", "region"), - "EMBEDDINGS_PROVIDER": ("embeddings", "provider"), - "EMBEDDINGS_MODEL": ("embeddings", "model"), - "EMBEDDINGS_VECTOR_DIM": ("embeddings", "vector_dim"), - "RERANK_PROVIDER": ("reranking", "provider"), - "RERANK_MODEL": ("reranking", "model"), - "QA_PROVIDER": ("qa", "provider"), - "QA_MODEL": ("qa", "model"), - "RESEARCH_PROVIDER": ("research", "provider"), - "RESEARCH_MODEL": ("research", "model"), - "CHUNK_SIZE": ("processing", "chunk_size"), - "CONTEXT_CHUNK_RADIUS": ("processing", "context_chunk_radius"), - "MARKDOWN_PREPROCESSOR": ("processing", "markdown_preprocessor"), - "OLLAMA_BASE_URL": ("providers", "ollama", "base_url"), - "VLLM_EMBEDDINGS_BASE_URL": ("providers", "vllm", "embeddings_base_url"), - "VLLM_RERANK_BASE_URL": ("providers", "vllm", "rerank_base_url"), - "VLLM_QA_BASE_URL": ("providers", "vllm", "qa_base_url"), - "VLLM_RESEARCH_BASE_URL": ("providers", "vllm", "research_base_url"), - "A2A_MAX_CONTEXTS": ("a2a", "max_contexts"), - } - - for env_var, path in env_mappings.items(): - value = os.getenv(env_var) - if value is not None: - # Special handling for MONITOR_DIRECTORIES - parse comma-separated list - if env_var == "MONITOR_DIRECTORIES": - if value.strip(): - value = [p.strip() for p in value.split(",") if p.strip()] - else: - value = [] - - if isinstance(path, tuple): - current = result - for key in path[:-1]: - if key not in current: - current[key] = {} - current = current[key] - current[path[-1]] = value - else: - result[path] = value - - return result diff --git a/haiku_rag_slim/haiku/rag/mcp.py b/haiku_rag_slim/haiku/rag/mcp.py index 3c18587c..25cd2fec 100644 --- a/haiku_rag_slim/haiku/rag/mcp.py +++ b/haiku_rag_slim/haiku/rag/mcp.py @@ -5,7 +5,7 @@ from fastmcp import FastMCP from pydantic import BaseModel from haiku.rag.client import HaikuRAG -from haiku.rag.config import Config +from haiku.rag.config import AppConfig, Config from haiku.rag.research.models import ResearchReport @@ -25,7 +25,7 @@ class DocumentResult(BaseModel): updated_at: str -def create_mcp_server(db_path: Path) -> FastMCP: +def create_mcp_server(db_path: Path, config: AppConfig = Config) -> FastMCP: """Create an MCP server with the specified database path.""" mcp = FastMCP("haiku-rag") @@ -37,7 +37,7 @@ def create_mcp_server(db_path: Path) -> FastMCP: ) -> str | None: """Add a document to the RAG system from a file path.""" try: - async with HaikuRAG(db_path) as rag: + async with HaikuRAG(db_path, config=config) as rag: result = await rag.create_document_from_source( Path(file_path), title=title, metadata=metadata or {} ) @@ -54,7 +54,7 @@ def create_mcp_server(db_path: Path) -> FastMCP: ) -> str | None: """Add a document to the RAG system from a URL.""" try: - async with HaikuRAG(db_path) as rag: + async with HaikuRAG(db_path, config=config) as rag: result = await rag.create_document_from_source( url, title=title, metadata=metadata or {} ) @@ -74,7 +74,7 @@ def create_mcp_server(db_path: Path) -> FastMCP: ) -> str | None: """Add a document to the RAG system from text content.""" try: - async with HaikuRAG(db_path) as rag: + async with HaikuRAG(db_path, config=config) as rag: document = await rag.create_document( content, uri, title=title, metadata=metadata or {} ) @@ -86,7 +86,7 @@ def create_mcp_server(db_path: Path) -> FastMCP: async def search_documents(query: str, limit: int = 5) -> list[SearchResult]: """Search the RAG system for documents using hybrid search (vector similarity + full-text search).""" try: - async with HaikuRAG(db_path) as rag: + async with HaikuRAG(db_path, config=config) as rag: results = await rag.search(query, limit) search_results = [] @@ -110,7 +110,7 @@ def create_mcp_server(db_path: Path) -> FastMCP: async def get_document(document_id: str) -> DocumentResult | None: """Get a document by its ID.""" try: - async with HaikuRAG(db_path) as rag: + async with HaikuRAG(db_path, config=config) as rag: document = await rag.get_document_by_id(document_id) if document is None: @@ -145,7 +145,7 @@ def create_mcp_server(db_path: Path) -> FastMCP: List of DocumentResult instances matching the criteria. """ try: - async with HaikuRAG(db_path) as rag: + async with HaikuRAG(db_path, config=config) as rag: documents = await rag.list_documents(limit, offset, filter) return [ @@ -167,7 +167,7 @@ def create_mcp_server(db_path: Path) -> FastMCP: async def delete_document(document_id: str) -> bool: """Delete a document by its ID.""" try: - async with HaikuRAG(db_path) as rag: + async with HaikuRAG(db_path, config=config) as rag: return await rag.delete_document(document_id) except Exception: return False @@ -189,18 +189,17 @@ def create_mcp_server(db_path: Path) -> FastMCP: The answer as a string. """ try: - async with HaikuRAG(db_path) as rag: + async with HaikuRAG(db_path, config=config) as rag: if deep: - from haiku.rag.config import Config from haiku.rag.qa.deep.dependencies import DeepQAContext from haiku.rag.qa.deep.graph import build_deep_qa_graph from haiku.rag.qa.deep.state import DeepQADeps, DeepQAState - graph = build_deep_qa_graph(config=Config) + graph = build_deep_qa_graph(config=config) context = DeepQAContext( original_question=question, use_citations=cite ) - state = DeepQAState.from_config(context=context, config=Config) + state = DeepQAState.from_config(context=context, config=config) deps = DeepQADeps(client=rag) result = await graph.run(state=state, deps=deps) @@ -231,10 +230,10 @@ def create_mcp_server(db_path: Path) -> FastMCP: from haiku.rag.research.graph import build_research_graph from haiku.rag.research.state import ResearchDeps, ResearchState - async with HaikuRAG(db_path) as rag: - graph = build_research_graph(config=Config) + async with HaikuRAG(db_path, config=config) as rag: + graph = build_research_graph(config=config) context = ResearchContext(original_question=question) - state = ResearchState.from_config(context=context, config=Config) + state = ResearchState.from_config(context=context, config=config) deps = ResearchDeps(client=rag) result = await graph.run(state=state, deps=deps) diff --git a/tests/test_cli.py b/tests/test_cli.py index f4e73fde..c936e3ad 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,7 +8,7 @@ runner = CliRunner() def test_list_documents(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.list_documents = AsyncMock() mock_app.return_value = mock_app_instance @@ -20,7 +20,7 @@ def test_list_documents(): def test_add_document_text(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.add_document_from_text = AsyncMock() mock_app.return_value = mock_app_instance @@ -35,7 +35,7 @@ def test_add_document_text(): def test_add_document_src(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.add_document_from_source = AsyncMock() mock_app.return_value = mock_app_instance @@ -47,7 +47,7 @@ def test_add_document_src(): def test_add_document_src_with_title(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.add_document_from_source = AsyncMock() mock_app.return_value = mock_app_instance @@ -63,7 +63,7 @@ def test_add_document_src_with_title(): def test_add_document_text_with_meta(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.add_document_from_text = AsyncMock() mock_app.return_value = mock_app_instance @@ -88,7 +88,7 @@ def test_add_document_text_with_meta(): def test_add_document_src_with_meta(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.add_document_from_source = AsyncMock() mock_app.return_value = mock_app_instance @@ -113,7 +113,7 @@ def test_add_document_src_with_meta(): def test_add_document_text_with_numeric_meta(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.add_document_from_text = AsyncMock() mock_app.return_value = mock_app_instance @@ -138,7 +138,7 @@ def test_add_document_text_with_numeric_meta(): def test_get_document(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.get_document = AsyncMock() mock_app.return_value = mock_app_instance @@ -150,7 +150,7 @@ def test_get_document(): def test_delete_document(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.delete_document = AsyncMock() mock_app.return_value = mock_app_instance @@ -162,7 +162,7 @@ def test_delete_document(): def test_search(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.search = AsyncMock() mock_app.return_value = mock_app_instance @@ -184,7 +184,7 @@ def test_serve_no_flags(): def test_serve_mcp_only(): """Test serve command with MCP only.""" - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.serve = AsyncMock() mock_app.return_value = mock_app_instance @@ -203,7 +203,7 @@ def test_serve_mcp_only(): def test_serve_mcp_stdio(): """Test serve command with MCP stdio transport.""" - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.serve = AsyncMock() mock_app.return_value = mock_app_instance @@ -218,7 +218,7 @@ def test_serve_mcp_stdio(): def test_serve_monitor_only(): """Test serve command with monitor only.""" - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.serve = AsyncMock() mock_app.return_value = mock_app_instance @@ -235,7 +235,7 @@ def test_serve_monitor_only(): def test_serve_a2a_only(): """Test serve command with A2A only.""" - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.serve = AsyncMock() mock_app.return_value = mock_app_instance @@ -254,7 +254,7 @@ def test_serve_a2a_only(): def test_serve_all_services(): """Test serve command with all services.""" - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.serve = AsyncMock() mock_app.return_value = mock_app_instance @@ -271,7 +271,7 @@ def test_serve_all_services(): def test_serve_custom_ports(): """Test serve command with custom ports.""" - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.serve = AsyncMock() mock_app.return_value = mock_app_instance @@ -295,7 +295,7 @@ def test_serve_stdio_without_mcp(): def test_ask(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.ask = AsyncMock() mock_app.return_value = mock_app_instance @@ -309,7 +309,7 @@ def test_ask(): def test_ask_with_cite(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.ask = AsyncMock() mock_app.return_value = mock_app_instance @@ -323,7 +323,7 @@ def test_ask_with_cite(): def test_ask_with_deep(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.ask = AsyncMock() mock_app.return_value = mock_app_instance @@ -337,7 +337,7 @@ def test_ask_with_deep(): def test_ask_with_deep_and_cite(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.ask = AsyncMock() mock_app.return_value = mock_app_instance @@ -351,7 +351,7 @@ def test_ask_with_deep_and_cite(): def test_ask_with_deep_and_verbose(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.ask = AsyncMock() mock_app.return_value = mock_app_instance @@ -365,7 +365,7 @@ def test_ask_with_deep_and_verbose(): def test_info(): - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.info = AsyncMock() mock_app.return_value = mock_app_instance @@ -378,7 +378,7 @@ def test_info(): def test_add_document_src_directory(tmp_path): """Test adding documents from a directory recursively.""" - with patch("haiku.rag.app.HaikuRAGApp") as mock_app: + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() mock_app_instance.add_document_from_source = AsyncMock() mock_app.return_value = mock_app_instance diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py index 1063dab8..62611f5f 100644 --- a/tests/test_config_loader.py +++ b/tests/test_config_loader.py @@ -1,11 +1,8 @@ -import os - import pytest from haiku.rag.config.loader import ( find_config_file, generate_default_config, - load_config_from_env, load_yaml_config, ) @@ -50,7 +47,7 @@ def test_find_config_file_user_config(tmp_path, monkeypatch): "haiku.rag.utils.get_default_data_dir", mock_get_default_data_dir ) - config_file = tmp_path / "config.yaml" + config_file = tmp_path / "haiku.rag.yaml" config_file.write_text("environment: production") found = find_config_file() @@ -114,68 +111,29 @@ def test_generate_default_config(): assert config["embeddings"]["vector_dim"] == 4096 -def test_load_config_from_env(monkeypatch): - """Test loading config from environment variables.""" - monkeypatch.setenv("ENV", "development") - monkeypatch.setenv("EMBEDDINGS_PROVIDER", "openai") - monkeypatch.setenv("EMBEDDINGS_MODEL", "text-embedding-3-small") - monkeypatch.setenv("EMBEDDINGS_VECTOR_DIM", "1536") - monkeypatch.setenv("QA_PROVIDER", "anthropic") - monkeypatch.setenv("QA_MODEL", "claude-3-haiku") - - config = load_config_from_env() - - assert config["environment"] == "development" - assert config["embeddings"]["provider"] == "openai" - assert config["embeddings"]["model"] == "text-embedding-3-small" - assert config["embeddings"]["vector_dim"] == "1536" - assert config["qa"]["provider"] == "anthropic" - assert config["qa"]["model"] == "claude-3-haiku" - - -def test_load_config_from_env_empty(): - """Test loading from env when no relevant vars set.""" - # Clear any env vars that might be set - env_vars = [ - "ENV", - "EMBEDDINGS_PROVIDER", - "QA_PROVIDER", - "OPENAI_API_KEY", - ] - original_values = {} - for var in env_vars: - original_values[var] = os.environ.get(var) - if var in os.environ: - del os.environ[var] - - try: - config = load_config_from_env() - # Should return empty or minimal dict - assert isinstance(config, dict) - finally: - # Restore original values - for var, value in original_values.items(): - if value is not None: - os.environ[var] = value - - def test_config_precedence_cwd_over_user(tmp_path, monkeypatch): """Test that cwd config takes precedence over user config.""" - monkeypatch.chdir(tmp_path) + # Create separate directories for cwd and user config + cwd_dir = tmp_path / "cwd" + cwd_dir.mkdir() + user_dir = tmp_path / "user" + user_dir.mkdir() - # Mock get_default_data_dir to return tmp_path + monkeypatch.chdir(cwd_dir) + + # Mock get_default_data_dir to return user_dir def mock_get_default_data_dir(): - return tmp_path + return user_dir monkeypatch.setattr( "haiku.rag.utils.get_default_data_dir", mock_get_default_data_dir ) # Create both configs - cwd_config = tmp_path / "haiku.rag.yaml" + cwd_config = cwd_dir / "haiku.rag.yaml" cwd_config.write_text("environment: from-cwd") - user_config = tmp_path / "config.yaml" + user_config = user_dir / "haiku.rag.yaml" user_config.write_text("environment: from-user") found = find_config_file()