diff --git a/CHANGELOG.md b/CHANGELOG.md index 722f211b..5607fae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - `ingester.api.root_path` (and `haiku-ingester serve --root-path`) serves the control plane under a sub-path for reverse-proxying; forwarded to FastAPI/uvicorn `root_path` and reflected in the dashboard's ``. +- YAML config string values support `${VAR}` / `${VAR:-default}` environment-variable interpolation, expanded at load time. `${VAR}` referencing an unset variable raises `MissingEnvVarError`; `$$` is a literal `$`. ## [0.56.0] - 2026-06-09 diff --git a/docs/configuration/index.md b/docs/configuration/index.md index a18bca85..a05de0a6 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -28,6 +28,22 @@ This creates a `haiku.rag.yaml` file in your current directory with all availabl - **macOS**: `~/Library/Application Support/haiku.rag/haiku.rag.yaml` - **Windows**: `C:/Users//AppData/Roaming/haiku.rag/haiku.rag.yaml` +## Environment Variables + +Any string value can reference an environment variable, so secrets stay out of the file and one config can serve multiple deployments: + +```yaml +ingester: + queue: + dburi: postgresql+asyncpg://haiku:${POSTGRES_PASSWORD}@db:5432/haiku_rag +``` + +- `${VAR}` is replaced with the value of `VAR`. If `VAR` is unset, loading fails with an error naming the variable. +- `${VAR:-default}` uses `default` when `VAR` is unset or empty. +- `$$` produces a literal `$`. + +Substitution happens after the YAML is parsed, so a value containing `:`, `@`, or `#` fills the string verbatim and never changes the document structure. + ## Minimal Configuration A minimal configuration file with defaults: diff --git a/haiku_rag_slim/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py index a8384995..678ff4bb 100644 --- a/haiku_rag_slim/haiku/rag/config/__init__.py +++ b/haiku_rag_slim/haiku/rag/config/__init__.py @@ -1,4 +1,6 @@ from haiku.rag.config.loader import ( + MissingEnvVarError, + expand_env_vars, find_config_file, generate_default_config, load_yaml_config, @@ -57,6 +59,8 @@ __all__ = [ "StorageConfig", "WebDAVSourceConfig", "WorkerConfig", + "MissingEnvVarError", + "expand_env_vars", "find_config_file", "generate_default_config", "get_config", diff --git a/haiku_rag_slim/haiku/rag/config/loader.py b/haiku_rag_slim/haiku/rag/config/loader.py index f899fa02..abc9892b 100644 --- a/haiku_rag_slim/haiku/rag/config/loader.py +++ b/haiku_rag_slim/haiku/rag/config/loader.py @@ -1,5 +1,6 @@ import logging import os +import re from pathlib import Path from typing import Any @@ -7,6 +8,41 @@ import yaml logger = logging.getLogger(__name__) +_ENV_VAR_PATTERN = re.compile(r"\$\$|\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}") + + +class MissingEnvVarError(ValueError): + """A ${VAR} in the config references an unset environment variable.""" + + +def _expand_str(value: str) -> str: + def replace(match: re.Match[str]) -> str: + if match.group(0) == "$$": + return "$" + name, default = match.group(1), match.group(2) + if name in os.environ and (default is None or os.environ[name] != ""): + return os.environ[name] + if default is not None: + return default + raise MissingEnvVarError( + f"Config references unset environment variable ${{{name}}}. " + f"Set it, or use ${{{name}:-default}} to provide a fallback." + ) + + return _ENV_VAR_PATTERN.sub(replace, value) + + +def expand_env_vars(data: Any) -> Any: + """Recursively expand ${VAR} / ${VAR:-default} references in string values. + Keys and non-string scalars are left untouched; $$ collapses to a literal $.""" + if isinstance(data, dict): + return {key: expand_env_vars(value) for key, value in data.items()} + if isinstance(data, list): + return [expand_env_vars(item) for item in data] + if isinstance(data, str): + return _expand_str(data) + return data + def find_config_file(cli_path: Path | None = None) -> Path | None: """Find the YAML config file using the search path. @@ -45,10 +81,10 @@ def find_config_file(cli_path: Path | None = None) -> Path | None: def load_yaml_config(path: Path) -> dict: - """Load and parse a YAML config file.""" + """Load and parse a YAML config file, expanding ${VAR} references.""" with open(path) as f: data = yaml.safe_load(f) - return data or {} + return expand_env_vars(data or {}) def generate_default_config() -> dict: diff --git a/tests/test_config.py b/tests/test_config.py index 05c3ebc1..fac05ad0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -374,3 +374,135 @@ def test_redact_secrets_masks_nested_secret_keys(): assert redacted["sources"][0]["password"] == "***" assert redacted["sources"][0]["url"] == "http://x" assert redacted["storage_options"]["aws_secret_access_key"] == "***" + + +def test_expand_env_var_set(tmp_path, monkeypatch): + """A ${VAR} referencing a set variable is substituted.""" + monkeypatch.setenv("HAIKU_TEST_MODEL", "my-model") + config_file = tmp_path / "test.yaml" + config_file.write_text(""" +embeddings: + model: + name: ${HAIKU_TEST_MODEL} +""") + + config = load_yaml_config(config_file) + assert config["embeddings"]["model"]["name"] == "my-model" + + +def test_expand_env_var_dburi_preserves_special_chars(tmp_path, monkeypatch): + """A password containing : and @ fills the string without breaking the URL.""" + monkeypatch.setenv("HAIKU_TEST_PGPW", "p@ss:word") + config_file = tmp_path / "test.yaml" + config_file.write_text(""" +ingester: + queue: + dburi: "postgresql+asyncpg://user:${HAIKU_TEST_PGPW}@host/db" +""") + + config = load_yaml_config(config_file) + assert ( + config["ingester"]["queue"]["dburi"] + == "postgresql+asyncpg://user:p@ss:word@host/db" + ) + + +def test_expand_env_var_unset_raises(tmp_path, monkeypatch): + """An unset ${VAR} without a default raises, naming the variable.""" + from haiku.rag.config.loader import MissingEnvVarError + + monkeypatch.delenv("HAIKU_TEST_MISSING", raising=False) + config_file = tmp_path / "test.yaml" + config_file.write_text("environment: ${HAIKU_TEST_MISSING}") + + with pytest.raises(MissingEnvVarError, match="HAIKU_TEST_MISSING"): + load_yaml_config(config_file) + + +def test_expand_env_var_default_when_unset(tmp_path, monkeypatch): + """${VAR:-default} falls back to the default when VAR is unset.""" + monkeypatch.delenv("HAIKU_TEST_MISSING", raising=False) + config_file = tmp_path / "test.yaml" + config_file.write_text("environment: ${HAIKU_TEST_MISSING:-production}") + + config = load_yaml_config(config_file) + assert config["environment"] == "production" + + +def test_expand_env_var_default_when_empty(tmp_path, monkeypatch): + """${VAR:-default} falls back to the default when VAR is set but empty.""" + monkeypatch.setenv("HAIKU_TEST_EMPTY", "") + config_file = tmp_path / "test.yaml" + config_file.write_text("environment: ${HAIKU_TEST_EMPTY:-production}") + + config = load_yaml_config(config_file) + assert config["environment"] == "production" + + +def test_expand_env_var_default_overridden_when_set(tmp_path, monkeypatch): + """${VAR:-default} uses the variable when it is set and non-empty.""" + monkeypatch.setenv("HAIKU_TEST_ENV", "staging") + config_file = tmp_path / "test.yaml" + config_file.write_text("environment: ${HAIKU_TEST_ENV:-production}") + + config = load_yaml_config(config_file) + assert config["environment"] == "staging" + + +def test_expand_env_var_dollar_escape(tmp_path): + """$$ collapses to a literal $, leaving ${...} text intact.""" + config_file = tmp_path / "test.yaml" + config_file.write_text("environment: $${NOT_A_VAR}") + + config = load_yaml_config(config_file) + assert config["environment"] == "${NOT_A_VAR}" + + +def test_expand_env_var_nested_in_list_and_dict(tmp_path, monkeypatch): + """Expansion recurses through lists and nested dicts.""" + monkeypatch.setenv("HAIKU_TEST_TOKEN", "abc123") + monkeypatch.setenv("HAIKU_TEST_KEY", "AKIA") + config_file = tmp_path / "test.yaml" + config_file.write_text(""" +ingester: + sources: + - type: http + id: arxiv + urls: + - https://example.com/${HAIKU_TEST_TOKEN}.pdf + storage_options: + aws_access_key_id: ${HAIKU_TEST_KEY} +""") + + config = load_yaml_config(config_file) + source = config["ingester"]["sources"][0] + assert source["urls"][0] == "https://example.com/abc123.pdf" + assert source["storage_options"]["aws_access_key_id"] == "AKIA" + + +def test_expand_env_var_leaves_non_strings_untouched(tmp_path): + """Non-string scalars pass through unchanged.""" + config_file = tmp_path / "test.yaml" + config_file.write_text(""" +embeddings: + model: + vector_dim: 1024 +ingester: + sources: + - type: s3 + storage_options: + allow_http: true +""") + + config = load_yaml_config(config_file) + assert config["embeddings"]["model"]["vector_dim"] == 1024 + assert config["ingester"]["sources"][0]["storage_options"]["allow_http"] is True + + +def test_expand_env_var_plain_string_unchanged(tmp_path): + """A string with no ${...} reference is returned as-is.""" + config_file = tmp_path / "test.yaml" + config_file.write_text("environment: production") + + config = load_yaml_config(config_file) + assert config["environment"] == "production"