Merge pull request #434 from ggozad/feat/secrets-from-env

Expand ${VAR} environment references in YAML config

CHANGELOG.md

@@ -4,6 +4,7 @@
 ### Added
 
 - `ingester.api.root_path` (and `haiku-ingester serve --root-path`) serves the control plane under a sub-path for reverse-proxying; forwarded to FastAPI/uvicorn `root_path` and reflected in the dashboard's `<base href>`.
+- YAML config string values support `${VAR}` / `${VAR:-default}` environment-variable interpolation, expanded at load time. `${VAR}` referencing an unset variable raises `MissingEnvVarError`; `$$` is a literal `$`.
 
 ## [0.56.0] - 2026-06-09
 

docs/configuration/index.md

@@ -28,6 +28,22 @@ This creates a `haiku.rag.yaml` file in your current directory with all availabl
     - **macOS**: `~/Library/Application Support/haiku.rag/haiku.rag.yaml`
     - **Windows**: `C:/Users/<USER>/AppData/Roaming/haiku.rag/haiku.rag.yaml`
 
+## Environment Variables
+
+Any string value can reference an environment variable, so secrets stay out of the file and one config can serve multiple deployments:
+
+```yaml
+ingester:
+  queue:
+    dburi: postgresql+asyncpg://haiku:${POSTGRES_PASSWORD}@db:5432/haiku_rag
+```
+
+- `${VAR}` is replaced with the value of `VAR`. If `VAR` is unset, loading fails with an error naming the variable.
+- `${VAR:-default}` uses `default` when `VAR` is unset or empty.
+- `$$` produces a literal `$`.
+
+Substitution happens after the YAML is parsed, so a value containing `:`, `@`, or `#` fills the string verbatim and never changes the document structure.
+
 ## Minimal Configuration
 
 A minimal configuration file with defaults:

haiku_rag_slim/haiku/rag/config/__init__.py

@@ -1,4 +1,6 @@
 from haiku.rag.config.loader import (
+    MissingEnvVarError,
+    expand_env_vars,
     find_config_file,
     generate_default_config,
     load_yaml_config,
@@ -57,6 +59,8 @@ __all__ = [
     "StorageConfig",
     "WebDAVSourceConfig",
     "WorkerConfig",
+    "MissingEnvVarError",
+    "expand_env_vars",
     "find_config_file",
     "generate_default_config",
     "get_config",

haiku_rag_slim/haiku/rag/config/loader.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import re
 from pathlib import Path
 from typing import Any
 
@@ -7,6 +8,41 @@ import yaml
 
 logger = logging.getLogger(__name__)
 
+_ENV_VAR_PATTERN = re.compile(r"\$\$|\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
+
+
+class MissingEnvVarError(ValueError):
+    """A ${VAR} in the config references an unset environment variable."""
+
+
+def _expand_str(value: str) -> str:
+    def replace(match: re.Match[str]) -> str:
+        if match.group(0) == "$$":
+            return "$"
+        name, default = match.group(1), match.group(2)
+        if name in os.environ and (default is None or os.environ[name] != ""):
+            return os.environ[name]
+        if default is not None:
+            return default
+        raise MissingEnvVarError(
+            f"Config references unset environment variable ${{{name}}}. "
+            f"Set it, or use ${{{name}:-default}} to provide a fallback."
+        )
+
+    return _ENV_VAR_PATTERN.sub(replace, value)
+
+
+def expand_env_vars(data: Any) -> Any:
+    """Recursively expand ${VAR} / ${VAR:-default} references in string values.
+    Keys and non-string scalars are left untouched; $$ collapses to a literal $."""
+    if isinstance(data, dict):
+        return {key: expand_env_vars(value) for key, value in data.items()}
+    if isinstance(data, list):
+        return [expand_env_vars(item) for item in data]
+    if isinstance(data, str):
+        return _expand_str(data)
+    return data
+
 
 def find_config_file(cli_path: Path | None = None) -> Path | None:
     """Find the YAML config file using the search path.
@@ -45,10 +81,10 @@ def find_config_file(cli_path: Path | None = None) -> Path | None:
 
 
 def load_yaml_config(path: Path) -> dict:
-    """Load and parse a YAML config file."""
+    """Load and parse a YAML config file, expanding ${VAR} references."""
     with open(path) as f:
         data = yaml.safe_load(f)
-    return data or {}
+    return expand_env_vars(data or {})
 
 
 def generate_default_config() -> dict:

tests/test_config.py

@@ -374,3 +374,135 @@ def test_redact_secrets_masks_nested_secret_keys():
     assert redacted["sources"][0]["password"] == "***"
     assert redacted["sources"][0]["url"] == "http://x"
     assert redacted["storage_options"]["aws_secret_access_key"] == "***"
+
+
+def test_expand_env_var_set(tmp_path, monkeypatch):
+    """A ${VAR} referencing a set variable is substituted."""
+    monkeypatch.setenv("HAIKU_TEST_MODEL", "my-model")
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("""
+embeddings:
+  model:
+    name: ${HAIKU_TEST_MODEL}
+""")
+
+    config = load_yaml_config(config_file)
+    assert config["embeddings"]["model"]["name"] == "my-model"
+
+
+def test_expand_env_var_dburi_preserves_special_chars(tmp_path, monkeypatch):
+    """A password containing : and @ fills the string without breaking the URL."""
+    monkeypatch.setenv("HAIKU_TEST_PGPW", "p@ss:word")
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("""
+ingester:
+  queue:
+    dburi: "postgresql+asyncpg://user:${HAIKU_TEST_PGPW}@host/db"
+""")
+
+    config = load_yaml_config(config_file)
+    assert (
+        config["ingester"]["queue"]["dburi"]
+        == "postgresql+asyncpg://user:p@ss:word@host/db"
+    )
+
+
+def test_expand_env_var_unset_raises(tmp_path, monkeypatch):
+    """An unset ${VAR} without a default raises, naming the variable."""
+    from haiku.rag.config.loader import MissingEnvVarError
+
+    monkeypatch.delenv("HAIKU_TEST_MISSING", raising=False)
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("environment: ${HAIKU_TEST_MISSING}")
+
+    with pytest.raises(MissingEnvVarError, match="HAIKU_TEST_MISSING"):
+        load_yaml_config(config_file)
+
+
+def test_expand_env_var_default_when_unset(tmp_path, monkeypatch):
+    """${VAR:-default} falls back to the default when VAR is unset."""
+    monkeypatch.delenv("HAIKU_TEST_MISSING", raising=False)
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("environment: ${HAIKU_TEST_MISSING:-production}")
+
+    config = load_yaml_config(config_file)
+    assert config["environment"] == "production"
+
+
+def test_expand_env_var_default_when_empty(tmp_path, monkeypatch):
+    """${VAR:-default} falls back to the default when VAR is set but empty."""
+    monkeypatch.setenv("HAIKU_TEST_EMPTY", "")
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("environment: ${HAIKU_TEST_EMPTY:-production}")
+
+    config = load_yaml_config(config_file)
+    assert config["environment"] == "production"
+
+
+def test_expand_env_var_default_overridden_when_set(tmp_path, monkeypatch):
+    """${VAR:-default} uses the variable when it is set and non-empty."""
+    monkeypatch.setenv("HAIKU_TEST_ENV", "staging")
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("environment: ${HAIKU_TEST_ENV:-production}")
+
+    config = load_yaml_config(config_file)
+    assert config["environment"] == "staging"
+
+
+def test_expand_env_var_dollar_escape(tmp_path):
+    """$$ collapses to a literal $, leaving ${...} text intact."""
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("environment: $${NOT_A_VAR}")
+
+    config = load_yaml_config(config_file)
+    assert config["environment"] == "${NOT_A_VAR}"
+
+
+def test_expand_env_var_nested_in_list_and_dict(tmp_path, monkeypatch):
+    """Expansion recurses through lists and nested dicts."""
+    monkeypatch.setenv("HAIKU_TEST_TOKEN", "abc123")
+    monkeypatch.setenv("HAIKU_TEST_KEY", "AKIA")
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("""
+ingester:
+  sources:
+    - type: http
+      id: arxiv
+      urls:
+        - https://example.com/${HAIKU_TEST_TOKEN}.pdf
+      storage_options:
+        aws_access_key_id: ${HAIKU_TEST_KEY}
+""")
+
+    config = load_yaml_config(config_file)
+    source = config["ingester"]["sources"][0]
+    assert source["urls"][0] == "https://example.com/abc123.pdf"
+    assert source["storage_options"]["aws_access_key_id"] == "AKIA"
+
+
+def test_expand_env_var_leaves_non_strings_untouched(tmp_path):
+    """Non-string scalars pass through unchanged."""
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("""
+embeddings:
+  model:
+    vector_dim: 1024
+ingester:
+  sources:
+    - type: s3
+      storage_options:
+        allow_http: true
+""")
+
+    config = load_yaml_config(config_file)
+    assert config["embeddings"]["model"]["vector_dim"] == 1024
+    assert config["ingester"]["sources"][0]["storage_options"]["allow_http"] is True
+
+
+def test_expand_env_var_plain_string_unchanged(tmp_path):
+    """A string with no ${...} reference is returned as-is."""
+    config_file = tmp_path / "test.yaml"
+    config_file.write_text("environment: production")
+
+    config = load_yaml_config(config_file)
+    assert config["environment"] == "production"