diff --git a/haiku_rag_slim/haiku/rag/client/documents.py b/haiku_rag_slim/haiku/rag/client/documents.py index cbcb5fa0..8f052caa 100644 --- a/haiku_rag_slim/haiku/rag/client/documents.py +++ b/haiku_rag_slim/haiku/rag/client/documents.py @@ -15,11 +15,6 @@ from haiku.rag.client.processing import ( ) from haiku.rag.client.titles import resolve_title from haiku.rag.converters import get_converter -from haiku.rag.ingester.sources import ( - FetchResult, - resolve_adhoc_fetcher, - resolve_configured_source, -) from haiku.rag.store.models.chunk import Chunk from haiku.rag.store.models.document import Document from haiku.rag.store.models.document_item import extract_items @@ -29,7 +24,7 @@ if TYPE_CHECKING: from docling_core.types.doc.document import DoclingDocument from haiku.rag.client import HaikuRAG - from haiku.rag.ingester.sources.base import Source + from haiku.rag.ingester.sources.base import FetchResult, Source logger = logging.getLogger(__name__) @@ -330,7 +325,7 @@ async def _refresh_doc_metadata( async def _ingest_fetch_result( client: "HaikuRAG", - result: FetchResult, + result: "FetchResult", *, title: str | None, user_metadata: dict, @@ -495,6 +490,8 @@ async def _reconcile_pdf_attachments( ): continue + from haiku.rag.ingester.sources.base import FetchResult + child_fr = FetchResult( uri=child_uri, body=data, @@ -601,6 +598,11 @@ async def create_document_from_source( # renamed/removed source surfaces as a DLQ instead of silently dropping # credentials. Ad-hoc CLI calls (no source_id) fall back to scheme-based # adapters when no configured source matches. + from haiku.rag.ingester.sources import ( + resolve_adhoc_fetcher, + resolve_configured_source, + ) + if source_id is not None: fetcher = resolve_configured_source(source_str, source_id, sources) else: diff --git a/haiku_rag_slim/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py index 0d98b571..a8384995 100644 --- a/haiku_rag_slim/haiku/rag/config/__init__.py +++ b/haiku_rag_slim/haiku/rag/config/__init__.py @@ -2,6 +2,7 @@ from haiku.rag.config.loader import ( find_config_file, generate_default_config, load_yaml_config, + redact_secrets, ) from haiku.rag.config.models import ( APIConfig, @@ -60,6 +61,7 @@ __all__ = [ "generate_default_config", "get_config", "load_yaml_config", + "redact_secrets", "set_config", ] diff --git a/haiku_rag_slim/haiku/rag/config/loader.py b/haiku_rag_slim/haiku/rag/config/loader.py index bf493bdb..f899fa02 100644 --- a/haiku_rag_slim/haiku/rag/config/loader.py +++ b/haiku_rag_slim/haiku/rag/config/loader.py @@ -1,6 +1,7 @@ import logging import os from pathlib import Path +from typing import Any import yaml @@ -56,3 +57,32 @@ def generate_default_config() -> dict: default_config = AppConfig() return default_config.model_dump(mode="json", exclude_none=False) + + +_SECRET_KEY_HINTS = ("key", "password", "token", "secret") + + +def _is_secret_key(key: str) -> bool: + lowered = key.lower() + return any(hint in lowered for hint in _SECRET_KEY_HINTS) + + +def redact_secrets(data: Any) -> Any: + """Recursively mask secret-bearing values in a config dump. Any scalar + whose key contains key/password/token/secret becomes "***" when set or + None when unset; everything else is preserved.""" + if isinstance(data, dict): + result = {} + for key, value in data.items(): + if ( + isinstance(key, str) + and _is_secret_key(key) + and not isinstance(value, (dict, list)) + ): + result[key] = "***" if value else None + else: + result[key] = redact_secrets(value) + return result + if isinstance(data, list): + return [redact_secrets(item) for item in data] + return data diff --git a/haiku_rag_slim/haiku/rag/ingester/api/routes/config.py b/haiku_rag_slim/haiku/rag/ingester/api/routes/config.py new file mode 100644 index 00000000..0d8c4079 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/api/routes/config.py @@ -0,0 +1,17 @@ +import yaml +from fastapi import APIRouter, Depends + +from haiku.rag.config import redact_secrets +from haiku.rag.ingester.api.schemas import ConfigResponse +from haiku.rag.ingester.api.server import APIState, get_state + +router = APIRouter(tags=["config"]) + + +@router.get("/config", response_model=ConfigResponse) +async def config(state: APIState = Depends(get_state)) -> ConfigResponse: + """The full effective configuration (defaults filled in, not the on-disk + file) as YAML, with secret-bearing values redacted.""" + data = redact_secrets(state.config.model_dump(mode="json", exclude_none=False)) + text = yaml.dump(data, sort_keys=False, default_flow_style=False) + return ConfigResponse(yaml=text) diff --git a/haiku_rag_slim/haiku/rag/ingester/api/routes/database.py b/haiku_rag_slim/haiku/rag/ingester/api/routes/database.py new file mode 100644 index 00000000..886f742b --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/api/routes/database.py @@ -0,0 +1,22 @@ +from fastapi import APIRouter, Depends, HTTPException, status + +from haiku.rag.ingester.api.server import APIState, get_state +from haiku.rag.store.engine import DatabaseInfo, gather_database_info + +router = APIRouter(tags=["database"]) + + +@router.get("/database", response_model=DatabaseInfo) +async def database(state: APIState = Depends(get_state)) -> DatabaseInfo: + """Read-only snapshot of the LanceDB target — stored version, embeddings, + per-table counts/sizes, vector index status, pending migrations and + package versions. The same data the `haiku-rag info` command prints. + + Opens a fresh read-only connection per call; not cached and not part of + the dashboard's polling loop.""" + if state.db_path is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="database path not configured", + ) + return await gather_database_info(state.config, state.db_path) diff --git a/haiku_rag_slim/haiku/rag/ingester/api/schemas.py b/haiku_rag_slim/haiku/rag/ingester/api/schemas.py index 9833923d..fcbccefc 100644 --- a/haiku_rag_slim/haiku/rag/ingester/api/schemas.py +++ b/haiku_rag_slim/haiku/rag/ingester/api/schemas.py @@ -70,6 +70,13 @@ class WorkerStats(BaseModel): total: int +class ConfigResponse(BaseModel): + """The full effective config (defaults filled in) as YAML, with secrets + redacted.""" + + yaml: str + + class StatsResponse(BaseModel): """Aggregated counters and per-source breakdowns that drive the dashboard. Cheap to compute (all SQL aggregations against the queue file).""" diff --git a/haiku_rag_slim/haiku/rag/ingester/api/server.py b/haiku_rag_slim/haiku/rag/ingester/api/server.py index 2ef34bf0..5985f691 100644 --- a/haiku_rag_slim/haiku/rag/ingester/api/server.py +++ b/haiku_rag_slim/haiku/rag/ingester/api/server.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING from fastapi import Depends, FastAPI, Request @@ -22,6 +23,7 @@ class APIState: sync_repo: "SyncStateRepo" pool: "WorkerPool | None" = None pollers: "PollerManager | None" = None + db_path: Path | None = None def get_state(request: Request) -> APIState: @@ -34,8 +36,12 @@ def build_app( auth_token: str | None = None, ) -> FastAPI: """Construct the ingester's FastAPI control plane.""" + from haiku.rag.ingester.api.routes import ( + config as config_route, + ) from haiku.rag.ingester.api.routes import ( dashboard, + database, dlq, health, jobs, @@ -63,5 +69,7 @@ def build_app( app.include_router(dlq.router, dependencies=auth_dep) app.include_router(stats.router, dependencies=auth_dep) app.include_router(providers.router, dependencies=auth_dep) + app.include_router(database.router, dependencies=auth_dep) + app.include_router(config_route.router, dependencies=auth_dep) return app diff --git a/haiku_rag_slim/haiku/rag/ingester/app.py b/haiku_rag_slim/haiku/rag/ingester/app.py index 18d6ad39..518934da 100644 --- a/haiku_rag_slim/haiku/rag/ingester/app.py +++ b/haiku_rag_slim/haiku/rag/ingester/app.py @@ -244,6 +244,7 @@ class IngesterApp: sync_repo=self._sync, pool=self._pool, pollers=self._pollers, + db_path=self._db_path, ) if ingester_cfg.api.auth_token is None: logger.warning("API auth_token is unset — control plane is unauthenticated") diff --git a/tests/ingester/test_api.py b/tests/ingester/test_api.py index d266d8b4..634db285 100644 --- a/tests/ingester/test_api.py +++ b/tests/ingester/test_api.py @@ -633,3 +633,119 @@ async def test_dashboard_served_unauthenticated(state): assert "/jobs?status=claimed" in body # Op badge helper is present so DELETE rows render distinctly. assert "opBadge" in body + + +# --- config --- + + +@pytest.mark.asyncio +async def test_config_returns_full_yaml_with_redacted_secrets(jobs, sync): + from haiku.rag.config import APIConfig, IngesterConfig + + config = AppConfig(ingester=IngesterConfig(api=APIConfig(auth_token="supersecret"))) + state = APIState(config=config, job_repo=jobs, sync_repo=sync) + async with _client(state) as client: + resp = await client.get("/config") + assert resp.status_code == 200 + text = resp.json()["yaml"] + # Full effective config: a default section the user never wrote is present. + assert "embeddings:" in text + assert "processing:" in text + # Secret is masked, not echoed. + assert "supersecret" not in text + assert "auth_token: '***'" in text + + +@pytest.mark.asyncio +async def test_config_requires_auth(state): + async with _client(state, auth_token="secret") as client: + resp = await client.get("/config") + assert resp.status_code == 401 + + +# --- database --- + + +async def _seed_lancedb(path): + """Create a minimal LanceDB with settings/documents/chunks tables so + gather_database_info has something real to report.""" + import json + + import lancedb + from lancedb.pydantic import LanceModel, Vector + from pydantic import Field + + class SettingsRecord(LanceModel): + id: str = Field(default="settings") + settings: str = Field(default="{}") + + class DocumentRecord(LanceModel): + id: str + content: str + + class ChunkRecord(LanceModel): + id: str + document_id: str + content: str + vector: Vector(3) # type: ignore + + db = await lancedb.connect_async(str(path)) + settings_tbl = await db.create_table("settings", schema=SettingsRecord) + docs_tbl = await db.create_table("documents", schema=DocumentRecord) + chunks_tbl = await db.create_table("chunks", schema=ChunkRecord) + await settings_tbl.add( + [ + SettingsRecord( + settings=json.dumps( + { + "version": "1.2.3", + "embeddings": { + "model": { + "provider": "openai", + "name": "text-embedding-3-small", + "vector_dim": 3, + } + }, + } + ) + ) + ] + ) + await docs_tbl.add([DocumentRecord(id="doc-1", content="hello")]) + await chunks_tbl.add( + [ChunkRecord(id="c1", document_id="doc-1", content="c", vector=[0.1, 0.2, 0.3])] + ) + + +@pytest.mark.asyncio +async def test_database_reports_info(tmp_path, jobs, sync): + db_path = tmp_path / "docs.lancedb" + await _seed_lancedb(db_path) + state = APIState(config=AppConfig(), job_repo=jobs, sync_repo=sync, db_path=db_path) + async with _client(state) as client: + resp = await client.get("/database") + assert resp.status_code == 200 + body = resp.json() + assert body["exists"] is True + assert body["stored_version"] == "1.2.3" + assert body["embeddings"]["provider"] == "openai" + assert body["embeddings"]["vector_dim"] == 3 + tables = {t["name"]: t for t in body["tables"]} + assert tables["documents"]["num_rows"] == 1 + assert tables["chunks"]["num_rows"] == 1 + assert tables["document_items"]["exists"] is False + assert body["vector_index"]["exists"] is False + + +@pytest.mark.asyncio +async def test_database_503_when_db_path_unset(state): + async with _client(state) as client: + resp = await client.get("/database") + assert resp.status_code == 503 + + +@pytest.mark.asyncio +async def test_database_requires_auth(state): + async with _client(state, auth_token="secret") as client: + resp = await client.get("/database") + assert resp.status_code == 401 diff --git a/tests/test_config.py b/tests/test_config.py index e99cf7a2..05c3ebc1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -350,3 +350,27 @@ analysis: assert cfg.analysis.model.name == "analysis-model" resolved = cfg.analysis.model or cfg.qa.model assert resolved.name == "analysis-model" + + +def test_redact_secrets_masks_nested_secret_keys(): + from haiku.rag.config.loader import redact_secrets + + data = { + "api_key": "sk-123", + "name": "ollama", + "ingester": {"api": {"auth_token": "secret", "host": "0.0.0.0"}}, + "missing_token": None, + "sources": [{"password": "pw", "url": "http://x"}], + "storage_options": {"aws_secret_access_key": "abc"}, + } + + redacted = redact_secrets(data) + + assert redacted["api_key"] == "***" + assert redacted["name"] == "ollama" + assert redacted["ingester"]["api"]["auth_token"] == "***" + assert redacted["ingester"]["api"]["host"] == "0.0.0.0" + assert redacted["missing_token"] is None + assert redacted["sources"][0]["password"] == "***" + assert redacted["sources"][0]["url"] == "http://x" + assert redacted["storage_options"]["aws_secret_access_key"] == "***"