Route eval scripts through the shared telemetry configure

evaluations/evaluations/benchmark.py

@@ -4,7 +4,6 @@ from collections.abc import Awaitable, Callable, Mapping
 from pathlib import Path
 from typing import Any, Literal, cast
 
-import logfire
 import typer
 from dotenv import find_dotenv, load_dotenv
 from huggingface_hub import HfApi, snapshot_download
@@ -26,6 +25,7 @@ from haiku.rag.client import HaikuRAG
 from haiku.rag.config import AppConfig, find_config_file, load_yaml_config
 from haiku.rag.config.models import ModelConfig
 from haiku.rag.logging import configure_cli_logging
+from haiku.rag.telemetry import configure as configure_telemetry
 from haiku.rag.utils import get_model, parse_model_option
 
 Target = Literal["rag-skill", "analysis-skill"]
@@ -42,10 +42,7 @@ HF_REPO_ID = "ggozad/haiku-rag-eval-dbs"
 
 # Scrubbing off: eval outputs are financial answers with words like "authorized"
 # that trip Logfire's secret scrubber and redact the model's answer text.
-logfire.configure(
-    send_to_logfire="if-token-present", service_name="evals", scrubbing=False
-)
-logfire.instrument_pydantic_ai()
+configure_telemetry(service_name="evals", scrubbing=False)
 configure_cli_logging()
 console = Console()
 

haiku_rag_slim/haiku/rag/telemetry.py

@@ -21,6 +21,7 @@ def configure(
     *,
     service_name: str | None = None,
     console: Literal[False] | None = False,
+    scrubbing: Literal[False] | None = None,
 ) -> None:
     """Configure Logfire and enable pydantic-ai instrumentation for the
     running process. Each CLI entry point calls this once at startup.
@@ -34,6 +35,9 @@ def configure(
     - console: False (default) suppresses span lines on stderr so they
       don't interleave with RichHandler logs. Pass None to let logfire
       decide (its own default applies).
+    - scrubbing: None (default) keeps logfire's secret scrubbing on. Pass
+      False to disable it when span content legitimately contains tokens
+      that trip the scrubber (e.g. eval answer text).
     """
     try:
         import logfire as _lf
@@ -55,6 +59,7 @@ def configure(
             service_version=service_version,
             send_to_logfire="if-token-present",
             console=console,
+            scrubbing=scrubbing,
         )
         _lf.instrument_pydantic_ai()
     except Exception:  # pragma: no cover

tests/test_telemetry.py

@@ -55,3 +55,16 @@ def test_service_version_is_package_version(captured_configure, monkeypatch):
     telemetry.configure(service_name="haiku-rag")
 
     assert captured_configure["service_version"] == metadata.version("haiku.rag-slim")
+
+
+def test_scrubbing_defaults_to_enabled(captured_configure):
+    telemetry.configure(service_name="haiku-rag")
+
+    # None is logfire's "scrubbing enabled" default.
+    assert captured_configure["scrubbing"] is None
+
+
+def test_scrubbing_can_be_disabled(captured_configure):
+    telemetry.configure(service_name="evals", scrubbing=False)
+
+    assert captured_configure["scrubbing"] is False