From a82673a900b85bd11f36eb0de3734dbb793c28c9 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 10 Jul 2026 11:38:33 +0300
Subject: [PATCH] Route eval scripts through the shared telemetry configure

---
 evaluations/evaluations/benchmark.py  |  7 ++-----
 haiku_rag_slim/haiku/rag/telemetry.py |  5 +++++
 tests/test_telemetry.py               | 13 +++++++++++++
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py
index a3f06aee..a14508be 100644
--- a/evaluations/evaluations/benchmark.py
+++ b/evaluations/evaluations/benchmark.py
@@ -4,7 +4,6 @@ from collections.abc import Awaitable, Callable, Mapping
 from pathlib import Path
 from typing import Any, Literal, cast
 
-import logfire
 import typer
 from dotenv import find_dotenv, load_dotenv
 from huggingface_hub import HfApi, snapshot_download
@@ -26,6 +25,7 @@ from haiku.rag.client import HaikuRAG
 from haiku.rag.config import AppConfig, find_config_file, load_yaml_config
 from haiku.rag.config.models import ModelConfig
 from haiku.rag.logging import configure_cli_logging
+from haiku.rag.telemetry import configure as configure_telemetry
 from haiku.rag.utils import get_model, parse_model_option
 
 Target = Literal["rag-skill", "analysis-skill"]
@@ -42,10 +42,7 @@ HF_REPO_ID = "ggozad/haiku-rag-eval-dbs"
 
 # Scrubbing off: eval outputs are financial answers with words like "authorized"
 # that trip Logfire's secret scrubber and redact the model's answer text.
-logfire.configure(
-    send_to_logfire="if-token-present", service_name="evals", scrubbing=False
-)
-logfire.instrument_pydantic_ai()
+configure_telemetry(service_name="evals", scrubbing=False)
 configure_cli_logging()
 console = Console()
 
diff --git a/haiku_rag_slim/haiku/rag/telemetry.py b/haiku_rag_slim/haiku/rag/telemetry.py
index 7ad3024c..5365668d 100644
--- a/haiku_rag_slim/haiku/rag/telemetry.py
+++ b/haiku_rag_slim/haiku/rag/telemetry.py
@@ -21,6 +21,7 @@ def configure(
     *,
     service_name: str | None = None,
     console: Literal[False] | None = False,
+    scrubbing: Literal[False] | None = None,
 ) -> None:
     """Configure Logfire and enable pydantic-ai instrumentation for the
     running process. Each CLI entry point calls this once at startup.
@@ -34,6 +35,9 @@ def configure(
     - console: False (default) suppresses span lines on stderr so they
       don't interleave with RichHandler logs. Pass None to let logfire
       decide (its own default applies).
+    - scrubbing: None (default) keeps logfire's secret scrubbing on. Pass
+      False to disable it when span content legitimately contains tokens
+      that trip the scrubber (e.g. eval answer text).
     """
     try:
         import logfire as _lf
@@ -55,6 +59,7 @@ def configure(
             service_version=service_version,
             send_to_logfire="if-token-present",
             console=console,
+            scrubbing=scrubbing,
         )
         _lf.instrument_pydantic_ai()
     except Exception:  # pragma: no cover
diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py
index 594210b7..9b033d58 100644
--- a/tests/test_telemetry.py
+++ b/tests/test_telemetry.py
@@ -55,3 +55,16 @@ def test_service_version_is_package_version(captured_configure, monkeypatch):
     telemetry.configure(service_name="haiku-rag")
 
     assert captured_configure["service_version"] == metadata.version("haiku.rag-slim")
+
+
+def test_scrubbing_defaults_to_enabled(captured_configure):
+    telemetry.configure(service_name="haiku-rag")
+
+    # None is logfire's "scrubbing enabled" default.
+    assert captured_configure["scrubbing"] is None
+
+
+def test_scrubbing_can_be_disabled(captured_configure):
+    telemetry.configure(service_name="evals", scrubbing=False)
+
+    assert captured_configure["scrubbing"] is False