From 3cc2d5e19e17cbe969edf88f51d47d3282f93df8 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 5 Mar 2026 13:21:31 +0200 Subject: [PATCH 1/3] Set appropriate temperature and max_tokens defaults --- CHANGELOG.md | 2 ++ docs/configuration/index.md | 6 +++++- docs/configuration/processing.md | 1 + docs/configuration/providers.md | 6 +++--- docs/configuration/qa-research.md | 3 +++ haiku_rag_slim/haiku/rag/config/models.py | 6 ++++++ 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed141488..6005b82a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ### Changed +- **Default model temperatures**: Set task-appropriate temperature defaults — 0.3 for QA, research, and title generation; 0.0 for RLM and picture description. Previously unset (provider defaults, typically 0.7–1.0). +- **Default title max_tokens**: Set `max_tokens=100` for title generation model to keep titles concise - **Test suite cleanup**: Removed stale VCR cassettes, dead fixtures, orphaned directories, and redundant tests. Strengthened weak assertions across search, context enhancement, and converter tests. Relocated misplaced `SearchResult._get_primary_label` test to `test_search.py` - **Parallel test execution**: Added `pytest-xdist` and enabled parallel test runs by default (`-n auto`), reducing test suite time from ~3.5 min to ~2 min diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 1754349f..917f8e1a 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -85,6 +85,7 @@ qa: provider: ollama name: gpt-oss enable_thinking: false + temperature: 0.3 max_iterations: 2 max_concurrency: 1 @@ -93,6 +94,7 @@ research: provider: "" # Empty to use qa settings name: "" enable_thinking: false + temperature: 0.3 max_iterations: 3 max_concurrency: 1 @@ -122,6 +124,8 @@ processing: provider: ollama name: gpt-oss enable_thinking: false + temperature: 0.3 + max_tokens: 100 conversion_options: do_ocr: true force_ocr: false @@ -156,7 +160,7 @@ custom_config = AppConfig( model=ModelConfig( provider="openai", name="gpt-4o", - temperature=0.7 + temperature=0.3 ) ), embeddings=EmbeddingsConfig( diff --git a/docs/configuration/processing.md b/docs/configuration/processing.md index 0dae70ba..a3b9bd35 100644 --- a/docs/configuration/processing.md +++ b/docs/configuration/processing.md @@ -120,6 +120,7 @@ conversion_options: model: provider: ollama # ollama, openai, or custom name: ministral-3 # VLM model name + temperature: 0.0 # Default: 0.0 (factual descriptions) timeout: 90 # Request timeout in seconds max_tokens: 200 # Maximum tokens in response ``` diff --git a/docs/configuration/providers.md b/docs/configuration/providers.md index 7f7bbded..07cbcf07 100644 --- a/docs/configuration/providers.md +++ b/docs/configuration/providers.md @@ -16,17 +16,17 @@ qa: model: provider: ollama name: gpt-oss - temperature: 0.7 + temperature: 0.3 max_tokens: 500 ``` **Available options:** -- **temperature**: Sampling temperature (0.0-1.0+) +- **temperature**: Sampling temperature (0.0-1.0+). Defaults vary by task: 0.3 for QA, research, and title generation; 0.0 for RLM and picture description. - Lower (0.0-0.3): Deterministic, focused responses - Medium (0.4-0.7): Balanced - Higher (0.8-1.0+): Creative, varied responses -- **max_tokens**: Maximum tokens in response +- **max_tokens**: Maximum tokens in response. Default: unset (provider default), except title generation (100). - **enable_thinking**: Control reasoning behavior (see below) - **base_url**: Custom endpoint for OpenAI-compatible servers (vLLM, LM Studio, etc.) diff --git a/docs/configuration/qa-research.md b/docs/configuration/qa-research.md index 0ebe931e..9a72b2bf 100644 --- a/docs/configuration/qa-research.md +++ b/docs/configuration/qa-research.md @@ -32,6 +32,7 @@ qa: provider: ollama name: gpt-oss enable_thinking: false + temperature: 0.3 # Default: 0.3 max_iterations: 2 # Maximum search iterations max_concurrency: 1 # Concurrent search operations ``` @@ -50,6 +51,7 @@ research: provider: "" # Empty to use qa settings name: "" # Empty to use qa model enable_thinking: false + temperature: 0.3 # Default: 0.3 max_iterations: 3 max_concurrency: 1 ``` @@ -69,6 +71,7 @@ rlm: model: provider: anthropic name: claude-sonnet-4-20250514 + temperature: 0.0 # Default: 0.0 (deterministic for code generation) code_timeout: 60.0 # Max seconds for code execution max_output_chars: 50000 # Truncate output after this many chars ``` diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index a07c857f..c621b99a 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -76,6 +76,7 @@ class QAConfig(BaseModel): provider="ollama", name="gpt-oss", enable_thinking=False, + temperature=0.3, ) ) max_iterations: int = 2 @@ -88,6 +89,7 @@ class ResearchConfig(BaseModel): provider="ollama", name="gpt-oss", enable_thinking=False, + temperature=0.3, ) ) max_iterations: int = 3 @@ -100,6 +102,7 @@ class RLMConfig(BaseModel): provider="ollama", name="gpt-oss", enable_thinking=False, + temperature=0.0, ) ) code_timeout: float = 60.0 @@ -114,6 +117,7 @@ class PictureDescriptionConfig(BaseModel): default_factory=lambda: ModelConfig( provider="ollama", name="ministral-3", + temperature=0.0, ) ) timeout: int = 90 @@ -162,6 +166,8 @@ class ProcessingConfig(BaseModel): provider="ollama", name="gpt-oss", enable_thinking=False, + temperature=0.3, + max_tokens=100, ) ) From 40ff54eac0a4d2bb15d4be245cdc9fac93162bbc Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 5 Mar 2026 13:54:18 +0200 Subject: [PATCH 2/3] Set params for evaluations judge --- CHANGELOG.md | 1 + evaluations/evaluations/benchmark.py | 28 ++++++++++++--------- evaluations/evaluations/evaluators/judge.py | 4 ++- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6005b82a..bd55fc13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - **Default model temperatures**: Set task-appropriate temperature defaults — 0.3 for QA, research, and title generation; 0.0 for RLM and picture description. Previously unset (provider defaults, typically 0.7–1.0). - **Default title max_tokens**: Set `max_tokens=100` for title generation model to keep titles concise +- **Evaluation judge**: Set `temperature=0.0` and `enable_thinking=True` for deterministic, higher-quality judging. Removed unused judge config from retrieval benchmarks. - **Test suite cleanup**: Removed stale VCR cassettes, dead fixtures, orphaned directories, and redundant tests. Strengthened weak assertions across search, context enhancement, and converter tests. Relocated misplaced `SearchResult._get_primary_label` test to `test_search.py` - **Parallel test execution**: Added `pytest-xdist` and enabled parallel test runs by default (`-n auto`), reducing test suite time from ~3.5 min to ~2 min diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 2dcae976..fea86afa 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -38,10 +38,10 @@ def build_experiment_metadata( dataset_key: str, test_cases: int, config: AppConfig, - judge_config: ModelConfig, + judge_config: ModelConfig | None = None, ) -> dict[str, Any]: """Build experiment metadata for Logfire tracking.""" - return { + metadata: dict[str, Any] = { "dataset": dataset_key, "test_cases": test_cases, "embedder_provider": config.embeddings.model.provider, @@ -61,12 +61,18 @@ def build_experiment_metadata( "qa_temperature": config.qa.model.temperature, "qa_max_tokens": config.qa.model.max_tokens, "qa_enable_thinking": config.qa.model.enable_thinking, - "judge_provider": judge_config.provider, - "judge_model": judge_config.name, - "judge_temperature": judge_config.temperature, - "judge_max_tokens": judge_config.max_tokens, - "judge_enable_thinking": judge_config.enable_thinking, } + if judge_config is not None: + metadata.update( + { + "judge_provider": judge_config.provider, + "judge_model": judge_config.name, + "judge_temperature": judge_config.temperature, + "judge_max_tokens": judge_config.max_tokens, + "judge_enable_thinking": judge_config.enable_thinking, + } + ) + return metadata async def populate_db( @@ -220,14 +226,10 @@ async def run_retrieval_benchmark( eval_name = name if name is not None else f"{spec.key}_retrieval_evaluation" - judge_config = ModelConfig( - provider="ollama", name="gpt-oss", enable_thinking=False - ) experiment_metadata = build_experiment_metadata( dataset_key=spec.key, test_cases=len(cases), config=config, - judge_config=judge_config, ) report = await dataset.evaluate( @@ -275,7 +277,9 @@ async def run_qa_benchmark( for index, doc in enumerate(corpus, start=1) ] - judge_config = ModelConfig(provider="ollama", name="gpt-oss", enable_thinking=False) + judge_config = ModelConfig( + provider="ollama", name="gpt-oss", enable_thinking=False, temperature=0.0 + ) judge_model = get_model(judge_config, config) evaluation_dataset = EvalDataset[str, str, dict[str, str]]( diff --git a/evaluations/evaluations/evaluators/judge.py b/evaluations/evaluations/evaluators/judge.py index a225d19b..14b6d83e 100644 --- a/evaluations/evaluations/evaluators/judge.py +++ b/evaluations/evaluations/evaluators/judge.py @@ -36,7 +36,9 @@ class LLMJudge: """LLM-as-judge for evaluating answer equivalence using Pydantic AI.""" def __init__(self, model: str = "gpt-oss", config: AppConfig | None = None): - model_config = ModelConfig(provider="ollama", name=model, enable_thinking=False) + model_config = ModelConfig( + provider="ollama", name=model, enable_thinking=True, temperature=0.0 + ) model_obj = get_model(model_config, config) # Create Pydantic AI agent From 5a0430857f73e21fe1bb4c5d333ece890572fa8b Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 5 Mar 2026 15:48:08 +0200 Subject: [PATCH 3/3] Enable thinking by default for QA agent --- CHANGELOG.md | 1 + docs/configuration/index.md | 4 ++-- docs/configuration/providers.md | 6 +++--- docs/configuration/qa-research.md | 2 +- haiku_rag_slim/haiku/rag/config/models.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd55fc13..52fcc00e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Changed - **Default model temperatures**: Set task-appropriate temperature defaults — 0.3 for QA, research, and title generation; 0.0 for RLM and picture description. Previously unset (provider defaults, typically 0.7–1.0). +- **QA thinking enabled by default**: `enable_thinking` now defaults to `True` for QA agent, improving answer quality with reasoning models. - **Default title max_tokens**: Set `max_tokens=100` for title generation model to keep titles concise - **Evaluation judge**: Set `temperature=0.0` and `enable_thinking=True` for deterministic, higher-quality judging. Removed unused judge config from retrieval benchmarks. - **Test suite cleanup**: Removed stale VCR cassettes, dead fixtures, orphaned directories, and redundant tests. Strengthened weak assertions across search, context enhancement, and converter tests. Relocated misplaced `SearchResult._get_primary_label` test to `test_search.py` diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 917f8e1a..e605c277 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -44,7 +44,7 @@ qa: model: provider: ollama name: gpt-oss - enable_thinking: false + enable_thinking: true ``` ## Complete Configuration Example @@ -84,7 +84,7 @@ qa: model: provider: ollama name: gpt-oss - enable_thinking: false + enable_thinking: true temperature: 0.3 max_iterations: 2 max_concurrency: 1 diff --git a/docs/configuration/providers.md b/docs/configuration/providers.md index 07cbcf07..cef725e6 100644 --- a/docs/configuration/providers.md +++ b/docs/configuration/providers.md @@ -37,7 +37,7 @@ The `enable_thinking` setting controls whether models use explicit reasoning ste ```yaml qa: model: - enable_thinking: false # Faster responses + enable_thinking: true # Better grounded answers research: model: @@ -63,8 +63,8 @@ See the [Pydantic AI thinking documentation](https://ai.pydantic.dev/thinking/) - **LM Studio**: Models supporting reasoning (gpt-oss, etc.) **When to use:** -- Disable for simple queries, RAG workflows, speed-critical applications -- Enable for complex reasoning, mathematical problems, research tasks +- Enable for QA, research, complex reasoning, and mathematical problems +- Disable for speed-critical applications, title generation, and simple tasks ## Embedding Providers diff --git a/docs/configuration/qa-research.md b/docs/configuration/qa-research.md index 9a72b2bf..8df15be3 100644 --- a/docs/configuration/qa-research.md +++ b/docs/configuration/qa-research.md @@ -31,7 +31,7 @@ qa: model: provider: ollama name: gpt-oss - enable_thinking: false + enable_thinking: true temperature: 0.3 # Default: 0.3 max_iterations: 2 # Maximum search iterations max_concurrency: 1 # Concurrent search operations diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index c621b99a..75395d68 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -75,7 +75,7 @@ class QAConfig(BaseModel): default_factory=lambda: ModelConfig( provider="ollama", name="gpt-oss", - enable_thinking=False, + enable_thinking=True, temperature=0.3, ) )