From abfc796c42e06173bd2109b33fedded004cb8674 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Tue, 30 Sep 2025 10:21:34 +0300
Subject: [PATCH] Use gpt-oss for evaluation LLMJudge, allow it to retry if it
 fails

---
 evaluations/llm_judge.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/evaluations/llm_judge.py b/evaluations/llm_judge.py
index 435e517b..7dcfccac 100644
--- a/evaluations/llm_judge.py
+++ b/evaluations/llm_judge.py
@@ -37,7 +37,7 @@ class LLMJudgeResponseSchema(BaseModel):
 class LLMJudge:
     """LLM-as-judge for evaluating answer equivalence using Pydantic AI."""
 
-    def __init__(self, model: str = "qwen3"):
+    def __init__(self, model: str = "gpt-oss"):
         # Create Ollama model
         ollama_model = OpenAIChatModel(
             model_name=model,
@@ -49,6 +49,7 @@ class LLMJudge:
             model=ollama_model,
             output_type=LLMJudgeResponseSchema,
             system_prompt=ANSWER_EQUIVALENCE_RUBRIC,
+            retries=3,
         )
 
     async def judge_answers(