danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use gpt-oss for evaluation LLMJudge, allow it to retry if it fails

Browse source
This commit is contained in:
Yiorgis Gozadinos 2025-09-30 10:21:34 +03:00
parent f62d3e552b
commit abfc796c42
No known key found for this signature in database
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
evaluations/llm_judge.py
View file

@ -37,7 +37,7 @@ class LLMJudgeResponseSchema(BaseModel):
class LLMJudge: class LLMJudge:
"""LLM-as-judge for evaluating answer equivalence using Pydantic AI.""" """LLM-as-judge for evaluating answer equivalence using Pydantic AI."""
def __init__(self, model: str = "qwen3"): def __init__(self, model: str = "gpt-oss"):
# Create Ollama model # Create Ollama model
ollama_model = OpenAIChatModel( ollama_model = OpenAIChatModel(
model_name=model, model_name=model,
@ -49,6 +49,7 @@ class LLMJudge:
model=ollama_model, model=ollama_model,
output_type=LLMJudgeResponseSchema, output_type=LLMJudgeResponseSchema,
system_prompt=ANSWER_EQUIVALENCE_RUBRIC, system_prompt=ANSWER_EQUIVALENCE_RUBRIC,
retries=3,
) )
async def judge_answers( async def judge_answers(