73d9d93db9
IBM's MTRAG benchmark (ClapNQ domain, pinned repo SHA): retrieval with Recall@k/nDCG@k against binary qrels, gold-prefix QA replaying reference conversation prefixes as message history, and live-session replay carrying the model's own answers and tool history across turns. Corpus population gains a bounded, resumable batched ingest path. ConversationInput case type with transcript rendering for the judge, eligibility-aware citation scoring, refusal precision/recall via a label-aware RefusalJudge, per-turn verdicts with judged-turn coverage, and per-turn tool-traffic attributes counted from each turn's new messages so the arrays survive prior-turn compaction.
27 lines
898 B
Python
27 lines
898 B
Python
from evaluations.evaluators.citation import CitationMAPEvaluator
|
|
from evaluations.evaluators.conversation import ConversationEvaluator
|
|
from evaluations.evaluators.judge import (
|
|
ANSWER_EQUIVALENCE_RUBRIC,
|
|
LLMJudge,
|
|
LLMJudgeResponseSchema,
|
|
)
|
|
from evaluations.evaluators.map import MAPEvaluator
|
|
from evaluations.evaluators.number_match import NumberMatchEvaluator
|
|
from evaluations.evaluators.refusal import REFUSAL_RUBRIC, RefusalJudge
|
|
from evaluations.evaluators.retrieval import NDCGEvaluator, RecallEvaluator
|
|
from evaluations.evaluators.transcript import TranscriptLLMJudge
|
|
|
|
__all__ = [
|
|
"ANSWER_EQUIVALENCE_RUBRIC",
|
|
"REFUSAL_RUBRIC",
|
|
"CitationMAPEvaluator",
|
|
"ConversationEvaluator",
|
|
"LLMJudge",
|
|
"LLMJudgeResponseSchema",
|
|
"MAPEvaluator",
|
|
"NDCGEvaluator",
|
|
"NumberMatchEvaluator",
|
|
"RecallEvaluator",
|
|
"RefusalJudge",
|
|
"TranscriptLLMJudge",
|
|
]
|