From 708d485634a77949b0c40abaf1e462bac0d04f10 Mon Sep 17 00:00:00 2001 From: "fanny.riols" Date: Thu, 12 Nov 2026 09:57:19 -0500 Subject: [PATCH] Use Gemini 3.8 Flash as the default audio judge --- docs/experiment_setup.md | 2 +- docs/metrics/README.md | 2 +- docs/metrics/agent_speech_fidelity.md | 4 ++-- docs/metrics/tts_fidelity.md | 4 ++-- docs/metrics/user_speech_fidelity.md | 4 ++-- src/eva/metrics/base.py | 2 +- tests/fixtures/metric_signatures.json | 6 +++--- website/src/data/metricsData.ts | 6 +++--- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/experiment_setup.md b/docs/experiment_setup.md index f11256bec..b3678228f 100644 --- a/docs/experiment_setup.md +++ b/docs/experiment_setup.md @@ -125,7 +125,7 @@ The table below lists the API-hosted models used as a judge. | Model ID | Provider | Type | Parameters | |---------------------------------------------|-------------|------|-----------------------------| | gpt-5.2 | OpenAI | LLM | reasoning: default | -| gemini-3-flash-preview | Google | LLM | reasoning: default | +| gemini-3.8-flash | Google | LLM | reasoning: default | | us.anthropic.claude-opus-4-6-v1 | AWS Bedrock | LLM | reasoning: default | diff --git a/docs/metrics/README.md b/docs/metrics/README.md index 24bd42701..0dfbd7e08 100644 --- a/docs/metrics/README.md +++ b/docs/metrics/README.md @@ -83,7 +83,7 @@ Each metric implements `BaseMetric.compute(context: MetricContext) -> MetricScor **LLM-as-Judge:** - Integer/boolean ratings (not decimals) to avoid precision issues - Structured prompts in `configs/prompts/judge.yaml` -- GPT-5.2 for text judges, Gemini 3 Flash for audio judges, Claude Opus for faithfulness +- GPT-5.2 for text judges, Gemini 3.8 Flash for audio judges, Claude Opus for faithfulness **Audio Evaluation:** - Audio encoded as base64 WAV, sent to Gemini via LiteLLM diff --git a/docs/metrics/agent_speech_fidelity.md b/docs/metrics/agent_speech_fidelity.md index 6f36b160e..322fefe46 100644 --- a/docs/metrics/agent_speech_fidelity.md +++ b/docs/metrics/agent_speech_fidelity.md @@ -17,7 +17,7 @@ To keep the EVA score **apples-to-apples across all pipeline setups**, the same ### Evaluation Method - **Type**: Audio Judge (multimodal LLM with audio input) -- **Model**: Gemini 3 Flash +- **Model**: Gemini 3.8 Flash - **Granularity**: Per-turn (each assistant turn evaluated independently) ### Input Data @@ -71,4 +71,4 @@ The judge receives the agent audio plus a redacted conversation trace and, for e - **Class**: `SpeechFidelityMetric` - **Base Class**: `SpeechFidelityBaseMetric` → `AudioJudgeMetric` - **Prompt**: `configs/prompts/judge.yaml` under `judge.agent_speech_fidelity` -- **Configuration**: `audio_judge_model` (default: Gemini 3 Flash), `aggregation` (default: "mean") +- **Configuration**: `audio_judge_model` (default: Gemini 3.8 Flash), `aggregation` (default: "mean") diff --git a/docs/metrics/tts_fidelity.md b/docs/metrics/tts_fidelity.md index 4123b5245..2537cdd37 100644 --- a/docs/metrics/tts_fidelity.md +++ b/docs/metrics/tts_fidelity.md @@ -18,7 +18,7 @@ Audio-based metric that evaluates whether the assistant's **spoken audio** accur ### Evaluation Method - **Type**: Audio Judge (multimodal LLM with audio input) -- **Model**: Gemini 3 Flash +- **Model**: Gemini 3.8 Flash - **Granularity**: Per-turn (each assistant turn evaluated independently) ### Input Data @@ -79,4 +79,4 @@ The judge compares intended text against spoken audio, focusing on: - **Class**: `TTSFidelityMetric` - **Base Class**: `SpeechFidelityBaseMetric` → `AudioJudgeMetric` - **Prompt**: `configs/prompts/judge.yaml` under `judge.tts_fidelity` -- **Configuration**: `audio_judge_model` (default: Gemini 3 Flash), `aggregation` (default: "mean") +- **Configuration**: `audio_judge_model` (default: Gemini 3.8 Flash), `aggregation` (default: "mean") diff --git a/docs/metrics/user_speech_fidelity.md b/docs/metrics/user_speech_fidelity.md index b4b8035c5..64785917a 100644 --- a/docs/metrics/user_speech_fidelity.md +++ b/docs/metrics/user_speech_fidelity.md @@ -11,7 +11,7 @@ Audio-based validation metric that evaluates whether the user simulator's **spok ### Evaluation Method - **Type**: Audio Judge (multimodal LLM with audio input) -- **Model**: Gemini 3 Flash +- **Model**: Gemini 3.8 Flash - **Granularity**: Per-turn (each user turn evaluated independently) ### Input Data @@ -67,5 +67,5 @@ This metric uses a 1-3 scale instead of binary 0-1 (like agent speech fidelity) - **Prompt location**: `configs/prompts/judge.yaml` under `judge.user_speech_fidelity` - Uses the same speech fidelity prompt structure as `agent_speech_fidelity` but with `evaluation_mode="user"` and user turns - **Configuration options**: - - `audio_judge_model`: LLM model (default: Gemini 3 Flash) + - `audio_judge_model`: LLM model (default: Gemini 3.8 Flash) - `aggregation`: Aggregation method (default: "mean") diff --git a/src/eva/metrics/base.py b/src/eva/metrics/base.py index 50459f186..d19fff544 100644 --- a/src/eva/metrics/base.py +++ b/src/eva/metrics/base.py @@ -662,7 +662,7 @@ class AudioJudgeMetric(BaseMetric): metric_type = MetricType.AUDIO_JUDGE # Subclasses can override - default_model = "gemini-3-flash-preview" + default_model = "gemini-3.8-flash" default_params: dict[str, Any] = {"temperature": 0.0, "max_tokens": 40000, "reasoning_effort": "minimal"} rating_scale: tuple[int, int] = (-2, 2) # Can vary by metric diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 553009644..184470abd 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -56,13 +56,13 @@ "SpeechFidelityMetric": { "name": "agent_speech_fidelity", "prompt_hash": "9cc8bda1a428", - "source_hash": "a3bfd5b63aba", + "source_hash": "c4e6f3d0b3cf", "version": "v0.5" }, "TTSFidelityMetric": { "name": "tts_fidelity", "prompt_hash": "982681428bc8", - "source_hash": "bf688bc70e3a", + "source_hash": "4846ca933c54", "version": "v0.4" }, "TaskCompletion": { @@ -98,7 +98,7 @@ "UserSpeechFidelityMetric": { "name": "user_speech_fidelity", "prompt_hash": "169ef50b834d", - "source_hash": "97456368615c", + "source_hash": "ec8ad53652e0", "version": "v0.3" } } diff --git a/website/src/data/metricsData.ts b/website/src/data/metricsData.ts index cca0c1d27..2e86f2bc7 100644 --- a/website/src/data/metricsData.ts +++ b/website/src/data/metricsData.ts @@ -55,7 +55,7 @@ export const metrics: MetricDefinition[] = [ badge: 'beta', category: 'eva-a', type: 'lalm_judge', - judgeModel: 'Gemini 3 Flash', + judgeModel: 'Gemini 3.8 Flash', judgeAccuracy: 0.8957, judgeScores: [ { label: 'accuracy', value: 0.8957, std: 0.0258 }, @@ -862,7 +862,7 @@ Transcribed: \`My phone number is 404-555.\` displayName: 'TTS Fidelity', category: 'debug', type: 'lalm_judge', - judgeModel: 'Gemini 3 Flash', + judgeModel: 'Gemini 3.8 Flash', judgeAccuracy: 0.8957, judgeScores: [ { label: 'accuracy', value: 0.8957, std: 0.0258 }, @@ -1119,7 +1119,7 @@ Respond in JSON format: displayName: 'User Speech Fidelity', category: 'validation', type: 'lalm_judge', - judgeModel: 'Gemini 3 Flash', + judgeModel: 'Gemini 3.8 Flash', description: 'Audio-based check that the user simulator\'s TTS output matches the intended text. Validates that the simulated user is actually saying what it\'s supposed to say.', inputs: 'User audio recording, intended user text', outputRange: '1-3 per turn (1=low fidelity, 2=medium, 3=high), aggregated as mean',