diff --git a/docs/experiment_setup.md b/docs/experiment_setup.md index f11256be..b3678228 100644 --- a/docs/experiment_setup.md +++ b/docs/experiment_setup.md @@ -125,7 +125,7 @@ The table below lists the API-hosted models used as a judge. | Model ID | Provider | Type | Parameters | |---------------------------------------------|-------------|------|-----------------------------| | gpt-5.2 | OpenAI | LLM | reasoning: default | -| gemini-3-flash-preview | Google | LLM | reasoning: default | +| gemini-3.8-flash | Google | LLM | reasoning: default | | us.anthropic.claude-opus-4-6-v1 | AWS Bedrock | LLM | reasoning: default | diff --git a/docs/metrics/README.md b/docs/metrics/README.md index 24bd4270..0dfbd7e0 100644 --- a/docs/metrics/README.md +++ b/docs/metrics/README.md @@ -83,7 +83,7 @@ Each metric implements `BaseMetric.compute(context: MetricContext) -> MetricScor **LLM-as-Judge:** - Integer/boolean ratings (not decimals) to avoid precision issues - Structured prompts in `configs/prompts/judge.yaml` -- GPT-5.2 for text judges, Gemini 3 Flash for audio judges, Claude Opus for faithfulness +- GPT-5.2 for text judges, Gemini 3.8 Flash for audio judges, Claude Opus for faithfulness **Audio Evaluation:** - Audio encoded as base64 WAV, sent to Gemini via LiteLLM diff --git a/docs/metrics/agent_speech_fidelity.md b/docs/metrics/agent_speech_fidelity.md index 6f36b160..322fefe4 100644 --- a/docs/metrics/agent_speech_fidelity.md +++ b/docs/metrics/agent_speech_fidelity.md @@ -17,7 +17,7 @@ To keep the EVA score **apples-to-apples across all pipeline setups**, the same ### Evaluation Method - **Type**: Audio Judge (multimodal LLM with audio input) -- **Model**: Gemini 3 Flash +- **Model**: Gemini 3.8 Flash - **Granularity**: Per-turn (each assistant turn evaluated independently) ### Input Data @@ -71,4 +71,4 @@ The judge receives the agent audio plus a redacted conversation trace and, for e - **Class**: `SpeechFidelityMetric` - **Base Class**: `SpeechFidelityBaseMetric` → `AudioJudgeMetric` - **Prompt**: `configs/prompts/judge.yaml` under `judge.agent_speech_fidelity` -- **Configuration**: `audio_judge_model` (default: Gemini 3 Flash), `aggregation` (default: "mean") +- **Configuration**: `audio_judge_model` (default: Gemini 3.8 Flash), `aggregation` (default: "mean") diff --git a/docs/metrics/tts_fidelity.md b/docs/metrics/tts_fidelity.md index 4123b524..2537cdd3 100644 --- a/docs/metrics/tts_fidelity.md +++ b/docs/metrics/tts_fidelity.md @@ -18,7 +18,7 @@ Audio-based metric that evaluates whether the assistant's **spoken audio** accur ### Evaluation Method - **Type**: Audio Judge (multimodal LLM with audio input) -- **Model**: Gemini 3 Flash +- **Model**: Gemini 3.8 Flash - **Granularity**: Per-turn (each assistant turn evaluated independently) ### Input Data @@ -79,4 +79,4 @@ The judge compares intended text against spoken audio, focusing on: - **Class**: `TTSFidelityMetric` - **Base Class**: `SpeechFidelityBaseMetric` → `AudioJudgeMetric` - **Prompt**: `configs/prompts/judge.yaml` under `judge.tts_fidelity` -- **Configuration**: `audio_judge_model` (default: Gemini 3 Flash), `aggregation` (default: "mean") +- **Configuration**: `audio_judge_model` (default: Gemini 3.8 Flash), `aggregation` (default: "mean") diff --git a/docs/metrics/user_speech_fidelity.md b/docs/metrics/user_speech_fidelity.md index b4b8035c..64785917 100644 --- a/docs/metrics/user_speech_fidelity.md +++ b/docs/metrics/user_speech_fidelity.md @@ -11,7 +11,7 @@ Audio-based validation metric that evaluates whether the user simulator's **spok ### Evaluation Method - **Type**: Audio Judge (multimodal LLM with audio input) -- **Model**: Gemini 3 Flash +- **Model**: Gemini 3.8 Flash - **Granularity**: Per-turn (each user turn evaluated independently) ### Input Data @@ -67,5 +67,5 @@ This metric uses a 1-3 scale instead of binary 0-1 (like agent speech fidelity) - **Prompt location**: `configs/prompts/judge.yaml` under `judge.user_speech_fidelity` - Uses the same speech fidelity prompt structure as `agent_speech_fidelity` but with `evaluation_mode="user"` and user turns - **Configuration options**: - - `audio_judge_model`: LLM model (default: Gemini 3 Flash) + - `audio_judge_model`: LLM model (default: Gemini 3.8 Flash) - `aggregation`: Aggregation method (default: "mean") diff --git a/src/eva/metrics/base.py b/src/eva/metrics/base.py index 50459f18..d19fff54 100644 --- a/src/eva/metrics/base.py +++ b/src/eva/metrics/base.py @@ -662,7 +662,7 @@ class AudioJudgeMetric(BaseMetric): metric_type = MetricType.AUDIO_JUDGE # Subclasses can override - default_model = "gemini-3-flash-preview" + default_model = "gemini-3.8-flash" default_params: dict[str, Any] = {"temperature": 0.0, "max_tokens": 40000, "reasoning_effort": "minimal"} rating_scale: tuple[int, int] = (-2, 2) # Can vary by metric diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 42fe0bfc..4118a327 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -56,13 +56,13 @@ "SpeechFidelityMetric": { "name": "agent_speech_fidelity", "prompt_hash": "3668f89a94f8", - "source_hash": "a3bfd5b63aba", + "source_hash": "c4e6f3d0b3cf", "version": "v0.5" }, "TTSFidelityMetric": { "name": "tts_fidelity", "prompt_hash": "b8a5fc23cbb2", - "source_hash": "bf688bc70e3a", + "source_hash": "4846ca933c54", "version": "v0.4" }, "TaskCompletion": { @@ -98,7 +98,7 @@ "UserSpeechFidelityMetric": { "name": "user_speech_fidelity", "prompt_hash": "df4cf200e659", - "source_hash": "97456368615c", + "source_hash": "ec8ad53652e0", "version": "v0.3" } } diff --git a/website/src/data/metricsData.ts b/website/src/data/metricsData.ts index cca0c1d2..2e86f2bc 100644 --- a/website/src/data/metricsData.ts +++ b/website/src/data/metricsData.ts @@ -55,7 +55,7 @@ export const metrics: MetricDefinition[] = [ badge: 'beta', category: 'eva-a', type: 'lalm_judge', - judgeModel: 'Gemini 3 Flash', + judgeModel: 'Gemini 3.8 Flash', judgeAccuracy: 0.8957, judgeScores: [ { label: 'accuracy', value: 0.8957, std: 0.0258 }, @@ -862,7 +862,7 @@ Transcribed: \`My phone number is 404-555.\` displayName: 'TTS Fidelity', category: 'debug', type: 'lalm_judge', - judgeModel: 'Gemini 3 Flash', + judgeModel: 'Gemini 3.8 Flash', judgeAccuracy: 0.8957, judgeScores: [ { label: 'accuracy', value: 0.8957, std: 0.0258 }, @@ -1119,7 +1119,7 @@ Respond in JSON format: displayName: 'User Speech Fidelity', category: 'validation', type: 'lalm_judge', - judgeModel: 'Gemini 3 Flash', + judgeModel: 'Gemini 3.8 Flash', description: 'Audio-based check that the user simulator\'s TTS output matches the intended text. Validates that the simulated user is actually saying what it\'s supposed to say.', inputs: 'User audio recording, intended user text', outputRange: '1-3 per turn (1=low fidelity, 2=medium, 3=high), aggregated as mean',