From 23df12464d967d12c31f09bed0ca22336d9b4841 Mon Sep 17 00:00:00 2001
From: "fanny.riols" <fanny.riols@servicenow.com>
Date: Thu, 29 Oct 2026 15:34:56 -0400
Subject: [PATCH 1/5] Add per-language guidance slot to judge prompts

Judge prompts can now include guidance for the conversation's language.
Each judge template gets a {language_guidance} placeholder, filled from
configs/prompts/judge_language_guidance/<metric>.yaml. English gets an
empty block, so English prompts render exactly as before.

- language_guidance.py loads and renders the guidance. A regional variant
  (fr-CA) gets its base language's text followed by its own entry.
- get_judge_prompt() takes the language, fills the slot, and includes the
  guidance in the prompt hash.
- The metric signature drift guard also hashes the guidance files.
- Judge call sites pass context.language.

No guidance files are added in this commit.
---
 configs/prompts/judge.yaml                    | 18 +++----
 src/eva/metrics/accuracy/speech_fidelity.py   |  1 +
 src/eva/metrics/base.py                       | 16 ++++--
 .../transcription_accuracy_key_entities.py    |  6 ++-
 src/eva/metrics/language_guidance.py          | 51 +++++++++++++++++++
 src/eva/metrics/signatures.py                 |  4 ++
 src/eva/metrics/speech_fidelity_base.py       |  1 +
 tests/fixtures/metric_signatures.json         | 34 ++++++-------
 tests/unit/metrics/test_language_guidance.py  | 47 +++++++++++++++++
 9 files changed, 146 insertions(+), 32 deletions(-)
 create mode 100644 src/eva/metrics/language_guidance.py
 create mode 100644 tests/unit/metrics/test_language_guidance.py

diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml
index 4cdadcad6..5dd2bb7d8 100644
--- a/configs/prompts/judge.yaml
+++ b/configs/prompts/judge.yaml
@@ -106,7 +106,7 @@ judge:
       - 1 (Not Concise / Causes Cognitive Overload) – One or more significant failure modes are present that materially increase cognitive load and would hinder comprehension in a voice conversation.
 
       Provide one entry per turn_id in the conversation.
-
+      {language_guidance}
       ## Response Format
       Provide your response as a valid JSON array, one entry per turn. Each entry must include the turn_id matching the turn number shown in the conversation above.
       - If the turn contains assistant content, rate it with 1, 2, or 3.
@@ -255,7 +255,7 @@ judge:
       - **1**: Any of the following:
         - Any dimension is rated 1 (clear issue within a single dimension)
         - Three or more dimensions are flagged (even if each is individually minor, widespread issues across many areas constitute a clear overall progression problem)
-
+      {language_guidance}
       ## Response Format
       Respond in JSON format. The "evidence" field must ALWAYS contain 1-2 sentences referencing specific parts of the transcript, even when flagged is false. When not flagged, briefly explain why no issue was found. The evidence should always be in English but keep the citations in the original language.
       {{
@@ -471,7 +471,7 @@ judge:
         - Any faithfulness issue that repeatedly prevents the conversation from progressing is also rated 1.
       
       For the final rating of the conversation, use the minimum rating across all dimensions as the overall faithfulness rating (i.e., if any dimension is rated 1, overall rating is 1; if all dimensions are 3, overall rating is 3; if there are no 1s but at least one 2, overall rating is 2).
-
+      {language_guidance}
       ## Response Format
       Provide evidence in English.
       Respond in JSON format:
@@ -631,7 +631,7 @@ judge:
       Carefully review each assistant turn below. Check each turn for any of the above violations.
 
       **If you find even any violations in a turn, the rating for that turn MUST be 0.**
-
+      {language_guidance}
       ### Assistant Turns
       {assistant_turns_formatted}
 
@@ -727,7 +727,7 @@ judge:
         - If a turn has both an entity error and missing/dropped content outside tagged regions, list both `entity_error` and `truncation`.
         - Garbled audio at an entity position is `entity_error`, not `garbled_hallucination` — check the intended text first to decide whether the affected region is an entity.
         - Do NOT use `truncation` for content lost to interruption tags, even if the interruption tags are not at the exact location where the truncation occured — that is expected behavior, not an error.
-
+        {language_guidance}
         ## Response Format
         Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the Intended Turns above:
         {{
@@ -804,7 +804,7 @@ judge:
         - **0 (Low Fidelity)**: One or more entities from the conversation trace are garbled, mispronounced, or indistinguishable in the audio.
 
         If the assistant does not speak any entities from the conversation trace in a turn (e.g., a greeting, filler, or turn where it only mentions entities not in the trace), set `has_entities` to false. These turns are excluded from scoring.
-
+        {language_guidance}
         ## Response Format
         Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the Conversation Trace above:
         {{
@@ -886,7 +886,7 @@ judge:
         - **1 (Low Fidelity)**: 
           - One or more entity errors (missing entities, incorrect entities, etc) OR
           - Some other major error that prevents the conversation from continuing in a sensible manner.
-      
+        {language_guidance}
         ## Response Format
         Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the Intended Turns above:
         {{
@@ -1057,7 +1057,7 @@ judge:
           "summary": "1 entity found. Phone number spans into interrupted region — skipped. Partial transcription (404-555) matches the portion before the interruption."
       }}
       ]
-      
+      {language_guidance}
       ## Response Format
        Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the User Turns to Evaluate section above:
       [
@@ -1191,7 +1191,7 @@ judge:
       ]
 
       Note Tool Call [3] does not appear: its only argument (record_id) came from a tool response, not the user.
-
+      {language_guidance}
       ## Response Format
       Respond with a JSON array. Output **one entry per tool call that uses at least one user-provided entity**, keyed by the tool call's `[N]` id. Tool calls with no user-provided entities, and turns where the user only speaks, must NOT appear.
       [
diff --git a/src/eva/metrics/accuracy/speech_fidelity.py b/src/eva/metrics/accuracy/speech_fidelity.py
index ea34dfde3..4aa33db63 100644
--- a/src/eva/metrics/accuracy/speech_fidelity.py
+++ b/src/eva/metrics/accuracy/speech_fidelity.py
@@ -65,6 +65,7 @@ async def compute(self, context: MetricContext) -> MetricScore:
             audio_b64 = self.encode_audio_segment(audio_segment)
 
             prompt = self.get_judge_prompt(
+                language=context.language,
                 conversation_trace_formatted=trace_formatted,
                 expected_language=context.language_display_name,
             )
diff --git a/src/eva/metrics/base.py b/src/eva/metrics/base.py
index dbd4948c1..8bcab0a7e 100644
--- a/src/eva/metrics/base.py
+++ b/src/eva/metrics/base.py
@@ -12,6 +12,7 @@
 from pipecat.transcriptions.language import Language
 from pydub import AudioSegment
 
+from eva.metrics.language_guidance import render_language_guidance
 from eva.metrics.utils import (
     aggregate_per_turn_scores,
     audio_to_base64,
@@ -210,14 +211,19 @@ def __init__(self, config: dict[str, Any] | None = None):
         self.logger = get_logger(f"metrics.{self.name}")
         self.prompt_manager = get_prompt_manager()
 
-    def get_judge_prompt(self, prompt_key: str = "user_prompt", **variables) -> str:
+    def get_judge_prompt(self, prompt_key: str = "user_prompt", language: str = "en", **variables) -> str:
         """Get judge prompt using PromptManager.
 
         Stamps the unrendered template's sha256[:12] into the prompt-hash contextvar so
         any MetricScore built afterwards in the same compute() picks it up automatically.
+        When the language has judge guidance, it is rendered into ``{language_guidance}``
+        and included in the hash.
         """
         prompt_path = f"judge.{self.name}.{prompt_key}"
-        _CURRENT_PROMPT_HASH.set(hash_prompt_template(self.prompt_manager.get_template(prompt_path)))
+        template = self.prompt_manager.get_template(prompt_path)
+        guidance = render_language_guidance(self.name, language)
+        _CURRENT_PROMPT_HASH.set(hash_prompt_template(f"{template}\n{guidance}" if guidance else template))
+        variables.setdefault("language_guidance", guidance)
         return self.prompt_manager.get_prompt(prompt_path, **variables)
 
     @abstractmethod
@@ -412,7 +418,7 @@ async def compute(self, context: MetricContext) -> MetricScore:
             prompt_vars = self.get_prompt_variables(context, transcript_text)
 
             # Get and format prompt
-            prompt = self.get_judge_prompt(**prompt_vars)
+            prompt = self.get_judge_prompt(language=context.language, **prompt_vars)
 
             # Call judge
             response, raw_response = await self.call_judge(prompt, context)
@@ -538,7 +544,9 @@ async def compute(self, context: MetricContext) -> MetricScore:
             if not transcript_text:
                 return MetricScore(name=self.name, score=0.0, normalized_score=0.0, error="No turns to evaluate")
 
-            prompt = self.get_judge_prompt(**self.get_prompt_variables(context, transcript_text))
+            prompt = self.get_judge_prompt(
+                language=context.language, **self.get_prompt_variables(context, transcript_text)
+            )
             response_text, usage = await self.llm_client.generate_text([{"role": "user", "content": prompt}])
             self._log_token_usage(context, self.llm_client.model, self.llm_client.params, prompt, usage, response_text)
             parsed = parse_judge_response_list(response_text)
diff --git a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py
index fef1618e6..602e7624b 100644
--- a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py
+++ b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py
@@ -247,11 +247,13 @@ def _build_prompt_and_turns(self, context: MetricContext) -> tuple[str, list[int
         """
         if context.is_audio_native:
             conversation, tool_call_ids = self._build_audio_native_conversation(context)
-            prompt = self.get_judge_prompt(prompt_key="audio_native_prompt", conversation=conversation)
+            prompt = self.get_judge_prompt(
+                prompt_key="audio_native_prompt", language=context.language, conversation=conversation
+            )
             return prompt, tool_call_ids
 
         turn_ids = self._get_turns_to_evaluate(context)
-        prompt = self.get_judge_prompt(user_turns=self._format_user_turns(turn_ids, context))
+        prompt = self.get_judge_prompt(language=context.language, user_turns=self._format_user_turns(turn_ids, context))
         return prompt, turn_ids
 
     @staticmethod
diff --git a/src/eva/metrics/language_guidance.py b/src/eva/metrics/language_guidance.py
new file mode 100644
index 000000000..93c80f5c3
--- /dev/null
+++ b/src/eva/metrics/language_guidance.py
@@ -0,0 +1,51 @@
+"""Per-language guidance blocks injected into judge prompts.
+
+Each judge prompt template contains a ``{language_guidance}`` placeholder. For English it
+renders to an empty string, so the rendered English prompt is unchanged. For other languages
+it renders the guidance written by native speakers in
+``configs/prompts/judge_language_guidance/<metric_name>.yaml``, a mapping of language code to
+guidance text. A regional variant (e.g. ``fr-CA``) gets its base language's guidance
+(``fr``) followed by its own entry, if any.
+"""
+
+from functools import cache
+from pathlib import Path
+
+import yaml
+from pipecat.transcriptions.language import Language
+
+from eva.models.config import LANGUAGE_DISPLAY_NAMES
+
+GUIDANCE_DIR = Path(__file__).parent.parent.parent.parent / "configs" / "prompts" / "judge_language_guidance"
+
+
+@cache
+def _load_metric_guidance(metric_name: str) -> dict[str, str]:
+    path = GUIDANCE_DIR / f"{metric_name}.yaml"
+    if not path.is_file():
+        return {}
+    with open(path, encoding="utf-8") as f:
+        data = yaml.safe_load(f) or {}
+    return {str(lang): text.strip() for lang, text in data.items() if isinstance(text, str) and text.strip()}
+
+
+def get_language_guidance_text(metric_name: str, language: str) -> str:
+    """Return the raw guidance text for a metric and language (empty if none)."""
+    guidance = _load_metric_guidance(metric_name)
+    base = language.split("-")[0]
+    parts = [guidance[base]] if base in guidance else []
+    if language != base and language in guidance:
+        parts.append(guidance[language])
+    return "\n\n".join(parts)
+
+
+def render_language_guidance(metric_name: str, language: str) -> str:
+    """Return the block that replaces ``{language_guidance}`` in a judge prompt (empty if none)."""
+    text = get_language_guidance_text(metric_name, language)
+    if not text:
+        return ""
+    try:
+        display_name = LANGUAGE_DISPLAY_NAMES.get(Language(language), language)
+    except ValueError:
+        display_name = language
+    return f"## {display_name}-specific guidance\n{text}\n"
diff --git a/src/eva/metrics/signatures.py b/src/eva/metrics/signatures.py
index bdcae19f7..c2273291e 100644
--- a/src/eva/metrics/signatures.py
+++ b/src/eva/metrics/signatures.py
@@ -20,6 +20,7 @@
 import eva.metrics.experience  # noqa: F401
 import eva.metrics.validation  # noqa: F401
 from eva.metrics.base import AudioJudgeMetric, BaseMetric, TextJudgeMetric
+from eva.metrics.language_guidance import GUIDANCE_DIR
 from eva.metrics.versioning import hash_prompt_template
 from eva.utils.prompt_manager import get_prompt_manager
 
@@ -76,6 +77,9 @@ def _prompt_hash_for_metric(cls: type[BaseMetric]) -> str | None:
     if not isinstance(templates, dict):
         raise KeyError(f"No judge prompts found for metric {cls.name!r}")
     combined = "\n".join(templates[key] for key in sorted(templates) if isinstance(templates[key], str))
+    guidance_file = GUIDANCE_DIR / f"{cls.name}.yaml"
+    if guidance_file.is_file():
+        combined += "\n" + guidance_file.read_text(encoding="utf-8")
     return hash_prompt_template(combined)
 
 
diff --git a/src/eva/metrics/speech_fidelity_base.py b/src/eva/metrics/speech_fidelity_base.py
index d6ca93e04..7241463f0 100644
--- a/src/eva/metrics/speech_fidelity_base.py
+++ b/src/eva/metrics/speech_fidelity_base.py
@@ -67,6 +67,7 @@ async def compute(self, context: MetricContext) -> MetricScore:
             intended_turns_formatted = self._format_intended_turns(intended_turns)
 
             prompt = self.get_judge_prompt(
+                language=context.language,
                 intended_turns_formatted=intended_turns_formatted,
                 expected_language=context.language_display_name,
             )
diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json
index 55d516e39..49b8ceb37 100644
--- a/tests/fixtures/metric_signatures.json
+++ b/tests/fixtures/metric_signatures.json
@@ -7,8 +7,8 @@
   },
   "ConcisenessJudgeMetric": {
     "name": "conciseness",
-    "prompt_hash": "57caf9a18a3c",
-    "source_hash": "9f183871e45a",
+    "prompt_hash": "5fdd118b6fd8",
+    "source_hash": "7d251695fd2f",
     "version": "v0.1"
   },
   "ConversationCorrectlyFinishedMetric": {
@@ -19,8 +19,8 @@
   },
   "ConversationProgressionJudgeMetric": {
     "name": "conversation_progression",
-    "prompt_hash": "174cdf93b398",
-    "source_hash": "84e43143d00d",
+    "prompt_hash": "f26237934000",
+    "source_hash": "0edf89e90978",
     "version": "v0.2"
   },
   "ConversationValidEndMetric": {
@@ -31,8 +31,8 @@
   },
   "FaithfulnessJudgeMetric": {
     "name": "faithfulness",
-    "prompt_hash": "78028821ce37",
-    "source_hash": "ee9b8ea530a4",
+    "prompt_hash": "741590f3d7c1",
+    "source_hash": "b2cf38e3fbc7",
     "version": "v0.3"
   },
   "ResponseSpeedMetric": {
@@ -49,20 +49,20 @@
   },
   "SpeakabilityJudgeMetric": {
     "name": "speakability",
-    "prompt_hash": "1e4f78cb051e",
-    "source_hash": "7abcd995909a",
+    "prompt_hash": "9d39f642366b",
+    "source_hash": "b76318002d8f",
     "version": "v0.3"
   },
   "SpeechFidelityMetric": {
     "name": "agent_speech_fidelity",
-    "prompt_hash": "c614dd12d4fe",
-    "source_hash": "5e61505dc729",
+    "prompt_hash": "889e34a06a58",
+    "source_hash": "aba2c27cb6af",
     "version": "v0.4"
   },
   "TTSFidelityMetric": {
     "name": "tts_fidelity",
-    "prompt_hash": "c3a02ab03f06",
-    "source_hash": "10b97af355f7",
+    "prompt_hash": "99e126158b75",
+    "source_hash": "317128c2ea72",
     "version": "v0.3"
   },
   "TaskCompletion": {
@@ -79,8 +79,8 @@
   },
   "TranscriptionAccuracyKeyEntitiesMetric": {
     "name": "transcription_accuracy_key_entities",
-    "prompt_hash": "8492d1f0d168",
-    "source_hash": "018436ab79ac",
+    "prompt_hash": "6531de5920bd",
+    "source_hash": "e8e81aec84d5",
     "version": "v0.4"
   },
   "TurnTakingMetric": {
@@ -92,13 +92,13 @@
   "UserBehavioralFidelityMetric": {
     "name": "user_behavioral_fidelity",
     "prompt_hash": "21527fa8d79b",
-    "source_hash": "7f5dcec51568",
+    "source_hash": "b8a700ea5b89",
     "version": "v0.1"
   },
   "UserSpeechFidelityMetric": {
     "name": "user_speech_fidelity",
-    "prompt_hash": "849e13c42e3b",
-    "source_hash": "5463b901a11b",
+    "prompt_hash": "1127744357ae",
+    "source_hash": "6625af8e00f9",
     "version": "v0.2"
   }
 }
diff --git a/tests/unit/metrics/test_language_guidance.py b/tests/unit/metrics/test_language_guidance.py
new file mode 100644
index 000000000..e5aa2fa0a
--- /dev/null
+++ b/tests/unit/metrics/test_language_guidance.py
@@ -0,0 +1,47 @@
+"""Tests for per-language judge guidance injection."""
+
+import pytest
+import yaml
+
+from eva.metrics import language_guidance
+from eva.metrics.language_guidance import GUIDANCE_DIR, get_language_guidance_text, render_language_guidance
+from eva.utils.prompt_manager import get_prompt_manager
+
+
+@pytest.fixture
+def guidance_dir(tmp_path, monkeypatch):
+    (tmp_path / "some_metric.yaml").write_text(
+        yaml.safe_dump({"fr": "French note.", "fr-CA": "Quebec note.", "de": "  "}), encoding="utf-8"
+    )
+    monkeypatch.setattr(language_guidance, "GUIDANCE_DIR", tmp_path)
+    language_guidance._load_metric_guidance.cache_clear()
+    yield tmp_path
+    language_guidance._load_metric_guidance.cache_clear()
+
+
+def test_english_and_missing_metric_render_empty(guidance_dir):
+    assert render_language_guidance("some_metric", "en") == ""
+    assert render_language_guidance("unknown_metric", "fr") == ""
+    assert render_language_guidance("some_metric", "de") == ""
+
+
+def test_regional_variant_gets_base_then_own_guidance(guidance_dir):
+    assert get_language_guidance_text("some_metric", "fr") == "French note."
+    assert get_language_guidance_text("some_metric", "fr-CA") == "French note.\n\nQuebec note."
+
+
+def test_rendered_block_has_header(guidance_dir):
+    assert render_language_guidance("some_metric", "fr") == "## European French-specific guidance\nFrench note.\n"
+
+
+def test_every_guidance_file_matches_a_judge_prompt_with_placeholder():
+    judge = get_prompt_manager().prompts["judge"]
+    for path in GUIDANCE_DIR.glob("*.yaml"):
+        metric = path.stem
+        assert metric in judge, f"{path.name} has no judge prompt"
+        templates = [t for t in judge[metric].values() if isinstance(t, str)]
+        assert all("{language_guidance}" in t for t in templates), f"{metric} prompt lacks {{language_guidance}}"
+        data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        assert "en" not in data, f"{path.name} must not define English guidance"
+        for text in data.values():
+            assert "{" not in text and "}" not in text, f"{path.name}: braces are not allowed in guidance text"

From f5775550504df60514c7f37d60b8f17a14e4beee Mon Sep 17 00:00:00 2001
From: "fanny.riols" <fanny.riols@servicenow.com>
Date: Thu, 29 Oct 2026 16:32:01 -0400
Subject: [PATCH 2/5] Harden judge language guidance and cover it with tests

- A guidance file whose top level is not a mapping, or whose value is not
  a string, now raises a clear error instead of crashing or being dropped.
- get_judge_prompt() requires language=, so a new caller can't silently
  skip the guidance on non-English runs. The rendered guidance is always
  the hashed one.
- The guidance block renders as a ### section after a blank line, so the
  sections that follow it don't read as its subsections. English still
  renders empty.
- Reuse base_language() and the language display names.
- user_behavioral_fidelity has no guidance slot, and the prompt says why.
- Tests: every guidance key is a supported non-English code, every judge
  template except user_behavioral_fidelity has the slot, get_judge_prompt
  injects and hashes the guidance, each judge base class forwards
  context.language, and English prompts match a golden render taken from
  main.
---
 configs/prompts/judge.yaml                   |   2 +
 src/eva/metrics/base.py                      |   9 +-
 src/eva/metrics/language_guidance.py         |  30 ++-
 src/eva/metrics/signatures.py                |   6 +-
 tests/fixtures/judge_prompts_en.json         |  12 ++
 tests/unit/metrics/test_language_guidance.py | 197 ++++++++++++++++++-
 tests/unit/metrics/test_metric_signatures.py |   3 +-
 7 files changed, 235 insertions(+), 24 deletions(-)
 create mode 100644 tests/fixtures/judge_prompts_en.json

diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml
index 5dd2bb7d8..c9cbd855f 100644
--- a/configs/prompts/judge.yaml
+++ b/configs/prompts/judge.yaml
@@ -507,6 +507,8 @@ judge:
       }}
 
 
+  # No {language_guidance} slot: this judge scores the simulated user against its own
+  # instructions (persona, goal, language directive), not against a language's norms.
   user_behavioral_fidelity:
     user_prompt: |
       You are an expert evaluator determining whether a simulated user's behavior has corrupted the voice agent evaluation. 
diff --git a/src/eva/metrics/base.py b/src/eva/metrics/base.py
index 8bcab0a7e..50459f186 100644
--- a/src/eva/metrics/base.py
+++ b/src/eva/metrics/base.py
@@ -211,19 +211,20 @@ def __init__(self, config: dict[str, Any] | None = None):
         self.logger = get_logger(f"metrics.{self.name}")
         self.prompt_manager = get_prompt_manager()
 
-    def get_judge_prompt(self, prompt_key: str = "user_prompt", language: str = "en", **variables) -> str:
+    def get_judge_prompt(self, prompt_key: str = "user_prompt", *, language: str, **variables) -> str:
         """Get judge prompt using PromptManager.
 
         Stamps the unrendered template's sha256[:12] into the prompt-hash contextvar so
         any MetricScore built afterwards in the same compute() picks it up automatically.
-        When the language has judge guidance, it is rendered into ``{language_guidance}``
-        and included in the hash.
+        ``language`` is required so that no caller silently drops the guidance: when the
+        language has judge guidance, it is rendered into ``{language_guidance}`` and
+        included in the hash.
         """
         prompt_path = f"judge.{self.name}.{prompt_key}"
         template = self.prompt_manager.get_template(prompt_path)
         guidance = render_language_guidance(self.name, language)
         _CURRENT_PROMPT_HASH.set(hash_prompt_template(f"{template}\n{guidance}" if guidance else template))
-        variables.setdefault("language_guidance", guidance)
+        variables["language_guidance"] = guidance
         return self.prompt_manager.get_prompt(prompt_path, **variables)
 
     @abstractmethod
diff --git a/src/eva/metrics/language_guidance.py b/src/eva/metrics/language_guidance.py
index 93c80f5c3..fe0c72ee8 100644
--- a/src/eva/metrics/language_guidance.py
+++ b/src/eva/metrics/language_guidance.py
@@ -12,12 +12,13 @@
 from pathlib import Path
 
 import yaml
-from pipecat.transcriptions.language import Language
 
-from eva.models.config import LANGUAGE_DISPLAY_NAMES
+from eva.models.config import LANGUAGE_DISPLAY_NAMES, base_language
 
 GUIDANCE_DIR = Path(__file__).parent.parent.parent.parent / "configs" / "prompts" / "judge_language_guidance"
 
+DISPLAY_NAMES_BY_CODE = {str(lang.value): name for lang, name in LANGUAGE_DISPLAY_NAMES.items()}
+
 
 @cache
 def _load_metric_guidance(metric_name: str) -> dict[str, str]:
@@ -26,13 +27,21 @@ def _load_metric_guidance(metric_name: str) -> dict[str, str]:
         return {}
     with open(path, encoding="utf-8") as f:
         data = yaml.safe_load(f) or {}
-    return {str(lang): text.strip() for lang, text in data.items() if isinstance(text, str) and text.strip()}
+    if not isinstance(data, dict):
+        raise ValueError(f"{path}: expected a mapping of language code to guidance text, got {type(data).__name__}")
+    guidance = {}
+    for lang, text in data.items():
+        if not isinstance(text, str):
+            raise ValueError(f"{path}: guidance for {lang!r} must be a string, got {type(text).__name__}")
+        if text.strip():
+            guidance[str(lang)] = text.strip()
+    return guidance
 
 
 def get_language_guidance_text(metric_name: str, language: str) -> str:
     """Return the raw guidance text for a metric and language (empty if none)."""
     guidance = _load_metric_guidance(metric_name)
-    base = language.split("-")[0]
+    base = base_language(language)
     parts = [guidance[base]] if base in guidance else []
     if language != base and language in guidance:
         parts.append(guidance[language])
@@ -40,12 +49,13 @@ def get_language_guidance_text(metric_name: str, language: str) -> str:
 
 
 def render_language_guidance(metric_name: str, language: str) -> str:
-    """Return the block that replaces ``{language_guidance}`` in a judge prompt (empty if none)."""
+    """Return the block that replaces ``{language_guidance}`` in a judge prompt (empty if none).
+
+    The block starts with a newline so it is separated from the section before it. English
+    renders empty, which leaves the template's own blank line as it was.
+    """
     text = get_language_guidance_text(metric_name, language)
     if not text:
         return ""
-    try:
-        display_name = LANGUAGE_DISPLAY_NAMES.get(Language(language), language)
-    except ValueError:
-        display_name = language
-    return f"## {display_name}-specific guidance\n{text}\n"
+    display_name = DISPLAY_NAMES_BY_CODE.get(language, language)
+    return f"\n### {display_name}-specific guidance\n{text}\n"
diff --git a/src/eva/metrics/signatures.py b/src/eva/metrics/signatures.py
index c2273291e..a1a19016d 100644
--- a/src/eva/metrics/signatures.py
+++ b/src/eva/metrics/signatures.py
@@ -3,8 +3,8 @@
 A metric's "signature" captures everything we want to detect changes to:
  - `version`: the manually-bumped string on the class
  - `source_hash`: sha256[:12] of `inspect.getsource(cls)` (class body)
- - `prompt_hash`: sha256[:12] of the unrendered judge prompt template, or
-                  None for non-judge metrics
+ - `prompt_hash`: sha256[:12] of the metric's unrendered judge prompt templates
+                  plus its per-language guidance file, or None for non-judge metrics
 
 The drift test compares the current signatures against a checked-in fixture
 and fails if anything changed without an explicit version bump + fixture regen.
@@ -68,6 +68,8 @@ def _prompt_hash_for_metric(cls: type[BaseMetric]) -> str | None:
     Hashes *every* prompt template under ``judge.{name}`` (sorted by key), not just
     ``user_prompt``, so a metric that ships multiple prompts — e.g. a per-pipeline
     variant like ``audio_native_prompt`` — has all of them covered by drift detection.
+    The metric's guidance file in ``configs/prompts/judge_language_guidance/`` is
+    included too, since it is rendered into those templates.
     A judge metric without any templates raises KeyError — that's a configuration
     bug we want surfaced.
     """
diff --git a/tests/fixtures/judge_prompts_en.json b/tests/fixtures/judge_prompts_en.json
new file mode 100644
index 000000000..c16678fb3
--- /dev/null
+++ b/tests/fixtures/judge_prompts_en.json
@@ -0,0 +1,12 @@
+{
+  "agent_speech_fidelity.user_prompt": "55128fc6e9d3",
+  "conciseness.user_prompt": "a0ac0a5fb1b1",
+  "conversation_progression.user_prompt": "c45acc2bc604",
+  "faithfulness.user_prompt": "9cedbcc31449",
+  "speakability.user_prompt": "3e194cc6b73a",
+  "transcription_accuracy_key_entities.audio_native_prompt": "315699fd0211",
+  "transcription_accuracy_key_entities.user_prompt": "bef4cfeb31b0",
+  "tts_fidelity.user_prompt": "17eb8a429b2c",
+  "user_behavioral_fidelity.user_prompt": "5f6361e25eba",
+  "user_speech_fidelity.user_prompt": "969c56fe32d5"
+}
diff --git a/tests/unit/metrics/test_language_guidance.py b/tests/unit/metrics/test_language_guidance.py
index e5aa2fa0a..90cddf30a 100644
--- a/tests/unit/metrics/test_language_guidance.py
+++ b/tests/unit/metrics/test_language_guidance.py
@@ -1,12 +1,41 @@
 """Tests for per-language judge guidance injection."""
 
+import hashlib
+import json
+import os
+import string
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
 import pytest
 import yaml
 
 from eva.metrics import language_guidance
-from eva.metrics.language_guidance import GUIDANCE_DIR, get_language_guidance_text, render_language_guidance
+from eva.metrics.accuracy.faithfulness import FaithfulnessJudgeMetric
+from eva.metrics.accuracy.speech_fidelity import SpeechFidelityMetric
+from eva.metrics.base import BaseMetric
+from eva.metrics.diagnostic.transcription_accuracy_key_entities import TranscriptionAccuracyKeyEntitiesMetric
+from eva.metrics.diagnostic.tts_fidelity import TTSFidelityMetric
+from eva.metrics.experience.conciseness import ConcisenessJudgeMetric
+from eva.metrics.language_guidance import (
+    DISPLAY_NAMES_BY_CODE,
+    GUIDANCE_DIR,
+    get_language_guidance_text,
+    render_language_guidance,
+)
+from eva.metrics.versioning import _CURRENT_PROMPT_HASH
+from eva.models.config import PipelineType, base_language
 from eva.utils.prompt_manager import get_prompt_manager
 
+from .conftest import make_judge_metric, make_metric_context
+
+# Scores the simulated user against its own instructions, so it has no guidance slot.
+JUDGES_WITHOUT_GUIDANCE = {"user_behavioral_fidelity"}
+
+GOLDEN_EN_PATH = Path(__file__).resolve().parents[3] / "tests" / "fixtures" / "judge_prompts_en.json"
+REGEN_GOLDEN_ENV = "EVA_REGEN_JUDGE_PROMPT_GOLDEN"
+
 
 @pytest.fixture
 def guidance_dir(tmp_path, monkeypatch):
@@ -31,17 +60,171 @@ def test_regional_variant_gets_base_then_own_guidance(guidance_dir):
 
 
 def test_rendered_block_has_header(guidance_dir):
-    assert render_language_guidance("some_metric", "fr") == "## European French-specific guidance\nFrench note.\n"
+    assert render_language_guidance("some_metric", "fr") == "\n### European French-specific guidance\nFrench note.\n"
+
+
+@pytest.mark.parametrize(
+    "content, error",
+    [
+        ("- one bullet\n- another bullet\n", "expected a mapping"),
+        ("fr:\n  - one bullet\n", "must be a string"),
+    ],
+)
+def test_malformed_guidance_file_raises(guidance_dir, content, error):
+    (guidance_dir / "bad_metric.yaml").write_text(content, encoding="utf-8")
+    with pytest.raises(ValueError, match=error):
+        get_language_guidance_text("bad_metric", "fr")
+
+
+def _judge_templates() -> list[tuple[str, str, str]]:
+    judge = get_prompt_manager().prompts["judge"]
+    return [
+        (metric, key, template)
+        for metric, prompts in judge.items()
+        if isinstance(prompts, dict)
+        for key, template in prompts.items()
+        if isinstance(template, str)
+    ]
+
 
+def test_every_judge_template_has_guidance_slot():
+    for metric, key, template in _judge_templates():
+        if metric in JUDGES_WITHOUT_GUIDANCE:
+            assert "{language_guidance}" not in template, f"judge.{metric}.{key} should not have a guidance slot"
+        else:
+            assert "{language_guidance}" in template, f"judge.{metric}.{key} lacks {{language_guidance}}"
 
-def test_every_guidance_file_matches_a_judge_prompt_with_placeholder():
+
+def test_every_guidance_file_is_valid():
     judge = get_prompt_manager().prompts["judge"]
+    known_languages = {code for code in DISPLAY_NAMES_BY_CODE if base_language(code) != "en"}
     for path in GUIDANCE_DIR.glob("*.yaml"):
         metric = path.stem
         assert metric in judge, f"{path.name} has no judge prompt"
-        templates = [t for t in judge[metric].values() if isinstance(t, str)]
-        assert all("{language_guidance}" in t for t in templates), f"{metric} prompt lacks {{language_guidance}}"
+        assert metric not in JUDGES_WITHOUT_GUIDANCE, f"{metric} prompt has no guidance slot"
         data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
-        assert "en" not in data, f"{path.name} must not define English guidance"
-        for text in data.values():
+        assert isinstance(data, dict), f"{path.name} must be a mapping of language code to text"
+        for lang, text in data.items():
+            assert lang in known_languages, f"{path.name}: {lang!r} is not a supported non-English language code"
+            assert isinstance(text, str), f"{path.name}: guidance for {lang!r} must be a string"
             assert "{" not in text and "}" not in text, f"{path.name}: braces are not allowed in guidance text"
+
+
+def test_get_judge_prompt_injects_guidance_and_changes_hash(tmp_path, monkeypatch):
+    metric = make_judge_metric(ConcisenessJudgeMetric)
+    (tmp_path / f"{metric.name}.yaml").write_text(yaml.safe_dump({"fr": "French note."}), encoding="utf-8")
+    monkeypatch.setattr(language_guidance, "GUIDANCE_DIR", tmp_path)
+    language_guidance._load_metric_guidance.cache_clear()
+    try:
+        english = metric.get_judge_prompt(language="en", conversation_turns="<turns>")
+        english_hash = _CURRENT_PROMPT_HASH.get()
+        french = metric.get_judge_prompt(language="fr", conversation_turns="<turns>")
+        french_hash = _CURRENT_PROMPT_HASH.get()
+    finally:
+        language_guidance._load_metric_guidance.cache_clear()
+
+    assert "### European French-specific guidance\nFrench note.\n" in french
+    assert "specific guidance" not in english
+    assert french_hash != english_hash
+
+
+class _Captured(Exception):
+    """Raised by the patched get_judge_prompt to stop compute() once the prompt is requested."""
+
+
+def _speech_fidelity_patches(metric):
+    return [
+        patch.object(metric, "load_role_audio", return_value=MagicMock()),
+        patch.object(metric, "encode_audio_segment", return_value=""),
+    ]
+
+
+@pytest.mark.parametrize(
+    "metric_cls, extra_patches",
+    [
+        # ConversationTextJudgeMetric
+        (FaithfulnessJudgeMetric, lambda m: [patch.object(m, "format_transcript", return_value="<transcript>")]),
+        # PerTurnConversationJudgeMetric
+        (ConcisenessJudgeMetric, lambda m: [patch.object(m, "format_transcript", return_value="<transcript>")]),
+        # SpeechFidelityBaseMetric
+        (
+            TTSFidelityMetric,
+            lambda m: [
+                *_speech_fidelity_patches(m),
+                patch.object(m, "_get_intended_turns", return_value={1: "Hello"}),
+            ],
+        ),
+        # SpeechFidelityMetric
+        (
+            SpeechFidelityMetric,
+            lambda m: [
+                *_speech_fidelity_patches(m),
+                patch.object(m, "_build_redacted_trace", return_value=[]),
+                patch.object(m, "_get_assistant_turn_ids", return_value=[1]),
+                patch.object(m, "_format_redacted_trace", return_value="<trace>"),
+            ],
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_judge_metrics_forward_context_language(metric_cls, extra_patches):
+    metric = make_judge_metric(metric_cls, mock_llm=True)
+    context = make_metric_context(language="fr-CA", conversation_trace=[])
+    spy = MagicMock(side_effect=_Captured)
+    patches = [patch.object(metric, "get_judge_prompt", spy), *extra_patches(metric)]
+    for p in patches:
+        p.start()
+    try:
+        await metric.compute(context)
+    finally:
+        for p in patches:
+            p.stop()
+
+    spy.assert_called_once()
+    assert spy.call_args.kwargs["language"] == "fr-CA"
+
+
+@pytest.mark.parametrize("pipeline_type", [PipelineType.CASCADE, PipelineType.S2S])
+def test_key_entities_forwards_context_language(pipeline_type):
+    metric = make_judge_metric(TranscriptionAccuracyKeyEntitiesMetric)
+    context = make_metric_context(
+        language="ja",
+        pipeline_type=pipeline_type,
+        intended_user_turns={0: "ABC123"},
+        transcribed_user_turns={0: "ABC123"},
+    )
+    spy = MagicMock(side_effect=_Captured)
+    with (
+        patch.object(metric, "get_judge_prompt", spy),
+        patch.object(metric, "_build_audio_native_conversation", return_value=("<conversation>", [1])),
+        pytest.raises(_Captured),
+    ):
+        metric._build_prompt_and_turns(context)
+
+    assert spy.call_args.kwargs["language"] == "ja"
+
+
+def _render_english_judge_prompts() -> dict[str, str]:
+    """Render every judge template in English with placeholder values, keyed by ``metric.prompt_key``."""
+    prompt_manager = get_prompt_manager()
+    hashes = {}
+    for metric, key, template in _judge_templates():
+        fields = {name for _, name, _, _ in string.Formatter().parse(template) if name}
+        variables = {name: f"<{name}>" for name in fields - {"language_guidance"}}
+        stub = SimpleNamespace(name=metric, prompt_manager=prompt_manager)
+        rendered = BaseMetric.get_judge_prompt(stub, key, language="en", **variables)
+        hashes[f"{metric}.{key}"] = hashlib.sha256(rendered.encode("utf-8")).hexdigest()[:12]
+    return hashes
+
+
+def test_english_judge_prompts_match_golden():
+    """English judge prompts are byte-identical to the golden renders.
+
+    An intended change to an English prompt must update the golden file:
+    ``EVA_REGEN_JUDGE_PROMPT_GOLDEN=1 uv run pytest tests/unit/metrics/test_language_guidance.py``.
+    """
+    current = _render_english_judge_prompts()
+    if os.environ.get(REGEN_GOLDEN_ENV):
+        GOLDEN_EN_PATH.write_text(json.dumps(current, indent=2, sort_keys=True) + "\n")
+    golden = json.loads(GOLDEN_EN_PATH.read_text())
+    assert current == golden, f"English judge prompts changed. If intended, regenerate with {REGEN_GOLDEN_ENV}=1."
diff --git a/tests/unit/metrics/test_metric_signatures.py b/tests/unit/metrics/test_metric_signatures.py
index 24ce61c5d..4998c54ab 100644
--- a/tests/unit/metrics/test_metric_signatures.py
+++ b/tests/unit/metrics/test_metric_signatures.py
@@ -3,7 +3,8 @@
 Each concrete metric class has three signature fields:
   - version: manually bumped string on the class
   - source_hash: sha256[:12] of inspect.getsource(cls)
-  - prompt_hash: sha256[:12] of judge.{name}.user_prompt template (None for code metrics)
+  - prompt_hash: sha256[:12] of all judge.{name}.* templates plus the metric's
+    language guidance file (None for code metrics)
 
 The fixture at tests/fixtures/metric_signatures.json is the source of truth for
 the *currently-released* state. The test compares the current signatures to it

From eb286fee7ebf3adbf0a4d0a46815246ac1da25b7 Mon Sep 17 00:00:00 2001
From: Fanny Riols <fanny.riols@servicenow.com>
Date: Thu, 29 Oct 2026 17:03:32 -0400
Subject: [PATCH 3/5] Apply suggestion from @fanny-riols

---
 configs/prompts/judge.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml
index c9cbd855f..5dd2bb7d8 100644
--- a/configs/prompts/judge.yaml
+++ b/configs/prompts/judge.yaml
@@ -507,8 +507,6 @@ judge:
       }}
 
 
-  # No {language_guidance} slot: this judge scores the simulated user against its own
-  # instructions (persona, goal, language directive), not against a language's norms.
   user_behavioral_fidelity:
     user_prompt: |
       You are an expert evaluator determining whether a simulated user's behavior has corrupted the voice agent evaluation. 

From 5568431f6d1e915071c61b776b739c65e0ba5d1b Mon Sep 17 00:00:00 2001
From: "fanny.riols" <fanny.riols@servicenow.com>
Date: Mon, 2 Nov 2026 10:02:26 -0500
Subject: [PATCH 4/5] Key entities: put the guidance slot before the examples

In both key-entities prompts, the guidance block came right after the
worked example's JSON, so it read as part of the example. It now ends
the Correctness Criteria, which it refines, and the example follows it.
English renders exactly as before.
---
 configs/prompts/judge.yaml            | 8 ++++----
 tests/fixtures/metric_signatures.json | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml
index 5dd2bb7d8..06fd938e9 100644
--- a/configs/prompts/judge.yaml
+++ b/configs/prompts/judge.yaml
@@ -962,7 +962,7 @@ judge:
       - Honorific/title variants: "Ms." vs "Miss", "Mr." vs "Mister", and language equivalents (e.g., French "Madame" vs "Mme", Japanese "様" vs "-san") are acceptable variants of the same title.
 
       Important note: The expected text will often feature things formatted like "one two three" instead of "123". Your goal is to evaluate the semantic equivalence, meaning these are considered equivalent if they were heard in audio.
-      
+      {language_guidance}
       ## Examples
       
       **Example Input:**
@@ -1057,7 +1057,7 @@ judge:
           "summary": "1 entity found. Phone number spans into interrupted region — skipped. Partial transcription (404-555) matches the portion before the interruption."
       }}
       ]
-      {language_guidance}
+      
       ## Response Format
        Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the User Turns to Evaluate section above:
       [
@@ -1140,7 +1140,7 @@ judge:
         - Large-unit grouping differences (e.g., Japanese 万-system: "ichi-man go-sen" / 一万五千 = 15,000)
 
       Important note: The user text will often feature things formatted like "one two three" instead of "123". Your goal is to evaluate semantic equivalence — these are considered equivalent if the value matches what the agent used.
-
+      {language_guidance}
       ## Examples
 
       The user gives an id in turn 1 and corrects it in turn 2; nothing is rated at those user turns — each **tool call** is rated on its own. Tool Call [1] used the pre-correction value EM123 (wrong, because the user had already corrected it) and Tool Call [2] used the corrected value EM456 (right). Tool Call [3] uses only a `record_id` that came from a tool response, so it is omitted entirely.
@@ -1191,7 +1191,7 @@ judge:
       ]
 
       Note Tool Call [3] does not appear: its only argument (record_id) came from a tool response, not the user.
-      {language_guidance}
+
       ## Response Format
       Respond with a JSON array. Output **one entry per tool call that uses at least one user-provided entity**, keyed by the tool call's `[N]` id. Tool calls with no user-provided entities, and turns where the user only speaks, must NOT appear.
       [
diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json
index 49b8ceb37..447d371b0 100644
--- a/tests/fixtures/metric_signatures.json
+++ b/tests/fixtures/metric_signatures.json
@@ -79,7 +79,7 @@
   },
   "TranscriptionAccuracyKeyEntitiesMetric": {
     "name": "transcription_accuracy_key_entities",
-    "prompt_hash": "6531de5920bd",
+    "prompt_hash": "dcf0fe2feed6",
     "source_hash": "e8e81aec84d5",
     "version": "v0.4"
   },

From f5025d364d5c1eb0de3bbc4bd5713ce2eeb90cd2 Mon Sep 17 00:00:00 2001
From: "fanny.riols" <fanny.riols@servicenow.com>
Date: Mon, 2 Nov 2026 10:11:04 -0500
Subject: [PATCH 5/5] Drop the language-forwarding tests

get_judge_prompt() requires language=, so a call site that forgets it
raises, and the existing metric tests already run compute() on every
call site. The forwarding tests only added a check on the value passed,
and they patched private helpers, so an unrelated refactor could break
them.
---
 tests/unit/metrics/test_language_guidance.py | 85 +-------------------
 1 file changed, 2 insertions(+), 83 deletions(-)

diff --git a/tests/unit/metrics/test_language_guidance.py b/tests/unit/metrics/test_language_guidance.py
index 90cddf30a..e66320962 100644
--- a/tests/unit/metrics/test_language_guidance.py
+++ b/tests/unit/metrics/test_language_guidance.py
@@ -6,17 +6,12 @@
 import string
 from pathlib import Path
 from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
 
 import pytest
 import yaml
 
 from eva.metrics import language_guidance
-from eva.metrics.accuracy.faithfulness import FaithfulnessJudgeMetric
-from eva.metrics.accuracy.speech_fidelity import SpeechFidelityMetric
 from eva.metrics.base import BaseMetric
-from eva.metrics.diagnostic.transcription_accuracy_key_entities import TranscriptionAccuracyKeyEntitiesMetric
-from eva.metrics.diagnostic.tts_fidelity import TTSFidelityMetric
 from eva.metrics.experience.conciseness import ConcisenessJudgeMetric
 from eva.metrics.language_guidance import (
     DISPLAY_NAMES_BY_CODE,
@@ -25,10 +20,10 @@
     render_language_guidance,
 )
 from eva.metrics.versioning import _CURRENT_PROMPT_HASH
-from eva.models.config import PipelineType, base_language
+from eva.models.config import base_language
 from eva.utils.prompt_manager import get_prompt_manager
 
-from .conftest import make_judge_metric, make_metric_context
+from .conftest import make_judge_metric
 
 # Scores the simulated user against its own instructions, so it has no guidance slot.
 JUDGES_WITHOUT_GUIDANCE = {"user_behavioral_fidelity"}
@@ -128,82 +123,6 @@ def test_get_judge_prompt_injects_guidance_and_changes_hash(tmp_path, monkeypatc
     assert french_hash != english_hash
 
 
-class _Captured(Exception):
-    """Raised by the patched get_judge_prompt to stop compute() once the prompt is requested."""
-
-
-def _speech_fidelity_patches(metric):
-    return [
-        patch.object(metric, "load_role_audio", return_value=MagicMock()),
-        patch.object(metric, "encode_audio_segment", return_value=""),
-    ]
-
-
-@pytest.mark.parametrize(
-    "metric_cls, extra_patches",
-    [
-        # ConversationTextJudgeMetric
-        (FaithfulnessJudgeMetric, lambda m: [patch.object(m, "format_transcript", return_value="<transcript>")]),
-        # PerTurnConversationJudgeMetric
-        (ConcisenessJudgeMetric, lambda m: [patch.object(m, "format_transcript", return_value="<transcript>")]),
-        # SpeechFidelityBaseMetric
-        (
-            TTSFidelityMetric,
-            lambda m: [
-                *_speech_fidelity_patches(m),
-                patch.object(m, "_get_intended_turns", return_value={1: "Hello"}),
-            ],
-        ),
-        # SpeechFidelityMetric
-        (
-            SpeechFidelityMetric,
-            lambda m: [
-                *_speech_fidelity_patches(m),
-                patch.object(m, "_build_redacted_trace", return_value=[]),
-                patch.object(m, "_get_assistant_turn_ids", return_value=[1]),
-                patch.object(m, "_format_redacted_trace", return_value="<trace>"),
-            ],
-        ),
-    ],
-)
-@pytest.mark.asyncio
-async def test_judge_metrics_forward_context_language(metric_cls, extra_patches):
-    metric = make_judge_metric(metric_cls, mock_llm=True)
-    context = make_metric_context(language="fr-CA", conversation_trace=[])
-    spy = MagicMock(side_effect=_Captured)
-    patches = [patch.object(metric, "get_judge_prompt", spy), *extra_patches(metric)]
-    for p in patches:
-        p.start()
-    try:
-        await metric.compute(context)
-    finally:
-        for p in patches:
-            p.stop()
-
-    spy.assert_called_once()
-    assert spy.call_args.kwargs["language"] == "fr-CA"
-
-
-@pytest.mark.parametrize("pipeline_type", [PipelineType.CASCADE, PipelineType.S2S])
-def test_key_entities_forwards_context_language(pipeline_type):
-    metric = make_judge_metric(TranscriptionAccuracyKeyEntitiesMetric)
-    context = make_metric_context(
-        language="ja",
-        pipeline_type=pipeline_type,
-        intended_user_turns={0: "ABC123"},
-        transcribed_user_turns={0: "ABC123"},
-    )
-    spy = MagicMock(side_effect=_Captured)
-    with (
-        patch.object(metric, "get_judge_prompt", spy),
-        patch.object(metric, "_build_audio_native_conversation", return_value=("<conversation>", [1])),
-        pytest.raises(_Captured),
-    ):
-        metric._build_prompt_and_turns(context)
-
-    assert spy.call_args.kwargs["language"] == "ja"
-
-
 def _render_english_judge_prompts() -> dict[str, str]:
     """Render every judge template in English with placeholder values, keyed by ``metric.prompt_key``."""
     prompt_manager = get_prompt_manager()