diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml index 4cdadcad6..06fd938e9 100644 --- a/configs/prompts/judge.yaml +++ b/configs/prompts/judge.yaml @@ -106,7 +106,7 @@ judge: - 1 (Not Concise / Causes Cognitive Overload) – One or more significant failure modes are present that materially increase cognitive load and would hinder comprehension in a voice conversation. Provide one entry per turn_id in the conversation. - + {language_guidance} ## Response Format Provide your response as a valid JSON array, one entry per turn. Each entry must include the turn_id matching the turn number shown in the conversation above. - If the turn contains assistant content, rate it with 1, 2, or 3. @@ -255,7 +255,7 @@ judge: - **1**: Any of the following: - Any dimension is rated 1 (clear issue within a single dimension) - Three or more dimensions are flagged (even if each is individually minor, widespread issues across many areas constitute a clear overall progression problem) - + {language_guidance} ## Response Format Respond in JSON format. The "evidence" field must ALWAYS contain 1-2 sentences referencing specific parts of the transcript, even when flagged is false. When not flagged, briefly explain why no issue was found. The evidence should always be in English but keep the citations in the original language. {{ @@ -471,7 +471,7 @@ judge: - Any faithfulness issue that repeatedly prevents the conversation from progressing is also rated 1. For the final rating of the conversation, use the minimum rating across all dimensions as the overall faithfulness rating (i.e., if any dimension is rated 1, overall rating is 1; if all dimensions are 3, overall rating is 3; if there are no 1s but at least one 2, overall rating is 2). - + {language_guidance} ## Response Format Provide evidence in English. Respond in JSON format: @@ -631,7 +631,7 @@ judge: Carefully review each assistant turn below. Check each turn for any of the above violations. **If you find even any violations in a turn, the rating for that turn MUST be 0.** - + {language_guidance} ### Assistant Turns {assistant_turns_formatted} @@ -727,7 +727,7 @@ judge: - If a turn has both an entity error and missing/dropped content outside tagged regions, list both `entity_error` and `truncation`. - Garbled audio at an entity position is `entity_error`, not `garbled_hallucination` — check the intended text first to decide whether the affected region is an entity. - Do NOT use `truncation` for content lost to interruption tags, even if the interruption tags are not at the exact location where the truncation occured — that is expected behavior, not an error. - + {language_guidance} ## Response Format Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the Intended Turns above: {{ @@ -804,7 +804,7 @@ judge: - **0 (Low Fidelity)**: One or more entities from the conversation trace are garbled, mispronounced, or indistinguishable in the audio. If the assistant does not speak any entities from the conversation trace in a turn (e.g., a greeting, filler, or turn where it only mentions entities not in the trace), set `has_entities` to false. These turns are excluded from scoring. - + {language_guidance} ## Response Format Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the Conversation Trace above: {{ @@ -886,7 +886,7 @@ judge: - **1 (Low Fidelity)**: - One or more entity errors (missing entities, incorrect entities, etc) OR - Some other major error that prevents the conversation from continuing in a sensible manner. - + {language_guidance} ## Response Format Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the Intended Turns above: {{ @@ -962,7 +962,7 @@ judge: - Honorific/title variants: "Ms." vs "Miss", "Mr." vs "Mister", and language equivalents (e.g., French "Madame" vs "Mme", Japanese "様" vs "-san") are acceptable variants of the same title. Important note: The expected text will often feature things formatted like "one two three" instead of "123". Your goal is to evaluate the semantic equivalence, meaning these are considered equivalent if they were heard in audio. - + {language_guidance} ## Examples **Example Input:** @@ -1140,7 +1140,7 @@ judge: - Large-unit grouping differences (e.g., Japanese 万-system: "ichi-man go-sen" / 一万五千 = 15,000) Important note: The user text will often feature things formatted like "one two three" instead of "123". Your goal is to evaluate semantic equivalence — these are considered equivalent if the value matches what the agent used. - + {language_guidance} ## Examples The user gives an id in turn 1 and corrects it in turn 2; nothing is rated at those user turns — each **tool call** is rated on its own. Tool Call [1] used the pre-correction value EM123 (wrong, because the user had already corrected it) and Tool Call [2] used the corrected value EM456 (right). Tool Call [3] uses only a `record_id` that came from a tool response, so it is omitted entirely. diff --git a/src/eva/metrics/accuracy/speech_fidelity.py b/src/eva/metrics/accuracy/speech_fidelity.py index ea34dfde3..4aa33db63 100644 --- a/src/eva/metrics/accuracy/speech_fidelity.py +++ b/src/eva/metrics/accuracy/speech_fidelity.py @@ -65,6 +65,7 @@ async def compute(self, context: MetricContext) -> MetricScore: audio_b64 = self.encode_audio_segment(audio_segment) prompt = self.get_judge_prompt( + language=context.language, conversation_trace_formatted=trace_formatted, expected_language=context.language_display_name, ) diff --git a/src/eva/metrics/base.py b/src/eva/metrics/base.py index dbd4948c1..50459f186 100644 --- a/src/eva/metrics/base.py +++ b/src/eva/metrics/base.py @@ -12,6 +12,7 @@ from pipecat.transcriptions.language import Language from pydub import AudioSegment +from eva.metrics.language_guidance import render_language_guidance from eva.metrics.utils import ( aggregate_per_turn_scores, audio_to_base64, @@ -210,14 +211,20 @@ def __init__(self, config: dict[str, Any] | None = None): self.logger = get_logger(f"metrics.{self.name}") self.prompt_manager = get_prompt_manager() - def get_judge_prompt(self, prompt_key: str = "user_prompt", **variables) -> str: + def get_judge_prompt(self, prompt_key: str = "user_prompt", *, language: str, **variables) -> str: """Get judge prompt using PromptManager. Stamps the unrendered template's sha256[:12] into the prompt-hash contextvar so any MetricScore built afterwards in the same compute() picks it up automatically. + ``language`` is required so that no caller silently drops the guidance: when the + language has judge guidance, it is rendered into ``{language_guidance}`` and + included in the hash. """ prompt_path = f"judge.{self.name}.{prompt_key}" - _CURRENT_PROMPT_HASH.set(hash_prompt_template(self.prompt_manager.get_template(prompt_path))) + template = self.prompt_manager.get_template(prompt_path) + guidance = render_language_guidance(self.name, language) + _CURRENT_PROMPT_HASH.set(hash_prompt_template(f"{template}\n{guidance}" if guidance else template)) + variables["language_guidance"] = guidance return self.prompt_manager.get_prompt(prompt_path, **variables) @abstractmethod @@ -412,7 +419,7 @@ async def compute(self, context: MetricContext) -> MetricScore: prompt_vars = self.get_prompt_variables(context, transcript_text) # Get and format prompt - prompt = self.get_judge_prompt(**prompt_vars) + prompt = self.get_judge_prompt(language=context.language, **prompt_vars) # Call judge response, raw_response = await self.call_judge(prompt, context) @@ -538,7 +545,9 @@ async def compute(self, context: MetricContext) -> MetricScore: if not transcript_text: return MetricScore(name=self.name, score=0.0, normalized_score=0.0, error="No turns to evaluate") - prompt = self.get_judge_prompt(**self.get_prompt_variables(context, transcript_text)) + prompt = self.get_judge_prompt( + language=context.language, **self.get_prompt_variables(context, transcript_text) + ) response_text, usage = await self.llm_client.generate_text([{"role": "user", "content": prompt}]) self._log_token_usage(context, self.llm_client.model, self.llm_client.params, prompt, usage, response_text) parsed = parse_judge_response_list(response_text) diff --git a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py index fef1618e6..602e7624b 100644 --- a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py +++ b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py @@ -247,11 +247,13 @@ def _build_prompt_and_turns(self, context: MetricContext) -> tuple[str, list[int """ if context.is_audio_native: conversation, tool_call_ids = self._build_audio_native_conversation(context) - prompt = self.get_judge_prompt(prompt_key="audio_native_prompt", conversation=conversation) + prompt = self.get_judge_prompt( + prompt_key="audio_native_prompt", language=context.language, conversation=conversation + ) return prompt, tool_call_ids turn_ids = self._get_turns_to_evaluate(context) - prompt = self.get_judge_prompt(user_turns=self._format_user_turns(turn_ids, context)) + prompt = self.get_judge_prompt(language=context.language, user_turns=self._format_user_turns(turn_ids, context)) return prompt, turn_ids @staticmethod diff --git a/src/eva/metrics/language_guidance.py b/src/eva/metrics/language_guidance.py new file mode 100644 index 000000000..fe0c72ee8 --- /dev/null +++ b/src/eva/metrics/language_guidance.py @@ -0,0 +1,61 @@ +"""Per-language guidance blocks injected into judge prompts. + +Each judge prompt template contains a ``{language_guidance}`` placeholder. For English it +renders to an empty string, so the rendered English prompt is unchanged. For other languages +it renders the guidance written by native speakers in +``configs/prompts/judge_language_guidance/.yaml``, a mapping of language code to +guidance text. A regional variant (e.g. ``fr-CA``) gets its base language's guidance +(``fr``) followed by its own entry, if any. +""" + +from functools import cache +from pathlib import Path + +import yaml + +from eva.models.config import LANGUAGE_DISPLAY_NAMES, base_language + +GUIDANCE_DIR = Path(__file__).parent.parent.parent.parent / "configs" / "prompts" / "judge_language_guidance" + +DISPLAY_NAMES_BY_CODE = {str(lang.value): name for lang, name in LANGUAGE_DISPLAY_NAMES.items()} + + +@cache +def _load_metric_guidance(metric_name: str) -> dict[str, str]: + path = GUIDANCE_DIR / f"{metric_name}.yaml" + if not path.is_file(): + return {} + with open(path, encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + if not isinstance(data, dict): + raise ValueError(f"{path}: expected a mapping of language code to guidance text, got {type(data).__name__}") + guidance = {} + for lang, text in data.items(): + if not isinstance(text, str): + raise ValueError(f"{path}: guidance for {lang!r} must be a string, got {type(text).__name__}") + if text.strip(): + guidance[str(lang)] = text.strip() + return guidance + + +def get_language_guidance_text(metric_name: str, language: str) -> str: + """Return the raw guidance text for a metric and language (empty if none).""" + guidance = _load_metric_guidance(metric_name) + base = base_language(language) + parts = [guidance[base]] if base in guidance else [] + if language != base and language in guidance: + parts.append(guidance[language]) + return "\n\n".join(parts) + + +def render_language_guidance(metric_name: str, language: str) -> str: + """Return the block that replaces ``{language_guidance}`` in a judge prompt (empty if none). + + The block starts with a newline so it is separated from the section before it. English + renders empty, which leaves the template's own blank line as it was. + """ + text = get_language_guidance_text(metric_name, language) + if not text: + return "" + display_name = DISPLAY_NAMES_BY_CODE.get(language, language) + return f"\n### {display_name}-specific guidance\n{text}\n" diff --git a/src/eva/metrics/signatures.py b/src/eva/metrics/signatures.py index bdcae19f7..a1a19016d 100644 --- a/src/eva/metrics/signatures.py +++ b/src/eva/metrics/signatures.py @@ -3,8 +3,8 @@ A metric's "signature" captures everything we want to detect changes to: - `version`: the manually-bumped string on the class - `source_hash`: sha256[:12] of `inspect.getsource(cls)` (class body) - - `prompt_hash`: sha256[:12] of the unrendered judge prompt template, or - None for non-judge metrics + - `prompt_hash`: sha256[:12] of the metric's unrendered judge prompt templates + plus its per-language guidance file, or None for non-judge metrics The drift test compares the current signatures against a checked-in fixture and fails if anything changed without an explicit version bump + fixture regen. @@ -20,6 +20,7 @@ import eva.metrics.experience # noqa: F401 import eva.metrics.validation # noqa: F401 from eva.metrics.base import AudioJudgeMetric, BaseMetric, TextJudgeMetric +from eva.metrics.language_guidance import GUIDANCE_DIR from eva.metrics.versioning import hash_prompt_template from eva.utils.prompt_manager import get_prompt_manager @@ -67,6 +68,8 @@ def _prompt_hash_for_metric(cls: type[BaseMetric]) -> str | None: Hashes *every* prompt template under ``judge.{name}`` (sorted by key), not just ``user_prompt``, so a metric that ships multiple prompts — e.g. a per-pipeline variant like ``audio_native_prompt`` — has all of them covered by drift detection. + The metric's guidance file in ``configs/prompts/judge_language_guidance/`` is + included too, since it is rendered into those templates. A judge metric without any templates raises KeyError — that's a configuration bug we want surfaced. """ @@ -76,6 +79,9 @@ def _prompt_hash_for_metric(cls: type[BaseMetric]) -> str | None: if not isinstance(templates, dict): raise KeyError(f"No judge prompts found for metric {cls.name!r}") combined = "\n".join(templates[key] for key in sorted(templates) if isinstance(templates[key], str)) + guidance_file = GUIDANCE_DIR / f"{cls.name}.yaml" + if guidance_file.is_file(): + combined += "\n" + guidance_file.read_text(encoding="utf-8") return hash_prompt_template(combined) diff --git a/src/eva/metrics/speech_fidelity_base.py b/src/eva/metrics/speech_fidelity_base.py index d6ca93e04..7241463f0 100644 --- a/src/eva/metrics/speech_fidelity_base.py +++ b/src/eva/metrics/speech_fidelity_base.py @@ -67,6 +67,7 @@ async def compute(self, context: MetricContext) -> MetricScore: intended_turns_formatted = self._format_intended_turns(intended_turns) prompt = self.get_judge_prompt( + language=context.language, intended_turns_formatted=intended_turns_formatted, expected_language=context.language_display_name, ) diff --git a/tests/fixtures/judge_prompts_en.json b/tests/fixtures/judge_prompts_en.json new file mode 100644 index 000000000..c16678fb3 --- /dev/null +++ b/tests/fixtures/judge_prompts_en.json @@ -0,0 +1,12 @@ +{ + "agent_speech_fidelity.user_prompt": "55128fc6e9d3", + "conciseness.user_prompt": "a0ac0a5fb1b1", + "conversation_progression.user_prompt": "c45acc2bc604", + "faithfulness.user_prompt": "9cedbcc31449", + "speakability.user_prompt": "3e194cc6b73a", + "transcription_accuracy_key_entities.audio_native_prompt": "315699fd0211", + "transcription_accuracy_key_entities.user_prompt": "bef4cfeb31b0", + "tts_fidelity.user_prompt": "17eb8a429b2c", + "user_behavioral_fidelity.user_prompt": "5f6361e25eba", + "user_speech_fidelity.user_prompt": "969c56fe32d5" +} diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 55d516e39..447d371b0 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -7,8 +7,8 @@ }, "ConcisenessJudgeMetric": { "name": "conciseness", - "prompt_hash": "57caf9a18a3c", - "source_hash": "9f183871e45a", + "prompt_hash": "5fdd118b6fd8", + "source_hash": "7d251695fd2f", "version": "v0.1" }, "ConversationCorrectlyFinishedMetric": { @@ -19,8 +19,8 @@ }, "ConversationProgressionJudgeMetric": { "name": "conversation_progression", - "prompt_hash": "174cdf93b398", - "source_hash": "84e43143d00d", + "prompt_hash": "f26237934000", + "source_hash": "0edf89e90978", "version": "v0.2" }, "ConversationValidEndMetric": { @@ -31,8 +31,8 @@ }, "FaithfulnessJudgeMetric": { "name": "faithfulness", - "prompt_hash": "78028821ce37", - "source_hash": "ee9b8ea530a4", + "prompt_hash": "741590f3d7c1", + "source_hash": "b2cf38e3fbc7", "version": "v0.3" }, "ResponseSpeedMetric": { @@ -49,20 +49,20 @@ }, "SpeakabilityJudgeMetric": { "name": "speakability", - "prompt_hash": "1e4f78cb051e", - "source_hash": "7abcd995909a", + "prompt_hash": "9d39f642366b", + "source_hash": "b76318002d8f", "version": "v0.3" }, "SpeechFidelityMetric": { "name": "agent_speech_fidelity", - "prompt_hash": "c614dd12d4fe", - "source_hash": "5e61505dc729", + "prompt_hash": "889e34a06a58", + "source_hash": "aba2c27cb6af", "version": "v0.4" }, "TTSFidelityMetric": { "name": "tts_fidelity", - "prompt_hash": "c3a02ab03f06", - "source_hash": "10b97af355f7", + "prompt_hash": "99e126158b75", + "source_hash": "317128c2ea72", "version": "v0.3" }, "TaskCompletion": { @@ -79,8 +79,8 @@ }, "TranscriptionAccuracyKeyEntitiesMetric": { "name": "transcription_accuracy_key_entities", - "prompt_hash": "8492d1f0d168", - "source_hash": "018436ab79ac", + "prompt_hash": "dcf0fe2feed6", + "source_hash": "e8e81aec84d5", "version": "v0.4" }, "TurnTakingMetric": { @@ -92,13 +92,13 @@ "UserBehavioralFidelityMetric": { "name": "user_behavioral_fidelity", "prompt_hash": "21527fa8d79b", - "source_hash": "7f5dcec51568", + "source_hash": "b8a700ea5b89", "version": "v0.1" }, "UserSpeechFidelityMetric": { "name": "user_speech_fidelity", - "prompt_hash": "849e13c42e3b", - "source_hash": "5463b901a11b", + "prompt_hash": "1127744357ae", + "source_hash": "6625af8e00f9", "version": "v0.2" } } diff --git a/tests/unit/metrics/test_language_guidance.py b/tests/unit/metrics/test_language_guidance.py new file mode 100644 index 000000000..e66320962 --- /dev/null +++ b/tests/unit/metrics/test_language_guidance.py @@ -0,0 +1,149 @@ +"""Tests for per-language judge guidance injection.""" + +import hashlib +import json +import os +import string +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from eva.metrics import language_guidance +from eva.metrics.base import BaseMetric +from eva.metrics.experience.conciseness import ConcisenessJudgeMetric +from eva.metrics.language_guidance import ( + DISPLAY_NAMES_BY_CODE, + GUIDANCE_DIR, + get_language_guidance_text, + render_language_guidance, +) +from eva.metrics.versioning import _CURRENT_PROMPT_HASH +from eva.models.config import base_language +from eva.utils.prompt_manager import get_prompt_manager + +from .conftest import make_judge_metric + +# Scores the simulated user against its own instructions, so it has no guidance slot. +JUDGES_WITHOUT_GUIDANCE = {"user_behavioral_fidelity"} + +GOLDEN_EN_PATH = Path(__file__).resolve().parents[3] / "tests" / "fixtures" / "judge_prompts_en.json" +REGEN_GOLDEN_ENV = "EVA_REGEN_JUDGE_PROMPT_GOLDEN" + + +@pytest.fixture +def guidance_dir(tmp_path, monkeypatch): + (tmp_path / "some_metric.yaml").write_text( + yaml.safe_dump({"fr": "French note.", "fr-CA": "Quebec note.", "de": " "}), encoding="utf-8" + ) + monkeypatch.setattr(language_guidance, "GUIDANCE_DIR", tmp_path) + language_guidance._load_metric_guidance.cache_clear() + yield tmp_path + language_guidance._load_metric_guidance.cache_clear() + + +def test_english_and_missing_metric_render_empty(guidance_dir): + assert render_language_guidance("some_metric", "en") == "" + assert render_language_guidance("unknown_metric", "fr") == "" + assert render_language_guidance("some_metric", "de") == "" + + +def test_regional_variant_gets_base_then_own_guidance(guidance_dir): + assert get_language_guidance_text("some_metric", "fr") == "French note." + assert get_language_guidance_text("some_metric", "fr-CA") == "French note.\n\nQuebec note." + + +def test_rendered_block_has_header(guidance_dir): + assert render_language_guidance("some_metric", "fr") == "\n### European French-specific guidance\nFrench note.\n" + + +@pytest.mark.parametrize( + "content, error", + [ + ("- one bullet\n- another bullet\n", "expected a mapping"), + ("fr:\n - one bullet\n", "must be a string"), + ], +) +def test_malformed_guidance_file_raises(guidance_dir, content, error): + (guidance_dir / "bad_metric.yaml").write_text(content, encoding="utf-8") + with pytest.raises(ValueError, match=error): + get_language_guidance_text("bad_metric", "fr") + + +def _judge_templates() -> list[tuple[str, str, str]]: + judge = get_prompt_manager().prompts["judge"] + return [ + (metric, key, template) + for metric, prompts in judge.items() + if isinstance(prompts, dict) + for key, template in prompts.items() + if isinstance(template, str) + ] + + +def test_every_judge_template_has_guidance_slot(): + for metric, key, template in _judge_templates(): + if metric in JUDGES_WITHOUT_GUIDANCE: + assert "{language_guidance}" not in template, f"judge.{metric}.{key} should not have a guidance slot" + else: + assert "{language_guidance}" in template, f"judge.{metric}.{key} lacks {{language_guidance}}" + + +def test_every_guidance_file_is_valid(): + judge = get_prompt_manager().prompts["judge"] + known_languages = {code for code in DISPLAY_NAMES_BY_CODE if base_language(code) != "en"} + for path in GUIDANCE_DIR.glob("*.yaml"): + metric = path.stem + assert metric in judge, f"{path.name} has no judge prompt" + assert metric not in JUDGES_WITHOUT_GUIDANCE, f"{metric} prompt has no guidance slot" + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + assert isinstance(data, dict), f"{path.name} must be a mapping of language code to text" + for lang, text in data.items(): + assert lang in known_languages, f"{path.name}: {lang!r} is not a supported non-English language code" + assert isinstance(text, str), f"{path.name}: guidance for {lang!r} must be a string" + assert "{" not in text and "}" not in text, f"{path.name}: braces are not allowed in guidance text" + + +def test_get_judge_prompt_injects_guidance_and_changes_hash(tmp_path, monkeypatch): + metric = make_judge_metric(ConcisenessJudgeMetric) + (tmp_path / f"{metric.name}.yaml").write_text(yaml.safe_dump({"fr": "French note."}), encoding="utf-8") + monkeypatch.setattr(language_guidance, "GUIDANCE_DIR", tmp_path) + language_guidance._load_metric_guidance.cache_clear() + try: + english = metric.get_judge_prompt(language="en", conversation_turns="") + english_hash = _CURRENT_PROMPT_HASH.get() + french = metric.get_judge_prompt(language="fr", conversation_turns="") + french_hash = _CURRENT_PROMPT_HASH.get() + finally: + language_guidance._load_metric_guidance.cache_clear() + + assert "### European French-specific guidance\nFrench note.\n" in french + assert "specific guidance" not in english + assert french_hash != english_hash + + +def _render_english_judge_prompts() -> dict[str, str]: + """Render every judge template in English with placeholder values, keyed by ``metric.prompt_key``.""" + prompt_manager = get_prompt_manager() + hashes = {} + for metric, key, template in _judge_templates(): + fields = {name for _, name, _, _ in string.Formatter().parse(template) if name} + variables = {name: f"<{name}>" for name in fields - {"language_guidance"}} + stub = SimpleNamespace(name=metric, prompt_manager=prompt_manager) + rendered = BaseMetric.get_judge_prompt(stub, key, language="en", **variables) + hashes[f"{metric}.{key}"] = hashlib.sha256(rendered.encode("utf-8")).hexdigest()[:12] + return hashes + + +def test_english_judge_prompts_match_golden(): + """English judge prompts are byte-identical to the golden renders. + + An intended change to an English prompt must update the golden file: + ``EVA_REGEN_JUDGE_PROMPT_GOLDEN=1 uv run pytest tests/unit/metrics/test_language_guidance.py``. + """ + current = _render_english_judge_prompts() + if os.environ.get(REGEN_GOLDEN_ENV): + GOLDEN_EN_PATH.write_text(json.dumps(current, indent=2, sort_keys=True) + "\n") + golden = json.loads(GOLDEN_EN_PATH.read_text()) + assert current == golden, f"English judge prompts changed. If intended, regenerate with {REGEN_GOLDEN_ENV}=1." diff --git a/tests/unit/metrics/test_metric_signatures.py b/tests/unit/metrics/test_metric_signatures.py index 24ce61c5d..4998c54ab 100644 --- a/tests/unit/metrics/test_metric_signatures.py +++ b/tests/unit/metrics/test_metric_signatures.py @@ -3,7 +3,8 @@ Each concrete metric class has three signature fields: - version: manually bumped string on the class - source_hash: sha256[:12] of inspect.getsource(cls) - - prompt_hash: sha256[:12] of judge.{name}.user_prompt template (None for code metrics) + - prompt_hash: sha256[:12] of all judge.{name}.* templates plus the metric's + language guidance file (None for code metrics) The fixture at tests/fixtures/metric_signatures.json is the source of truth for the *currently-released* state. The test compares the current signatures to it