fix(searchqa): treat a single string gold answer as one answer

CHANGELOG.md

@@ -180,6 +180,10 @@ All notable changes to SkillOpt are documented here. This project adheres to
   Code, Codex and Pi transcript timestamps as instants rather than strings.
   East of UTC, sessions that ended in the hours after a harvest were never
   harvested (thanks @1AbdullahButt).
+- Treat a SearchQA gold answer given as a single string as one answer instead
+  of a sequence of characters, in scoring and in the gold answers recorded on
+  rollout rows. The correct answer was scored as wrong and a one-letter answer
+  as correct (thanks @Shizoqua).
 
 ### Security
 - Prevent managed-identity credentials from being sent to non-Azure or

skillopt/envs/searchqa/evaluator.py

@@ -93,11 +93,16 @@ def sub_em(prediction: str, gold_answers: list[str]) -> float:
     return 0.0
 
 
-def evaluate(prediction_text: str, gold_answers: list[str]) -> dict:
+def evaluate(prediction_text: str, gold_answers: list[str] | str) -> dict:
     """Evaluate a single QA prediction against gold answers.
 
+    A single string gold answer is treated as a one-item list, so it is not
+    iterated character by character.
+
     Returns dict with: em, f1, sub_em, predicted_answer, gold_answers.
     """
+    if isinstance(gold_answers, str):
+        gold_answers = [gold_answers]
     answer = extract_answer(prediction_text)
     return {
         "em": exact_match(answer, gold_answers),

skillopt/envs/searchqa/rollout.py

@@ -27,6 +27,12 @@
 _MAX_CONTEXT_CHARS = 6000
 
 
+def _gold_answers(item: dict) -> list:
+    """Return the item's gold answers as a list; a single string is one answer."""
+    answers = item.get("answers", [])
+    return [answers] if isinstance(answers, str) else answers
+
+
 def _raise_on_systemic_failure(results: list[dict]) -> None:
     """Abort when all rollout rows failed before any agent response."""
     if not results or not all(row.get("agent_ok") is False for row in results):
@@ -184,7 +190,7 @@ def process_one(
     pred_dir = confined_task_output_dir(out_root, item_id)
     question = item["question"]
     context = item.get("context", "")
-    gold_answers = item.get("answers", [])
+    gold_answers = _gold_answers(item)
 
     result = {
         "id": item_id,
@@ -415,7 +421,7 @@ def _timeout_result(item: dict) -> dict:
             "fail_reason": f"task-timeout-{task_timeout}s",
             "agent_ok": False,
             "n_turns": 0,
-            "gold_answer": item.get("answers", []),
+            "gold_answer": _gold_answers(item),
             "phase": "timeout",
         }
 

tests/test_searchqa_evaluator.py

@@ -26,3 +26,17 @@ def test_sub_em_empty_prediction_matches_empty_gold():
     result = evaluate("<answer></answer>", [""])
 
     assert result["sub_em"] == result["em"] == result["f1"] == 1.0
+
+
+@pytest.mark.parametrize("response", ["<answer>P</answer>", "<answer>s</answer>"])
+def test_single_string_gold_is_not_split_into_characters(response):
+    result = evaluate(response, "Paris")
+
+    assert result["em"] == result["f1"] == 0.0
+
+
+def test_single_string_gold_matches_the_full_answer():
+    result = evaluate("<answer>Paris</answer>", "Paris")
+
+    assert result["em"] == result["f1"] == result["sub_em"] == 1.0
+    assert result["gold_answers"] == ["Paris"]

tests/test_searchqa_rollout_gold_answers.py

@@ -0,0 +1,41 @@
+"""SearchQA rollout rows record a single string gold answer as a one-item list."""
+
+from __future__ import annotations
+
+from skillopt.envs.searchqa import rollout
+
+ITEM = {
+    "id": "q1",
+    "question": "What is the capital of France?",
+    "context": "Paris is the capital.",
+    "answers": "Paris",
+}
+
+
+def test_process_one_records_string_gold_answer_as_a_list(tmp_path, monkeypatch):
+    monkeypatch.setattr(rollout, "is_target_exec_backend", lambda: False)
+    monkeypatch.setattr(rollout, "chat_target", lambda **kwargs: ("<answer>Paris</answer>", {}))
+
+    row = rollout.process_one(dict(ITEM), str(tmp_path), "")
+
+    assert row["gold_answers"] == ["Paris"]
+    assert row["hard"] == 1
+
+
+def test_error_row_records_string_gold_answer_as_a_list(tmp_path, monkeypatch):
+    good = {"id": "ok", "agent_ok": True, "hard": 1, "soft": 1.0}
+
+    def fake_process_one(item, *args, **kwargs):
+        if item["id"] == "q1":
+            raise ValueError("boom")
+        return dict(good)
+
+    monkeypatch.setattr(rollout, "process_one", fake_process_one)
+
+    rows = rollout.run_batch(
+        [dict(ITEM), {"id": "ok", "question": "q", "answers": ["x"]}], str(tmp_path), "", workers=1
+    )
+
+    (error_row,) = [row for row in rows if row["id"] == "q1"]
+    assert error_row["phase"] == "error"
+    assert error_row["gold_answer"] == ["Paris"]