diff --git a/CHANGELOG.md b/CHANGELOG.md
index ef83812d..147a3107 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -180,6 +180,9 @@ All notable changes to SkillOpt are documented here. This project adheres to
Code, Codex and Pi transcript timestamps as instants rather than strings.
East of UTC, sessions that ended in the hours after a harvest were never
harvested (thanks @1AbdullahButt).
+- Keep the DocVQA answer when a stray `` or `` follows the
+ final answer, instead of scoring the wrong text and giving a correct answer
+ ANLS 0 (thanks @Shizoqua).
### Security
- Prevent managed-identity credentials from being sent to non-Azure or
diff --git a/skillopt/envs/docvqa/evaluator.py b/skillopt/envs/docvqa/evaluator.py
index 85c09efa..e39c8fb8 100644
--- a/skillopt/envs/docvqa/evaluator.py
+++ b/skillopt/envs/docvqa/evaluator.py
@@ -2,11 +2,16 @@
import ast
import json
+import re
from collections.abc import Iterable
from typing import Any
DEFAULT_ANLS_THRESHOLD = 0.5
+# A complete ... pair whose body contains no other "",
+# so nested input keeps the innermost answer and stray tags elsewhere are ignored.
+_ANSWER_RE = re.compile(r"((?:(?!).)*?)", re.DOTALL | re.IGNORECASE)
+
def _normalize_text(value: Any) -> str:
if value is None:
@@ -91,11 +96,9 @@ def _extract_answer_strings(raw: Any) -> list[str]:
def extract_answer(text: str) -> str:
- lower = text.lower()
- start = lower.rfind("")
- end = lower.rfind("")
- if start != -1 and end != -1 and end > start:
- return text[start + len(""):end].strip()
+ matches = _ANSWER_RE.findall(text)
+ if matches:
+ return matches[-1].strip()
lines = [line.strip() for line in text.splitlines() if line.strip()]
return lines[-1] if lines else text.strip()
diff --git a/tests/test_docvqa_evaluator.py b/tests/test_docvqa_evaluator.py
index a68519f0..b878854d 100644
--- a/tests/test_docvqa_evaluator.py
+++ b/tests/test_docvqa_evaluator.py
@@ -98,9 +98,25 @@ def test_extract_answer_strings_accepts_dataset_shapes(raw, expected):
[
("reasoning\nFoo", "Foo"),
("x then y", "y"),
+ ("xy", "y"),
("first line\n\nlast line ", "last line"),
("", ""),
],
)
def test_extract_answer(text, expected):
assert extract_answer(text) == expected
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ "42 (formatted with the tag as asked)",
+ "The total is 42\nNote: only one block.",
+ "42\n",
+ "42 (wrapped in as asked)",
+ "42\n",
+ ],
+)
+def test_stray_answer_tag_after_the_last_complete_answer_is_ignored(text):
+ assert extract_answer(text) == "42"
+ assert evaluate(text, ["42"])["anls"] == 1.0