From 3e888414afe057caf3567b6df3bc2ad63dfc50e4 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:51:43 +0000 Subject: [PATCH 1/2] fix(docvqa): keep the answer when a stray follows it extract_answer took the last "" and the last "" and only used them when the closing tag came after the opening one. If the response mentioned "" again after the final answer, for example "42 (formatted with the tag as asked)", the pair was rejected and the last line of text was scored instead, so a correct answer got ANLS 0. ANLS is the DocVQA soft score and hard is ANLS >= 0.999, so both the training reward and reported scores dropped. Pair the last "" with the last "" before it. This stays a linear scan, and nested input such as "xy" still returns "y" as before. --- CHANGELOG.md | 3 +++ skillopt/envs/docvqa/evaluator.py | 6 ++++-- tests/test_docvqa_evaluator.py | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef83812d9..06aaac42a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,6 +180,9 @@ All notable changes to SkillOpt are documented here. This project adheres to Code, Codex and Pi transcript timestamps as instants rather than strings. East of UTC, sessions that ended in the hours after a harvest were never harvested (thanks @1AbdullahButt). +- Keep the DocVQA answer when a stray `` appears after the last + ``, instead of falling back to the last line and scoring a correct + answer as ANLS 0 (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/envs/docvqa/evaluator.py b/skillopt/envs/docvqa/evaluator.py index 85c09efa7..4cffe6cf4 100644 --- a/skillopt/envs/docvqa/evaluator.py +++ b/skillopt/envs/docvqa/evaluator.py @@ -92,9 +92,11 @@ def _extract_answer_strings(raw: Any) -> list[str]: def extract_answer(text: str) -> str: lower = text.lower() - start = lower.rfind("") + # Pair the last with the last before it, so a stray + # "" after the final answer does not discard that answer. end = lower.rfind("") - if start != -1 and end != -1 and end > start: + start = lower.rfind("", 0, end) if end != -1 else -1 + if start != -1: return text[start + len(""):end].strip() lines = [line.strip() for line in text.splitlines() if line.strip()] return lines[-1] if lines else text.strip() diff --git a/tests/test_docvqa_evaluator.py b/tests/test_docvqa_evaluator.py index a68519f0b..01516e377 100644 --- a/tests/test_docvqa_evaluator.py +++ b/tests/test_docvqa_evaluator.py @@ -98,9 +98,23 @@ def test_extract_answer_strings_accepts_dataset_shapes(raw, expected): [ ("reasoning\nFoo", "Foo"), ("x then y", "y"), + ("xy", "y"), ("first line\n\nlast line ", "last line"), ("", ""), ], ) def test_extract_answer(text, expected): assert extract_answer(text) == expected + + +@pytest.mark.parametrize( + "text", + [ + "42 (formatted with the tag as asked)", + "The total is 42\nNote: only one block.", + "42\n", + ], +) +def test_stray_answer_tag_after_the_last_complete_answer_is_ignored(text): + assert extract_answer(text) == "42" + assert evaluate(text, ["42"])["anls"] == 1.0 From d39f3c30c5a9cd11a15a47cf4ca30a5df7ec42a7 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:11:29 +0000 Subject: [PATCH 2/2] fix(docvqa): also ignore a stray after the final answer Pairing the last with the last before it still broke on a stray closing tag: "42 (wrapped in as asked)" extracted "42 (wrapped in" and scored ANLS 0. Match complete ... pairs whose body contains no other and take the last one. That handles a stray tag in either direction, keeps "xy" returning "y", and stays fast on long or repetitive input. --- CHANGELOG.md | 6 +++--- skillopt/envs/docvqa/evaluator.py | 15 ++++++++------- tests/test_docvqa_evaluator.py | 2 ++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06aaac42a..147a31070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,9 +180,9 @@ All notable changes to SkillOpt are documented here. This project adheres to Code, Codex and Pi transcript timestamps as instants rather than strings. East of UTC, sessions that ended in the hours after a harvest were never harvested (thanks @1AbdullahButt). -- Keep the DocVQA answer when a stray `` appears after the last - ``, instead of falling back to the last line and scoring a correct - answer as ANLS 0 (thanks @Shizoqua). +- Keep the DocVQA answer when a stray `` or `` follows the + final answer, instead of scoring the wrong text and giving a correct answer + ANLS 0 (thanks @Shizoqua). ### Security - Prevent managed-identity credentials from being sent to non-Azure or diff --git a/skillopt/envs/docvqa/evaluator.py b/skillopt/envs/docvqa/evaluator.py index 4cffe6cf4..e39c8fb8f 100644 --- a/skillopt/envs/docvqa/evaluator.py +++ b/skillopt/envs/docvqa/evaluator.py @@ -2,11 +2,16 @@ import ast import json +import re from collections.abc import Iterable from typing import Any DEFAULT_ANLS_THRESHOLD = 0.5 +# A complete ... pair whose body contains no other "", +# so nested input keeps the innermost answer and stray tags elsewhere are ignored. +_ANSWER_RE = re.compile(r"((?:(?!).)*?)", re.DOTALL | re.IGNORECASE) + def _normalize_text(value: Any) -> str: if value is None: @@ -91,13 +96,9 @@ def _extract_answer_strings(raw: Any) -> list[str]: def extract_answer(text: str) -> str: - lower = text.lower() - # Pair the last with the last before it, so a stray - # "" after the final answer does not discard that answer. - end = lower.rfind("") - start = lower.rfind("", 0, end) if end != -1 else -1 - if start != -1: - return text[start + len(""):end].strip() + matches = _ANSWER_RE.findall(text) + if matches: + return matches[-1].strip() lines = [line.strip() for line in text.splitlines() if line.strip()] return lines[-1] if lines else text.strip() diff --git a/tests/test_docvqa_evaluator.py b/tests/test_docvqa_evaluator.py index 01516e377..b878854d5 100644 --- a/tests/test_docvqa_evaluator.py +++ b/tests/test_docvqa_evaluator.py @@ -113,6 +113,8 @@ def test_extract_answer(text, expected): "42 (formatted with the tag as asked)", "The total is 42\nNote: only one block.", "42\n", + "42 (wrapped in as asked)", + "42\n", ], ) def test_stray_answer_tag_after_the_last_complete_answer_is_ignored(text):