From ede579e088e3a43ae1704053279d912eb6cfc7e8 Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Fri, 4 Sep 2026 07:19:39 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Support=20parenthesis-delimited=20b?= =?UTF-8?q?locks=20such=20as=20`@article(...)`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standard BibTeX allows `(...)` as block delimiters, but the splitter only recognised `{...}`, so such blocks were silently swallowed as implicit comments (#533). They now parse like brace-delimited blocks for entries, `@string`, `@preamble` and `@comment`. Co-Authored-By: Claude Fable 5.1 --- bibtexparser/splitter.py | 159 ++++++------- .../test_splitter_parenthesis_blocks.py | 209 ++++++++++++++++++ 2 files changed, 290 insertions(+), 78 deletions(-) create mode 100644 tests/splitter_tests/test_splitter_parenthesis_blocks.py diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index a6b5ae9..e8f153c 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -3,7 +3,6 @@ from .exceptions import BlockAbortedException from .exceptions import ParserStateException -from .exceptions import RegexMismatchException from .library import Library from .model import DuplicateFieldKeyBlock from .model import Entry @@ -16,6 +15,16 @@ logger = logging.getLogger(__name__) +# "Marks" are the characters the splitter jumps between: value delimiters, field +# separators, newlines (for line counting) and block starts (`@type` followed by `{` or `(`). +# The opening delimiter is not part of the block start mark, so that a `{` after an +# `@` within a value (e.g. `LeQua @ {CLEF}`) is still counted as a mark of its own. +_BLOCK_START = r"@[\w]*( |\t)*(?=[{(])" +_MARK_PATTERN = re.compile(r"(? re.Match | None: self._current_line += 1 - def _move_to_closed_bracket(self) -> int: - """Index of the curly bracket closing a just opened one.""" - num_additional_brackets = 0 + def _open_block(self, m: re.Match) -> None: + """Consume the delimiter opening the block started by mark `m`, and set the closing one.""" + if self.bibstr[m.end()] == "(": + self._closing_delimiter = ")" + # `(` is not a mark, hence the block-specific marks start right after it. + self._markiter = _PAREN_BLOCK_MARK_PATTERN.finditer(self.bibstr, m.end() + 1) + else: + self._closing_delimiter = "}" + # The `{` is a mark (guaranteed to be the next one by the block start regex) + self._next_mark(accept_eof=False) + + def _close_block(self) -> None: + """Undo `_open_block` once the block was parsed (or its parsing aborted).""" + if self._closing_delimiter == ")": + # Resume default marks right after the last consumed mark, + # which is either put back by an abort, or a single character. + if self._unaccepted_mark is not None: + resume_index = self._unaccepted_mark.end() + else: + resume_index = self._current_char_index + 1 + self._markiter = _MARK_PATTERN.finditer(self.bibstr, resume_index) + self._closing_delimiter = "}" + + def _move_to_closing_delimiter(self, track_quotes: bool) -> int: + """Index of the delimiter closing the current block, skipping nested `{...}`. + + With `track_quotes`, a `)` within a `"..."` string is skipped as well: + Unlike `}`, which must be balanced within such strings, a `)` may occur there. + Free-text blocks (comments) do not track quotes, as these may be unbalanced there. + """ + num_open_curls = 0 + in_quotes = False + track_quotes = track_quotes and self._closing_delimiter == ")" while True: m = self._next_mark(accept_eof=False) if m.group(0) == "{": - num_additional_brackets += 1 - elif m.group(0) == "}": - if num_additional_brackets == 0: - return m.start() - else: - num_additional_brackets -= 1 + num_open_curls += 1 + elif m.group(0) == "}" and num_open_curls > 0: + num_open_curls -= 1 + elif num_open_curls == 0 and m.group(0) == '"' and track_quotes: + in_quotes = not in_quotes + elif num_open_curls == 0 and m.group(0) == self._closing_delimiter and not in_quotes: + return m.start() elif m.group(0).startswith("@") and self._is_at_line_start(m.start()): # Only abort if the @ is at the start of a line. # This allows @ signs in field values (e.g., "LeQua @ {CLEF}") @@ -144,7 +187,7 @@ def _move_to_closed_bracket(self) -> int: end_index=m.start() - 1, ) - def _move_to_comma_or_closing_curly_bracket( + def _move_to_comma_or_closing_delimiter( self, currently_quote_escaped: bool = False, num_open_curls: int = 0 ) -> int: """Index of the end of the field, taking quote-escape into account.""" @@ -197,7 +240,7 @@ def _is_escaped(): self._unaccepted_mark = next_mark return next_mark.start() # Check for end of entry: - elif next_mark.group(0) == "}" and not _is_escaped(): + elif next_mark.group(0) == self._closing_delimiter and not _is_escaped(): self._unaccepted_mark = next_mark return next_mark.start() @@ -213,7 +256,7 @@ def _is_escaped(): elif num_open_curls > 0: looking_for = "`}`" else: - looking_for = "`,` or `}`" + looking_for = f"`,` or `{self._closing_delimiter}`" raise BlockAbortedException( abort_reason=f"Unexpected block start: `{next_mark.group(0)}`. " @@ -230,12 +273,12 @@ def _move_to_end_of_entry(self, first_key_start: int) -> tuple[list[Field], int, key_start = first_key_start while True: equals_mark = self._next_mark(accept_eof=False) - if equals_mark.group(0) == "}": + if equals_mark.group(0) == self._closing_delimiter: dangling_key = self.bibstr[key_start : equals_mark.start()].strip() if dangling_key: raise BlockAbortedException( abort_reason=f"Expected a `=` after entry key `{dangling_key}`, " - "but found the end of the entry (`}`).", + f"but found the end of the entry (`{self._closing_delimiter}`).", end_index=equals_mark.end(), ) # End of entry @@ -254,7 +297,7 @@ def _move_to_end_of_entry(self, first_key_start: int) -> tuple[list[Field], int, start_line = self._current_line key_end = equals_mark.start() value_start = equals_mark.end() - value_end = self._move_to_comma_or_closing_curly_bracket( + value_end = self._move_to_comma_or_closing_delimiter( currently_quote_escaped=False, num_open_curls=0 ) @@ -271,8 +314,8 @@ def _move_to_end_of_entry(self, first_key_start: int) -> tuple[list[Field], int, after_field_mark = self._next_mark(accept_eof=False) if after_field_mark.group(0) == ",": key_start = after_field_mark.end() - elif after_field_mark.group(0) == "}": - # If next mark is a closing bracket, put it back (will return in next loop iteration) + elif after_field_mark.group(0) == self._closing_delimiter: + # If next mark is the closing delimiter, put it back (will return in next loop iteration) self._unaccepted_mark = after_field_mark # Advance past the value, else the check above aborts a valid entry. key_start = after_field_mark.start() @@ -280,8 +323,8 @@ def _move_to_end_of_entry(self, first_key_start: int) -> tuple[list[Field], int, else: self._unaccepted_mark = after_field_mark raise BlockAbortedException( - abort_reason="Expected either a `,` or `}` after a closed entry field value, " - f"but found a {after_field_mark.group(0)} before.", + abort_reason=f"Expected either a `,` or `{self._closing_delimiter}` " + f"after a closed entry field value, but found a {after_field_mark.group(0)} before.", end_index=after_field_mark.start(), ) @@ -291,9 +334,7 @@ def split(self) -> Library: Returns: A new library containing the split blocks. """ - self._markiter = re.finditer( - r"(? Library: start_line = self._current_line try: # Start new block parsing + self._open_block(m) if m_val.startswith("@comment"): - library.add(self._handle_explicit_comment(), fail_on_duplicate_key=False) + library.add(self._handle_explicit_comment(m), fail_on_duplicate_key=False) elif m_val.startswith("@preamble"): - library.add(self._handle_preamble(), fail_on_duplicate_key=False) + library.add(self._handle_preamble(m), fail_on_duplicate_key=False) elif m_val.startswith("@string"): library.add(self._handle_string(m), fail_on_duplicate_key=False) else: @@ -357,6 +399,7 @@ def split(self) -> Library: ) raise + self._close_block() self._reset_block_status(current_char_index=self._current_char_index + 1) else: # Part of implicit comment @@ -370,42 +413,22 @@ def split(self) -> Library: return library - def _handle_explicit_comment(self) -> ExplicitComment: + def _handle_explicit_comment(self, m) -> ExplicitComment: """Handle explicit comment block. Return end index""" - start_index = self._current_char_index start_line = self._current_line - start_bracket_mark = self._next_mark(accept_eof=False) - if start_bracket_mark.group(0) != "{": - self._unaccepted_mark = start_bracket_mark - # Note: The following should never happen, as we check for the "{" in the regex - raise RegexMismatchException( - first_match="@comment{", - expected_match="{", - second_match=start_bracket_mark.group(0), - ) - end_bracket_index = self._move_to_closed_bracket() - comment_str = self.bibstr[start_bracket_mark.end() : end_bracket_index].strip() + end_index = self._move_to_closing_delimiter(track_quotes=False) return ExplicitComment( start_line=start_line, - comment=comment_str, - raw=self.bibstr[start_index : end_bracket_index + 1], + comment=self.bibstr[m.end() + 1 : end_index].strip(), + raw=self.bibstr[m.start() : end_index + 1], ) def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: """Handle entry block. Return end index""" start_line = self._current_line entry_type = m_val[1:].strip() - start_bracket_mark = self._next_mark(accept_eof=False) - if start_bracket_mark.group(0) != "{": - self._unaccepted_mark = start_bracket_mark - # Note: The following should never happen, as we check for the "{" in the regex - raise ParserStateException( - message="matched a regex that should end with `{`, " - "e.g. `@article{`, " - "but no closing bracket was found." - ) comma_mark = self._next_mark(accept_eof=False) - if comma_mark.group(0) == "}": + if comma_mark.group(0) == self._closing_delimiter: # This is an entry without any comma after the key, and with no fields # Used e.g. by RefTeX (see issue #384) key = self.bibstr[m.end() + 1 : comma_mark.start()].strip() @@ -436,17 +459,8 @@ def _handle_entry(self, m, m_val) -> Entry | ParsingFailedBlock: def _handle_string(self, m) -> String: """Handle string block. Return end index""" - # Get next mark, which should be an equals sign - start_i = self._current_char_index start_line = self._current_line - start_bracket_mark = self._next_mark(accept_eof=False) - if start_bracket_mark.group(0) != "{": - self._unaccepted_mark = start_bracket_mark - # Note: The following should never happen, as we check for the "{" in the regex - raise ParserStateException( - message="matched a string def regex (`@string{`) that " - "should end with `{`, but no closing bracket was found." - ) + # Get next mark, which should be an equals sign equals_mark = self._next_mark(accept_eof=False) if equals_mark.group(0) != "=": self._unaccepted_mark = equals_mark @@ -457,32 +471,21 @@ def _handle_string(self, m) -> String: ) key = self.bibstr[m.end() + 1 : equals_mark.start()].strip() value_start = equals_mark.end() - end_i = self._move_to_closed_bracket() - value = self.bibstr[value_start:end_i].strip() + end_index = self._move_to_closing_delimiter(track_quotes=True) + value = self.bibstr[value_start:end_index].strip() return String( start_line=start_line, key=key, value=value, - raw=self.bibstr[start_i : end_i + 1], + raw=self.bibstr[m.start() : end_index + 1], ) - def _handle_preamble(self) -> Preamble: + def _handle_preamble(self, m) -> Preamble: """Handle preamble block. Return end index""" - start_i = self._current_char_index start_line = self._current_line - start_bracket_mark = self._next_mark(accept_eof=False) - if start_bracket_mark.group(0) != "{": - self._unaccepted_mark = start_bracket_mark - # Note: The following should never happen, as we check for the "{" in the regex - raise ParserStateException( - message="matched a preamble def regex (`@preamble{`) that " - "should end with `{`, but no closing bracket was found." - ) - - end_bracket_index = self._move_to_closed_bracket() - preamble = self.bibstr[start_bracket_mark.end() : end_bracket_index] + end_index = self._move_to_closing_delimiter(track_quotes=True) return Preamble( start_line=start_line, - value=preamble, - raw=self.bibstr[start_i : end_bracket_index + 1], + value=self.bibstr[m.end() + 1 : end_index], + raw=self.bibstr[m.start() : end_index + 1], ) diff --git a/tests/splitter_tests/test_splitter_parenthesis_blocks.py b/tests/splitter_tests/test_splitter_parenthesis_blocks.py new file mode 100644 index 0000000..ae45be3 --- /dev/null +++ b/tests/splitter_tests/test_splitter_parenthesis_blocks.py @@ -0,0 +1,209 @@ +"""Blocks delimited by parentheses, e.g. `@article(...)`, which standard BibTeX allows.""" + +import pytest + +import bibtexparser +from bibtexparser.model import Entry +from bibtexparser.model import ExplicitComment +from bibtexparser.model import ParsingFailedBlock +from bibtexparser.model import Preamble +from bibtexparser.model import String +from bibtexparser.splitter import Splitter + + +def test_entry_with_parenthesis(): + """Issue #533: such entries used to be swallowed as implicit comments.""" + bibtex_str = "@article(test2002, author = {A}, year = {2002})" + library = Splitter(bibtex_str).split() + + assert len(library.blocks) == 1 + entry = library.entries[0] + assert entry.entry_type == "article" + assert entry.key == "test2002" + assert entry.fields_dict["author"].value == "{A}" + assert entry.fields_dict["year"].value == "{2002}" + assert entry.raw == bibtex_str + + +@pytest.mark.parametrize( + "spacing", + ["", " ", "\t", " \t "], + ids=["none", "space", "tab", "mixed"], +) +def test_whitespace_between_type_and_parenthesis(spacing: str): + library = Splitter(f"@article{spacing}(key, year = {{2002}})").split() + + assert len(library.blocks) == 1 + assert library.entries[0].key == "key" + + +def test_entry_without_fields(): + library = Splitter("@article(onlykey)").split() + + assert len(library.blocks) == 1 + assert library.entries[0].key == "onlykey" + assert library.entries[0].fields == [] + + +@pytest.mark.parametrize( + ("bibtex_str", "expected_value"), + [ + pytest.param('@article(k, title = "a ) b")', '"a ) b"', id="quoted"), + pytest.param("@article(k, title = {a ) b})", "{a ) b}", id="braced"), + pytest.param("@article(k, title = {a (b) c})", "{a (b) c}", id="nested"), + pytest.param("@article(k, title = {a {( b} c})", "{a {( b} c}", id="nested_braces"), + ], +) +def test_parenthesis_in_field_value_does_not_close_entry(bibtex_str: str, expected_value: str): + library = Splitter(bibtex_str).split() + + assert len(library.blocks) == 1 + assert library.entries[0].fields_dict["title"].value == expected_value + assert library.entries[0].raw == bibtex_str + + +@pytest.mark.parametrize("delimiters", ["{}", "()"]) +def test_at_sign_followed_by_brace_within_value(delimiters: str): + """Regression: an `@word {` within a value must not be mistaken for a block start, + nor must its `{` be lost for brace counting (cf. issue #488).""" + opening, closing = delimiters + bibtex_str = ( + f"@article{opening}k,\n" + " title = {LeQua @ {CLEF} 2022: {A} Shared Task},\n" + " note = {see @foo{bar} and @baz(qux)},\n" + " year = {2021}\n" + f"{closing}" + ) + library = Splitter(bibtex_str).split() + + assert len(library.blocks) == 1 + entry = library.entries[0] + assert entry.fields_dict["title"].value == "{LeQua @ {CLEF} 2022: {A} Shared Task}" + assert entry.fields_dict["note"].value == "{see @foo{bar} and @baz(qux)}" + assert entry.fields_dict["year"].value == "{2021}" + assert entry.raw == bibtex_str + + +@pytest.mark.parametrize( + ("bibtex_str", "expected_value"), + [ + pytest.param('@string(s = "a ) b")', '"a ) b"', id="string"), + pytest.param('@string(s = "a {"} b")', '"a {"} b"', id="string_escaped_quote"), + pytest.param('@preamble(")" # foo)', '")" # foo', id="preamble"), + ], +) +def test_parenthesis_in_quoted_string_or_preamble_value(bibtex_str: str, expected_value: str): + library = Splitter(bibtex_str).split() + + assert len(library.blocks) == 1 + assert library.blocks[0].value == expected_value + assert library.blocks[0].raw == bibtex_str + + +def test_string_preamble_and_comment_with_parenthesis(): + bibtex_str = ( + '@string(foo = "bar")\n' + '@preamble( "\\newcommand{\\x}{y}" # foo )\n' + '@comment(some {braced} "unbalanced comment)' + ) + library = Splitter(bibtex_str).split() + + assert [type(block) for block in library.blocks] == [String, Preamble, ExplicitComment] + assert library.strings[0].key == "foo" + assert library.strings[0].value == '"bar"' + assert library.preambles[0].value.strip() == '"\\newcommand{\\x}{y}" # foo' + assert library.comments[0].comment == 'some {braced} "unbalanced comment' + assert [block.start_line for block in library.blocks] == [0, 1, 2] + + +def test_mixed_delimiters_in_one_file(): + bibtex_str = ( + "@article{curly, year = {2001}}\n" + "@article(round, year = {2002})\n" + "some implicit comment\n" + "@article{curly2, year = {2003}}\n" + "@article(round2,\n year = {2004}\n)" + ) + library = Splitter(bibtex_str).split() + + assert [entry.key for entry in library.entries] == ["curly", "round", "curly2", "round2"] + assert [entry.start_line for entry in library.entries] == [0, 1, 3, 4] + assert len(library.comments) == 1 + assert library.comments[0].start_line == 2 + assert len(library.failed_blocks) == 0 + + +@pytest.mark.parametrize( + "broken_block", + [ + pytest.param("@article(broken,\n year = {2002},\n", id="new_block_while_expecting_key"), + pytest.param("@article(broken,\n year = {2002\n", id="new_block_within_value"), + pytest.param("@article(broken,\n year = {2002}\n", id="new_block_while_expecting_comma"), + pytest.param("@article(broken\n", id="new_block_while_expecting_key_comma"), + pytest.param("@string(broken\n", id="new_block_in_string"), + pytest.param("@comment(broken\n", id="new_block_in_comment"), + pytest.param("@article(broken, year = {2002}}\n", id="curly_instead_of_parenthesis"), + ], +) +@pytest.mark.parametrize("next_delimiter", ["{", "("]) +def test_recovery_after_broken_parenthesis_block(broken_block: str, next_delimiter: str): + """After a failed `(`-block, the next block is parsed normally, whatever its delimiter.""" + closing = "}" if next_delimiter == "{" else ")" + next_block = f"@article{next_delimiter}good, year = {{2003}}{closing}" + library = Splitter(broken_block + next_block).split() + + assert [type(block) for block in library.blocks] == [ParsingFailedBlock, Entry] + assert library.failed_blocks[0].start_line == 0 + assert library.failed_blocks[0].raw.startswith(broken_block.rstrip()) + assert library.entries[0].key == "good" + assert library.entries[0].fields_dict["year"].value == "{2003}" + assert library.entries[0].start_line == broken_block.count("\n") + + +@pytest.mark.parametrize("delimiters", ["{}", "()"]) +def test_recovery_from_unclosed_block_at_parenthesis_block(delimiters: str): + """Like for `@type{`, an `@type(` at line start ends an unclosed preceding block.""" + opening, closing = delimiters + bibtex_str = f"@article{opening}broken, year = {{2002\n@article(good, year = {{2003}})" + library = Splitter(bibtex_str).split() + + assert [type(block) for block in library.blocks] == [ParsingFailedBlock, Entry] + assert library.entries[0].key == "good" + assert library.entries[0].start_line == 1 + + +def test_unclosed_parenthesis_block_at_eof(): + library = Splitter("@article(broken, year = {2002}").split() + + assert [type(block) for block in library.blocks] == [ParsingFailedBlock] + + +@pytest.mark.parametrize( + "bibtex_str", + [ + pytest.param("@article{k(1), note = {f(x)}}", id="parenthesis_in_key_and_value"), + pytest.param("@article{k, note = a ) b}", id="unenclosed_closing_parenthesis"), + pytest.param('@article{k, note = "( unbalanced"}', id="unbalanced_in_quotes"), + ], +) +def test_parenthesis_in_curly_block_is_plain_text(bibtex_str: str): + library = Splitter(bibtex_str).split() + + assert len(library.blocks) == 1 + assert library.entries[0].raw == bibtex_str + + +def test_parse_and_write_roundtrip(): + """End-to-end: default middlewares apply, and the entry is written back (with braces).""" + library = bibtexparser.parse_string( + "@article(test2002,\n author = {Doe, John},\n year = 2002\n)" + ) + + assert len(library.failed_blocks) == 0 + entry = library.entries[0] + assert entry.fields_dict["author"].value == "Doe, John" + assert entry.fields_dict["year"].value == "2002" + + written = bibtexparser.write_string(library) + assert "@article{test2002," in written + assert bibtexparser.parse_string(written).entries[0].key == "test2002"