From 93711848b920c1ee6d19835a7d9bd42bdfb80f62 Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Thu, 3 Sep 2026 22:21:40 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Make=20deep-copying=20of=20blocks?= =?UTF-8?q?=20and=20fields=20cheap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `BlockMiddleware.transform_block` deep-copies every block unless `allow_inplace_modification` is set, which is the default for the unparse stack. With the generic `copy.deepcopy` this dominated `write_string`/`write_file` (~85% of a profile on a real-world .bib). Implement `__deepcopy__` for `Block` and `Field` via a shared, attribute-aware helper: immutable primitives are shared, plain lists and dicts and nested blocks/fields are copied directly, anything else is delegated to `copy.deepcopy`. Semantics are unchanged (independent copies, memo honoured, cycles and shared references preserved, subclasses and custom `__deepcopy__` overrides supported). Per-entry deepcopy drops from ~34us to ~11us; `write_string` on a 41 MB / 91 MB .bib from 5.7s / 9.4s to 3.6s / 6.0s. Co-Authored-By: Claude Fable 5.1 --- bibtexparser/model.py | 100 ++++++++++++++++++ tests/test_entrypoint.py | 37 +++++++ tests/test_model.py | 215 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 352 insertions(+) diff --git a/bibtexparser/model.py b/bibtexparser/model.py index de332d2..efcd91a 100644 --- a/bibtexparser/model.py +++ b/bibtexparser/model.py @@ -1,8 +1,100 @@ import abc +from copy import deepcopy from typing import Any +from typing import TypeVar _ALLOWED_ENCLOSINGS = (None, "{", '"', "no-enclosing") +# --- Fast deep-copying of blocks and fields ---------------------------------- +# Middleware deep-copies every block by default (see ``BlockMiddleware``), which +# dominated the runtime of ``write_string`` with the generic ``copy.deepcopy``. +# The helpers below implement ``__deepcopy__`` for ``Block`` and ``Field`` with +# identical semantics (fully independent copies, ``memo`` honoured, cycles and +# shared references preserved, subclasses supported), but without the generic +# ``__reduce_ex__`` machinery and without dispatching immutable primitives. + +# Types whose instances are immutable and hence may be shared between an +# object and its deep copy (which is also what ``copy.deepcopy`` does for them). +# Exact type matches only: subclasses of these types go through ``deepcopy``. +_ATOMIC_TYPES = frozenset({str, int, float, bool, type(None)}) +_MISSING = object() + +_T = TypeVar("_T") + + +def _deepcopy_value(value: _T, memo: dict[int, Any]) -> _T: + """``copy.deepcopy(value, memo)``, with shortcuts for the types making up blocks. + + Immutable primitives are shared, plain lists and dicts as well as objects + using the fast ``__deepcopy__`` of this module are copied directly + (respecting ``memo`` exactly like ``copy.deepcopy`` does). + Anything else is delegated to ``copy.deepcopy``. + + Note: Unlike ``copy.deepcopy``, no "keep alive" list of originals is + maintained in ``memo``. That is only needed for temporaries created while + copying (e.g. ``__reduce_ex__`` state); everything copied here is reachable + from (and kept alive by) the object the caller passed to ``deepcopy``. + """ + value_type = type(value) + if value_type in _ATOMIC_TYPES: + return value + if value_type is list: + key = id(value) + copied = memo.get(key, _MISSING) + if copied is not _MISSING: + return copied + memo[key] = copied = [] + for item in value: + item_type = type(item) + if item_type in _ATOMIC_TYPES: + copied.append(item) + elif getattr(item_type, "__deepcopy__", None) is _deepcopy_model_object: + # Inlined memo check, as lists of fields are the most common case + item_copy = memo.get(id(item), _MISSING) + if item_copy is _MISSING: + item_copy = _deepcopy_model_object(item, memo) + copied.append(item_copy) + else: + copied.append(_deepcopy_value(item, memo)) + return copied + if value_type is dict: + key = id(value) + copied = memo.get(key, _MISSING) + if copied is not _MISSING: + return copied + memo[key] = copied = {} + for item_key, item in value.items(): + copied[ + item_key if type(item_key) in _ATOMIC_TYPES else _deepcopy_value(item_key, memo) + ] = (item if type(item) in _ATOMIC_TYPES else _deepcopy_value(item, memo)) + return copied + if getattr(value_type, "__deepcopy__", None) is _deepcopy_model_object: + copied = memo.get(id(value), _MISSING) + if copied is not _MISSING: + return copied + return _deepcopy_model_object(value, memo) + return deepcopy(value, memo) + + +def _deepcopy_model_object(obj: _T, memo: dict[int, Any]) -> _T: + """Deep-copies ``obj`` by copying its ``__dict__``, attribute by attribute. + + Used as ``__deepcopy__`` of ``Block`` and ``Field``. Semantically equivalent + to the generic ``copy.deepcopy`` of a plain object: all attribute values + (lists, dicts, nested blocks, exceptions, ...) are deep-copied with the + shared ``memo``, so references that are shared within one ``deepcopy`` + call stay shared in the copy, and cycles are handled. + Subclasses (including their additional attributes) are supported. + """ + cls = obj.__class__ + copied = cls.__new__(cls) + # Register the copy before recursing, so cyclic references resolve to it. + memo[id(obj)] = copied + copied_dict = copied.__dict__ + for key, value in obj.__dict__.items(): + copied_dict[key] = value if type(value) in _ATOMIC_TYPES else _deepcopy_value(value, memo) + return copied + def _validated_enclosing(enclosing: str | None) -> str | None: if enclosing not in _ALLOWED_ENCLOSINGS: @@ -88,6 +180,10 @@ def __hash__(self) -> int: # Subclasses add a cheap identifier: both are None if not parsed. return hash((type(self), self._start_line_in_file, self._raw)) + # Fast path for the frequent deep copies made by middleware + # (see ``BlockMiddleware.transform_block``). Same semantics as the default. + __deepcopy__ = _deepcopy_model_object + class String(Block): """Bibtex Blocks of the ``@string`` type, e.g. ``@string{me = "My Name"}``.""" @@ -309,6 +405,10 @@ def __hash__(self) -> int: # may be mutable or unhashable (e.g. a list after middleware was applied). return hash((type(self), self._start_line, self._key)) + # Fast path, see ``Block.__deepcopy__``. Non-primitive values + # (e.g. lists or ``NameParts`` after middleware was applied) are deep-copied. + __deepcopy__ = _deepcopy_model_object + def __str__(self) -> str: return f"Field (line: {self.start_line}, key: `{self.key}`): `{self.value}`" diff --git a/tests/test_entrypoint.py b/tests/test_entrypoint.py index b4656a7..ac376af 100644 --- a/tests/test_entrypoint.py +++ b/tests/test_entrypoint.py @@ -1,6 +1,7 @@ """Testing the parse_file and write_file functions.""" import os +import pickle import tempfile import warnings @@ -401,3 +402,39 @@ def test_parse_string_into_existing_library_keeps_block_order(): "Entry", ] assert [entry.key for entry in library.entries] == ["first", "second"] + + +def test_write_string_roundtrip_is_stable_and_leaves_library_untouched(): + """The default unparse stack deep-copies blocks; output and library must be unaffected.""" + bibtex = ( + "@string{me = {My Name}}\n\n" + "@preamble{\\newcommand{\\foo}{bar}}\n\n" + "% An implicit comment\n\n" + "@comment{An explicit comment}\n\n" + "@article{key,\n" + "\tauthor = {John Doe and Jane Smith},\n" + '\ttitle = "Some Title",\n' + "\tmonth = jan,\n" + "\tyear = 2020\n" + "}\n" + ) + library = parse_string(bibtex) + # Output of the default unparse stack (verbatim, do not "fix" without a reason) + expected = ( + "@string{me = {My Name}}\n\n\n" + "@preamble{\\newcommand{\\foo}{bar}}\n\n\n" + "% An implicit comment\n\n\n" + "@comment{An explicit comment}\n\n\n" + "@article{key,\n" + "\tauthor = {John Doe and Jane Smith},\n" + "\ttitle = {Some Title},\n" + "\tmonth = jan,\n" + "\tyear = {2020}\n" + "}\n" + ) + blocks_before = [pickle.loads(pickle.dumps(block)) for block in library.blocks] + + assert write_string(library) == expected + # Writing again yields the identical output, i.e. writing did not mutate the library + assert write_string(library) == expected + assert library.blocks == blocks_before diff --git a/tests/test_model.py b/tests/test_model.py index 575875f..1c1f5a5 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,3 +1,4 @@ +import pickle from copy import copy from copy import deepcopy from textwrap import dedent @@ -6,10 +7,14 @@ import bibtexparser from bibtexparser.middlewares import NameParts +from bibtexparser.model import DuplicateBlockKeyBlock +from bibtexparser.model import DuplicateFieldKeyBlock from bibtexparser.model import Entry from bibtexparser.model import ExplicitComment from bibtexparser.model import Field from bibtexparser.model import ImplicitComment +from bibtexparser.model import MiddlewareErrorBlock +from bibtexparser.model import ParsingFailedBlock from bibtexparser.model import Preamble from bibtexparser.model import String @@ -545,3 +550,213 @@ def test_entry_fields_shorthand(): assert len([f for f in entry.fields if f.key == "myNewField"]) == 0 with pytest.raises(KeyError): entry["myNewField"] + + +# --- Deep-copying of blocks and fields --------------------------------------- +# Blocks and fields implement a fast `__deepcopy__` (middleware deep-copies every +# block by default). The following tests ensure it behaves like the generic one. + + +def _all_block_types(): + entry = Entry("article", "key", [Field("field", "value", 1)], 1, "raw") + entry.set_parser_metadata("meta", {"nested": ["list"]}) + return [ + entry, + String("key", "value", 2, "raw", enclosing="{"), + Preamble("value", 3, "raw"), + ExplicitComment("comment", 4, "raw"), + ImplicitComment("comment", 5, "raw"), + ParsingFailedBlock(ValueError("error"), 6, "raw"), + MiddlewareErrorBlock(Entry("article", "key", []), ValueError("error")), + DuplicateBlockKeyBlock("key", Entry("article", "key", []), Entry("book", "key", []), 7), + DuplicateFieldKeyBlock( + {"a", "b"}, Entry("article", "key", [Field("a", "1"), Field("a", "2")]) + ), + ] + + +def _assert_same_state(block, other): + """Asserts that two distinct block instances carry the same state. + + Blocks holding an exception (``ParsingFailedBlock``) are never ``==`` + to a copy of themselves, as exceptions compare by identity only.""" + assert block is not other + assert type(block) is type(other) + if isinstance(block, ParsingFailedBlock): + assert type(block.error) is type(other.error) + assert str(block.error) == str(other.error) + assert block.error is not other.error + state = {key: value for key, value in block.__dict__.items() if key != "_error"} + other_state = {key: value for key, value in other.__dict__.items() if key != "_error"} + assert state == other_state + else: + assert block == other + + +@pytest.mark.parametrize("block", _all_block_types(), ids=lambda b: type(b).__name__) +def test_block_deepcopy_is_equal_but_independent(block): + block_copy = deepcopy(block) + + _assert_same_state(block_copy, block) + assert hash(block_copy) == hash(block) + assert block_copy.__dict__.keys() == block.__dict__.keys() + # Mutable attributes are copied, not shared + assert block_copy.parser_metadata is not block.parser_metadata + block_copy.set_parser_metadata("added", True) + assert block.get_parser_metadata("added") is None + assert block_copy != block + + +@pytest.mark.parametrize("block", _all_block_types(), ids=lambda b: type(b).__name__) +def test_block_deepcopy_matches_pickle_roundtrip(block): + # Pickling reconstructs all state the way the generic `copy.deepcopy` does. + pickled_copy = pickle.loads(pickle.dumps(block)) + deep_copy = deepcopy(block) + _assert_same_state(deep_copy, pickled_copy) + + +def test_entry_deepcopy_nested_values_are_independent(): + entry = Entry("article", "key", [Field("field", "value", 1)], 1, "raw") + entry.set_parser_metadata("meta", {"nested": ["list"]}) + entry_copy = deepcopy(entry) + + entry_copy.fields.append(Field("new", "value")) + entry_copy.fields[0].value = "changed" + entry_copy.parser_metadata["meta"]["nested"].append("added") + assert len(entry.fields) == 1 + assert entry.fields[0].value == "value" + assert entry.parser_metadata == {"meta": {"nested": ["list"]}} + + entry.fields[0].key = "renamed" + entry.set_parser_metadata("other", 1) + assert entry_copy.fields[0].key == "field" + assert entry_copy.get_parser_metadata("other") is None + + +@pytest.mark.parametrize( + "value", + [ + ["Doe, John", "Smith, Jane"], + [NameParts(first=["John"], last=["Doe"]), NameParts(first=["Jane"], last=["Smith"])], + NameParts(first=["John"], von=["von"], last=["Doe"], jr=["Jr."]), + ("a", "tuple"), + {"a": ["dict"]}, + ], + ids=["list", "nameparts_list", "nameparts", "tuple", "dict"], +) +def test_field_deepcopy_non_primitive_values_are_deep_copied(value): + field = Field("author", value, 1, enclosing="{") + field_copy = deepcopy(field) + + assert field_copy == field + assert field_copy is not field + assert field_copy.value == value + assert field_copy.key == "author" + assert field_copy.start_line == 1 + assert field_copy.enclosing == "{" + if isinstance(value, tuple): + # Tuples of immutables are shared by the generic `copy.deepcopy` as well + return + assert field_copy.value is not field.value + if isinstance(value, list): + original_length = len(value) + field_copy.value.append("added") + assert len(field.value) == original_length + if isinstance(value[0], NameParts): + assert field_copy.value[0] is not field.value[0] + field_copy.value[0].first.append("added") + assert field.value[0].first == ["John"] + if isinstance(value, NameParts): + field_copy.value.first.append("added") + assert field.value.first == ["John"] + if isinstance(value, dict): + field_copy.value["a"].append("added") + assert field.value == {"a": ["dict"]} + + +def test_field_deepcopy_shares_immutable_primitives(): + # Same behavior as generic `copy.deepcopy`: atomic values are not duplicated + field = Field("key", "value", 1, enclosing="{") + field_copy = deepcopy(field) + assert field_copy.value is field.value + assert field_copy.key is field.key + assert field_copy.enclosing is field.enclosing + # Changing the copy does not affect the original + field_copy.value = "new value" + assert field.value == "value" + assert field.enclosing == "{" + assert field_copy.enclosing is None + + +def test_deepcopy_memo_keeps_shared_references_shared(): + previous = Entry("article", "key", [Field("field", "value")], 1, "raw") + duplicate = Entry("article", "key", [Field("field", "other")], 2, "raw2") + error_block = DuplicateBlockKeyBlock("key", previous, duplicate, 2, "raw2") + + previous_copy, error_block_copy = deepcopy([previous, error_block]) + + assert error_block_copy.previous_block is previous_copy + assert error_block_copy.previous_block is not previous + assert error_block_copy.ignore_error_block == duplicate + assert error_block_copy.ignore_error_block is not duplicate + assert error_block_copy.error is not error_block.error + assert str(error_block_copy.error) == str(error_block.error) + + # A block deep-copied on its own also deep-copies the referenced blocks + lone_copy = deepcopy(error_block) + assert lone_copy.previous_block == previous + assert lone_copy.previous_block is not previous + + +def test_deepcopy_handles_cyclic_references(): + entry = Entry("article", "key", [Field("field", "value")]) + entry.set_parser_metadata("self", entry) + entry.set_parser_metadata("field", entry.fields[0]) + + entry_copy = deepcopy(entry) + assert entry_copy.get_parser_metadata("self") is entry_copy + assert entry_copy.get_parser_metadata("field") is entry_copy.fields[0] + + +def test_deepcopy_of_entry_subclass_yields_subclass(): + class CustomEntry(Entry): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.custom_attribute = ["custom"] + + entry = CustomEntry("article", "key", [Field("field", "value")], 1, "raw") + entry_copy = deepcopy(entry) + + assert type(entry_copy) is CustomEntry + assert entry_copy == entry + assert entry_copy.custom_attribute == ["custom"] + assert entry_copy.custom_attribute is not entry.custom_attribute + assert entry_copy.fields == entry.fields + assert entry_copy.fields is not entry.fields + + +def test_deepcopy_of_str_subclass_values_is_not_shared(): + class CustomStr(str): + pass + + field = Field("key", CustomStr("value")) + field_copy = deepcopy(field) + assert type(field_copy.value) is CustomStr + assert field_copy.value == "value" + assert field_copy.value is not field.value + + +def test_deepcopy_honours_custom_deepcopy_of_nested_subclass(): + class CustomEntry(Entry): + def __deepcopy__(self, memo): + copied = Entry.__deepcopy__(self, memo) + copied.set_parser_metadata("custom_deepcopy", True) + return copied + + entry = CustomEntry("article", "key", [Field("field", "value")]) + error_block = DuplicateBlockKeyBlock("key", entry, Entry("article", "key", [])) + + error_block_copy = deepcopy(error_block) + assert type(error_block_copy.previous_block) is CustomEntry + assert error_block_copy.previous_block.get_parser_metadata("custom_deepcopy") is True + assert entry.get_parser_metadata("custom_deepcopy") is None