diff --git a/bibtexparser/entrypoint.py b/bibtexparser/entrypoint.py index 46df5fd..68d9e10 100644 --- a/bibtexparser/entrypoint.py +++ b/bibtexparser/entrypoint.py @@ -1,4 +1,5 @@ import codecs +import logging import warnings from collections.abc import Iterable from copy import deepcopy @@ -16,9 +17,16 @@ from .writer import BibtexFormat from .writer import write +logger = logging.getLogger(__name__) + #: Marks a seeded copy of a pre-existing `@string`, dropped before merging back. _PREEXISTING_STRING_KEY = "bibtexparser_preexisting_string" +#: Number of blocks from which on `write_string`/`write_file` warn if the unparse +#: stack deep-copies blocks. Copying costs roughly 30-60 µs per entry, i.e. it +#: starts to dominate the write time at this size. +LARGE_LIBRARY_WARNING_THRESHOLD = 10_000 + def _build_parse_stack( parse_stack: Iterable[Middleware] | None, @@ -80,6 +88,29 @@ def _build_unparse_stack( return list(prepend_middleware) + list(unparse_stack) +def _warn_if_large_library_is_copied(library: Library, unparse_stack: list[Middleware]) -> None: + """Warn if writing ``library`` will deep-copy its blocks and that is likely slow. + + Middlewares with ``allow_inplace_modification=False`` (the default unparse stack + is built that way) deep-copy every block they transform, which dominates the + write time of large libraries. + """ + n_blocks = len(library.blocks) + if n_blocks < LARGE_LIBRARY_WARNING_THRESHOLD: + return + if all(middleware.allow_inplace_modification for middleware in unparse_stack): + return + logger.warning( + f"Writing a library with {n_blocks} blocks: the unparse stack deep-copies blocks " + "(it contains middlewares with allow_inplace_modification=False), " + "which is slow for large libraries. " + "If you do not need the library after writing, pass an unparse stack whose " + "middlewares all allow in-place modification, e.g. " + "`unparse_stack=bibtexparser.middlewares.default_unparse_stack(" + "allow_inplace_modification=True)`." + ) + + def _handle_deprecated_write_params( unparse_stack: Iterable[Middleware] | None, prepend_middleware: Iterable[Middleware] | None, @@ -287,6 +318,9 @@ def write_file( Only applicable if `unparse_stack` is None. :param bibtex_format: Customized BibTeX format to use (optional). :param encoding: Encoding of the .bib file. Default encoding is ``"UTF-8"``. + Writing a library with at least ``LARGE_LIBRARY_WARNING_THRESHOLD`` blocks logs a warning + if the unparse stack deep-copies blocks (middlewares with ``allow_inplace_modification=False``), + as that is slow; pass an all-in-place stack to avoid it. .. deprecated:: (next version) Parameters 'parse_stack' and 'append_middleware' are deprecated, will be deleted soon. @@ -324,6 +358,9 @@ def write_string( :param prepend_middleware: List of middleware to prepend to the default stack. Only applicable if `unparse_stack` is None. :param bibtex_format: Customized BibTeX format to use (optional). + Writing a library with at least ``LARGE_LIBRARY_WARNING_THRESHOLD`` blocks logs a warning + if the unparse stack deep-copies blocks (middlewares with ``allow_inplace_modification=False``), + as that is slow; pass an all-in-place stack to avoid it. .. deprecated:: (next version) Parameters 'parse_stack' and 'append_middleware' are deprecated. @@ -333,8 +370,11 @@ def write_string( unparse_stack, prepend_middleware, kwargs, "write_string" ) + stack = _build_unparse_stack(unparse_stack, prepend_middleware) + _warn_if_large_library_is_copied(library, stack) + middleware: Middleware - for middleware in _build_unparse_stack(unparse_stack, prepend_middleware): + for middleware in stack: library = middleware.transform(library=library) return write(library, bibtex_format=bibtex_format) diff --git a/tests/test_entrypoint.py b/tests/test_entrypoint.py index b4656a7..c8a46ab 100644 --- a/tests/test_entrypoint.py +++ b/tests/test_entrypoint.py @@ -1,5 +1,6 @@ """Testing the parse_file and write_file functions.""" +import logging import os import tempfile import warnings @@ -10,7 +11,9 @@ from bibtexparser import parse_string from bibtexparser import write_file from bibtexparser import write_string +from bibtexparser.entrypoint import LARGE_LIBRARY_WARNING_THRESHOLD from bibtexparser.library import Library +from bibtexparser.middlewares import default_unparse_stack from bibtexparser.model import DuplicateBlockKeyBlock from bibtexparser.model import Entry from bibtexparser.model import Field @@ -401,3 +404,70 @@ def test_parse_string_into_existing_library_keeps_block_order(): "Entry", ] assert [entry.key for entry in library.entries] == ["first", "second"] + + +def _library_with_n_entries(n: int) -> Library: + return Library( + [ + Entry(entry_type="article", key=f"key{i}", fields=[Field(key="title", value="T")]) + for i in range(n) + ] + ) + + +def test_large_library_warning_threshold_is_reasonable(): + assert 1_000 <= LARGE_LIBRARY_WARNING_THRESHOLD <= 100_000 + + +def test_write_string_warns_for_large_library_with_copying_stack(monkeypatch, caplog): + """The default unparse stack deep-copies blocks, which is slow for large libraries.""" + monkeypatch.setattr("bibtexparser.entrypoint.LARGE_LIBRARY_WARNING_THRESHOLD", 5) + library = _library_with_n_entries(5) + + with caplog.at_level(logging.WARNING, logger="bibtexparser.entrypoint"): + write_string(library) + + warnings_ = [r for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings_) == 1 + assert "Writing a library with 5 blocks" in warnings_[0].message + assert "allow_inplace_modification=True" in warnings_[0].message + + +def test_write_file_warns_for_large_library_with_copying_stack(monkeypatch, caplog): + monkeypatch.setattr("bibtexparser.entrypoint.LARGE_LIBRARY_WARNING_THRESHOLD", 5) + library = _library_with_n_entries(5) + + with tempfile.NamedTemporaryFile(mode="w", suffix=".bib", delete=False) as f: + temp_path = f.name + try: + with caplog.at_level(logging.WARNING, logger="bibtexparser.entrypoint"): + write_file(temp_path, library) + finally: + os.unlink(temp_path) + + assert caplog.text.count("Writing a library with 5 blocks") == 1 + + +def test_write_string_does_not_warn_below_threshold(monkeypatch, caplog): + monkeypatch.setattr("bibtexparser.entrypoint.LARGE_LIBRARY_WARNING_THRESHOLD", 5) + library = _library_with_n_entries(4) + + with caplog.at_level(logging.WARNING, logger="bibtexparser.entrypoint"): + write_string(library) + + assert "Writing a library" not in caplog.text + + +def test_write_string_does_not_warn_with_inplace_stack(monkeypatch, caplog): + """The suggested workaround must itself not warn, and must produce identical output.""" + expected = write_string(_library_with_n_entries(5)) + monkeypatch.setattr("bibtexparser.entrypoint.LARGE_LIBRARY_WARNING_THRESHOLD", 5) + + with caplog.at_level(logging.WARNING, logger="bibtexparser.entrypoint"): + actual = write_string( + _library_with_n_entries(5), + unparse_stack=default_unparse_stack(allow_inplace_modification=True), + ) + + assert "Writing a library" not in caplog.text + assert actual == expected