From dcf0e4e31a1d08a313b6f06c6ec4d53433696aa0 Mon Sep 17 00:00:00 2001 From: Mohith26 <109003762+Mohith26@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:58:25 -0500 Subject: [PATCH] Load namespace modules lazily in the generated base clients import dropbox used to import every namespace module, including team_log, because base.py and base_team.py import all of them at module level. The generator now emits LazyNamespace proxies that import a namespace on first use, and base_team is generated by the same generator so it gets them too. Closes #269 --- dropbox/_lazy_namespace.py | 46 +++++++++ dropbox/base.py | 44 ++++---- dropbox/base_team.py | 48 +++++---- generate_base_client.py | 12 ++- generator/dropbox_python_client.stoneg.py | 16 ++- scripts/benchmark_import.py | 62 ++++++++++++ test/unit/test_lazy_namespace.py | 117 ++++++++++++++++++++++ 7 files changed, 300 insertions(+), 45 deletions(-) create mode 100644 dropbox/_lazy_namespace.py create mode 100644 scripts/benchmark_import.py create mode 100644 test/unit/test_lazy_namespace.py diff --git a/dropbox/_lazy_namespace.py b/dropbox/_lazy_namespace.py new file mode 100644 index 00000000..ecaa9ab4 --- /dev/null +++ b/dropbox/_lazy_namespace.py @@ -0,0 +1,46 @@ +"""Lazy references to generated namespace modules. + +The generated base clients (``dropbox.base`` and ``dropbox.base_team``) have a +method for every route in every namespace. Importing every namespace module up +front costs about 30 MB and a noticeable amount of import time, most of it from +``team_log``, even for apps that only ever call a handful of ``files`` routes. + +The base clients refer to namespaces through :class:`LazyNamespace` instead, so a +namespace module is only imported the first time one of its routes or types is +used. +""" + +import importlib + + +class LazyNamespace(object): + """Stands in for a namespace module and imports it on first attribute access. + + Importing goes through :func:`importlib.import_module`, which takes the + per-module import lock, so concurrent first accesses from several threads + are safe and all see the same fully initialized module. + """ + + __slots__ = ("_name", "_module") + + def __init__(self, name): + self._name = name + self._module = None + + def _load(self): + module = self._module + if module is None: + module = importlib.import_module(self._name) + self._module = module + return module + + def __getattr__(self, attr): + # Only called for attributes not found on the proxy itself. + return getattr(self._load(), attr) + + def __dir__(self): + return dir(self._load()) + + def __repr__(self): + state = "loaded" if self._module is not None else "not loaded" + return "".format(self._name, state) diff --git a/dropbox/base.py b/dropbox/base.py index 08102d75..c0e0012b 100644 --- a/dropbox/base.py +++ b/dropbox/base.py @@ -6,27 +6,29 @@ from abc import ABCMeta, abstractmethod import warnings -from dropbox import account -from dropbox import async_ -from dropbox import auth -from dropbox import check -from dropbox import common -from dropbox import contacts -from dropbox import file_properties -from dropbox import file_requests -from dropbox import files -from dropbox import openid -from dropbox import paper -from dropbox import riviera -from dropbox import secondary_emails -from dropbox import seen_state -from dropbox import sharing -from dropbox import team -from dropbox import team_common -from dropbox import team_log -from dropbox import team_policies -from dropbox import users -from dropbox import users_common +from dropbox._lazy_namespace import LazyNamespace + +account = LazyNamespace("dropbox.account") +async_ = LazyNamespace("dropbox.async_") +auth = LazyNamespace("dropbox.auth") +check = LazyNamespace("dropbox.check") +common = LazyNamespace("dropbox.common") +contacts = LazyNamespace("dropbox.contacts") +file_properties = LazyNamespace("dropbox.file_properties") +file_requests = LazyNamespace("dropbox.file_requests") +files = LazyNamespace("dropbox.files") +openid = LazyNamespace("dropbox.openid") +paper = LazyNamespace("dropbox.paper") +riviera = LazyNamespace("dropbox.riviera") +secondary_emails = LazyNamespace("dropbox.secondary_emails") +seen_state = LazyNamespace("dropbox.seen_state") +sharing = LazyNamespace("dropbox.sharing") +team = LazyNamespace("dropbox.team") +team_common = LazyNamespace("dropbox.team_common") +team_log = LazyNamespace("dropbox.team_log") +team_policies = LazyNamespace("dropbox.team_policies") +users = LazyNamespace("dropbox.users") +users_common = LazyNamespace("dropbox.users_common") class DropboxBase(object): diff --git a/dropbox/base_team.py b/dropbox/base_team.py index c60f960d..5427d0ab 100644 --- a/dropbox/base_team.py +++ b/dropbox/base_team.py @@ -6,34 +6,38 @@ from abc import ABCMeta, abstractmethod import warnings -from dropbox import account -from dropbox import async_ -from dropbox import auth -from dropbox import check -from dropbox import common -from dropbox import contacts -from dropbox import file_properties -from dropbox import file_requests -from dropbox import files -from dropbox import openid -from dropbox import paper -from dropbox import riviera -from dropbox import secondary_emails -from dropbox import seen_state -from dropbox import sharing -from dropbox import team -from dropbox import team_common -from dropbox import team_log -from dropbox import team_policies -from dropbox import users -from dropbox import users_common +from dropbox._lazy_namespace import LazyNamespace + +account = LazyNamespace("dropbox.account") +async_ = LazyNamespace("dropbox.async_") +auth = LazyNamespace("dropbox.auth") +check = LazyNamespace("dropbox.check") +common = LazyNamespace("dropbox.common") +contacts = LazyNamespace("dropbox.contacts") +file_properties = LazyNamespace("dropbox.file_properties") +file_requests = LazyNamespace("dropbox.file_requests") +files = LazyNamespace("dropbox.files") +openid = LazyNamespace("dropbox.openid") +paper = LazyNamespace("dropbox.paper") +riviera = LazyNamespace("dropbox.riviera") +secondary_emails = LazyNamespace("dropbox.secondary_emails") +seen_state = LazyNamespace("dropbox.seen_state") +sharing = LazyNamespace("dropbox.sharing") +team = LazyNamespace("dropbox.team") +team_common = LazyNamespace("dropbox.team_common") +team_log = LazyNamespace("dropbox.team_log") +team_policies = LazyNamespace("dropbox.team_policies") +users = LazyNamespace("dropbox.users") +users_common = LazyNamespace("dropbox.users_common") class DropboxTeamBase(object): __metaclass__ = ABCMeta @abstractmethod - def request(self, route, namespace, request_arg, request_binary, timeout=None): + def request( + self, route, namespace, request_arg, request_binary, timeout=None, extra_headers=None + ): pass # ------------------------------------------ diff --git a/generate_base_client.py b/generate_base_client.py index 7fc3d5b7..92ea0d5a 100755 --- a/generate_base_client.py +++ b/generate_base_client.py @@ -100,7 +100,17 @@ def main(): o = subprocess.check_output( ( - ["python", "-m", "stone.cli", "python_client", dropbox_pkg_path] + [ + "python", + "-m", + "stone.cli", + os.path.join( + os.path.dirname(__file__), + "generator", + "dropbox_python_client.stoneg.py", + ), + dropbox_pkg_path, + ] + specs + ["-a", "host", "-a", "style", "-a", "auth", "-a", "scope"] + [ diff --git a/generator/dropbox_python_client.stoneg.py b/generator/dropbox_python_client.stoneg.py index b8d7446a..1a578027 100644 --- a/generator/dropbox_python_client.stoneg.py +++ b/generator/dropbox_python_client.stoneg.py @@ -1,6 +1,6 @@ from stone.backends import python_client from stone.backends.helpers import fmt_underscores -from stone.backends.python_helpers import fmt_func +from stone.backends.python_helpers import fmt_func, fmt_namespace from stone.ir import ( is_nullable_type, is_struct_type, @@ -45,6 +45,20 @@ def generate(self, api): self.emit() self._generate_route_methods(api.namespaces.values()) + def _generate_imports(self, namespaces): + """Refer to namespace modules lazily. + + Importing every namespace up front costs about 30 MB, mostly team_log, + even when an app only calls a few routes. Each namespace is imported on + first use instead (see dropbox/_lazy_namespace.py). + """ + self.emit("from {}._lazy_namespace import LazyNamespace".format(self.args.types_package)) + self.emit() + for namespace in namespaces: + if namespace.data_types: + name = fmt_namespace(namespace.name) + self.emit('{} = LazyNamespace("{}.{}")'.format(name, self.args.types_package, name)) + @staticmethod def _supports_extra_headers(namespace, route): return namespace.name == "files" and route.name == "download" diff --git a/scripts/benchmark_import.py b/scripts/benchmark_import.py new file mode 100644 index 00000000..13994fe9 --- /dev/null +++ b/scripts/benchmark_import.py @@ -0,0 +1,62 @@ +"""Measure what `import dropbox` costs on its own. + +Each run uses a fresh interpreter. requests and stone are imported first, so the +numbers only cover the SDK's own modules. Times are medians over `runs` +interpreters; traced memory (tracemalloc) is the median of 3 separate runs. + + python scripts/benchmark_import.py [runs] +""" + +import json +import statistics +import subprocess +import sys + +CODE = r""" +import json, resource, sys, time, tracemalloc +import requests +import stone.backends.python_rsrc.stone_validators + +trace = sys.argv[1] == "trace" +rss_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +if trace: + tracemalloc.start() +start = time.perf_counter() +import dropbox +elapsed = time.perf_counter() - start +traced = tracemalloc.get_traced_memory()[0] if trace else 0 +rss_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +# ru_maxrss is bytes on macOS and kilobytes on Linux +scale = 1 if sys.platform == "darwin" else 1024 +print(json.dumps({ + "ms": elapsed * 1000, + "traced_mb": traced / 1e6, + "rss_mb": (rss_after - rss_before) * scale / 1e6, + "modules": sorted(m for m in sys.modules if m.startswith("dropbox.")), +})) +""" + + +def main(): + runs = int(sys.argv[1]) if len(sys.argv) > 1 else 15 + + def run(mode): + out = subprocess.check_output([sys.executable, "-c", CODE, mode]) + return json.loads(out) + + # Timing runs don't use tracemalloc, which slows imports down a lot. + results = [run("time") for _ in range(runs)] + traced = [run("trace")["traced_mb"] for _ in range(3)] + summary = { + "runs": runs, + "import_ms_median": round(statistics.median(r["ms"] for r in results), 1), + "traced_mb_median": round(statistics.median(traced), 1), + "rss_mb_median": round(statistics.median(r["rss_mb"] for r in results), 1), + "dropbox_modules_loaded": len(results[-1]["modules"]), + "team_log_loaded": "dropbox.team_log" in results[-1]["modules"], + } + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/test/unit/test_lazy_namespace.py b/test/unit/test_lazy_namespace.py new file mode 100644 index 00000000..4280180e --- /dev/null +++ b/test/unit/test_lazy_namespace.py @@ -0,0 +1,117 @@ +import json +import subprocess +import sys +import textwrap +from unittest import mock + +import pytest + +from dropbox import base, base_team +from dropbox._lazy_namespace import LazyNamespace + + +def _run(code): + """Run code in a fresh interpreter so sys.modules starts empty.""" + out = subprocess.check_output([sys.executable, "-c", textwrap.dedent(code)]) + return json.loads(out.decode("utf-8").strip().splitlines()[-1]) + + +class TestLazyNamespace: + def test_importing_dropbox_does_not_load_team_namespaces(self): + loaded = _run( + """ + import json, sys + import dropbox + print(json.dumps(sorted(m for m in sys.modules if m.startswith("dropbox.")))) + """ + ) + assert "dropbox.team_log" not in loaded + assert "dropbox.team" not in loaded + assert "dropbox.files" in loaded + + def test_namespace_is_loaded_on_first_route_call(self): + result = _run( + """ + import json, sys + from unittest import mock + import dropbox + + dbx = dropbox.DropboxTeam("token") + before = "dropbox.team_log" in sys.modules + with mock.patch.object(dbx, "request", return_value="ok") as request: + dbx.team_log_get_events(limit=5) + route, namespace, arg, binary = request.call_args[0] + from dropbox import team_log + print(json.dumps({ + "before": before, + "after": "dropbox.team_log" in sys.modules, + "same_route": route is team_log.get_events, + "namespace": namespace, + "arg_type": type(arg).__name__, + "limit": arg.limit, + })) + """ + ) + assert result == { + "before": False, + "after": True, + "same_route": True, + "namespace": "team_log", + "arg_type": "GetTeamEventsArg", + "limit": 5, + } + + def test_concurrent_first_access_sees_one_module(self): + result = _run( + """ + import json, threading + from dropbox._lazy_namespace import LazyNamespace + + ns = LazyNamespace("dropbox.team_log") + barrier = threading.Barrier(16) + seen = [] + + def worker(): + barrier.wait() + seen.append(ns.get_events) + + threads = [threading.Thread(target=worker) for _ in range(16)] + for t in threads: + t.start() + for t in threads: + t.join() + print(json.dumps({"count": len(seen), "distinct": len({id(r) for r in seen})})) + """ + ) + assert result == {"count": 16, "distinct": 1} + + def test_proxy_behaves_like_the_module(self): + from dropbox import files + + ns = LazyNamespace("dropbox.files") + assert "not loaded" in repr(ns) + assert ns.DownloadArg is files.DownloadArg + assert "loaded" in repr(ns) and "not loaded" not in repr(ns) + assert "DownloadArg" in dir(ns) + with pytest.raises(AttributeError): + ns.does_not_exist + + def test_missing_module_raises_import_error_on_use(self): + ns = LazyNamespace("dropbox.no_such_namespace") + with pytest.raises(ImportError): + ns.anything + + def test_base_clients_use_lazy_namespaces(self): + assert isinstance(base.team_log, LazyNamespace) + assert isinstance(base_team.team_log, LazyNamespace) + + def test_user_route_still_builds_real_types(self): + from dropbox import Dropbox, files + + dbx = Dropbox("token") + with mock.patch.object(dbx, "request", return_value="ok") as request: + dbx.files_get_metadata("/a") + route, namespace, arg, binary = request.call_args[0] + assert route is files.get_metadata + assert isinstance(arg, files.GetMetadataArg) + assert arg.path == "/a"