From 6d1c52d8f77110d3a786a93284505eb57303851f Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Wed, 23 Sep 2026 16:23:42 -0500 Subject: [PATCH 1/4] Normalize extension imports across CLI, API, and multiprocessing workers --- .github/workflows/extension-tests.yml | 43 ++++++ activitysim/cli/run.py | 27 +--- activitysim/cli/test/test_extensions.py | 176 +++++++++++++++++++++++ activitysim/core/extensions.py | 32 +++++ activitysim/core/mp_tasks.py | 14 +- activitysim/core/test/test_extensions.py | 162 +++++++++++++++++++++ activitysim/core/workflow/runner.py | 9 +- activitysim/core/workflow/state.py | 68 +++------ docs/dev-guide/core-workflow-api.md | 29 ++++ 9 files changed, 479 insertions(+), 81 deletions(-) create mode 100644 .github/workflows/extension-tests.yml create mode 100644 activitysim/cli/test/test_extensions.py create mode 100644 activitysim/core/extensions.py create mode 100644 activitysim/core/test/test_extensions.py diff --git a/.github/workflows/extension-tests.yml b/.github/workflows/extension-tests.yml new file mode 100644 index 0000000000..3cecdcb04f --- /dev/null +++ b/.github/workflows/extension-tests.yml @@ -0,0 +1,43 @@ +name: Extension Loading + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + extensions: + name: Extensions (${{ matrix.os }}, Python ${{ matrix.python }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ['3.10'] + include: + - os: ubuntu-latest + python: '3.12' + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: '0.7.12' + enable-cache: true + cache-dependency-glob: uv.lock + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install ActivitySim + run: uv sync --locked --python ${{ matrix.python }} + - name: Test parent and worker extension imports + run: >- + uv run --locked --python ${{ matrix.python }} pytest + activitysim/core/test/test_extensions.py + activitysim/cli/test/test_extensions.py + --junitxml=extension-results.xml + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: extension-results-${{ matrix.os }}-${{ matrix.python }} + path: extension-results.xml diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index ad91c4f167..be6d493dda 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -17,6 +17,7 @@ from activitysim.core import chunk, config, mem, timing, tracing, workflow from activitysim.core.configuration import FileSystem, Settings from activitysim.core.run_id import RunId +from activitysim.core.extensions import import_extension from activitysim.abm.models.settings_checker import check_model_settings @@ -24,6 +25,7 @@ INJECTABLES = [ + "working_dir", "data_dir", "configs_dir", "data_model_dir", @@ -163,31 +165,13 @@ def inject_arg(name, value): if args.working_dir: # activitysim will look in the current working directory for # 'configs', 'data', and 'output' folders by default + args.working_dir = os.path.abspath(args.working_dir) os.chdir(args.working_dir) inject_arg("run_id", state.tracing.run_id) - if args.ext: - for e in args.ext: - basepath, extpath = os.path.split(e) - if not basepath: - basepath = "." - sys.path.insert(0, os.path.abspath(basepath)) - try: - importlib.import_module(extpath) - except ImportError as err: - logger.exception("ImportError") - raise - except Exception as err: - logger.exception(f"Error {err}") - raise - finally: - del sys.path[0] - inject_arg("imported_extensions", args.ext) - else: - inject_arg("imported_extensions", ()) - state.filesystem = FileSystem.parse_args(args) + state.import_extensions(args.ext or [], append=False) for config_dir in state.filesystem.get_configs_dir(): if not config_dir.is_dir(): print(f"missing config directory: {config_dir}", file=sys.stderr) @@ -405,8 +389,9 @@ def run(args): if extension_names: for ext in extension_names: try: + extension = import_extension(ext) settings_checker_ext = importlib.import_module( - ext + ".settings_checker" + extension.__name__ + ".settings_checker" ) extension_checker_settings.update( settings_checker_ext.EXTENSION_CHECKER_SETTINGS diff --git a/activitysim/cli/test/test_extensions.py b/activitysim/cli/test/test_extensions.py new file mode 100644 index 0000000000..0b0ad770d0 --- /dev/null +++ b/activitysim/cli/test/test_extensions.py @@ -0,0 +1,176 @@ +"""No-data, end-to-end regressions for CLI and Python API extension loading.""" + +from __future__ import annotations + +import os +import subprocess +import sys + +import pytest + +EXTENSION = """\ +import multiprocessing +from pathlib import Path +from activitysim.core import workflow +from .value import VALUE + +@workflow.step(cache=True, kind="cached_object", overloading=True) +def network_los_preload(state: workflow.State): + return None + +@workflow.step(cache=True, kind="cached_object", overloading=True) +def shadow_pricing_info(state: workflow.State): + return None + +@workflow.step(cache=True, kind="cached_object", overloading=True) +def shadow_pricing_choice_info(state: workflow.State): + return None + +@workflow.step +def extension_hello(state: workflow.State): + process = multiprocessing.current_process().name + Path(state.get_output_file_path("hello.txt")).write_text(f"{VALUE}:{process}") +""" + +SETTINGS = """\ +models: [extension_hello] +num_processes: 1 +multiprocess_steps: + - name: mp_hello + begin: extension_hello + num_processes: 1 +check_model_settings: true +memory_profile: false +sharrow: false +use_shadow_pricing: false +""" + +API_RUNNER = """\ +import multiprocessing +import os +import sys +from pathlib import Path +from activitysim import abm +from activitysim.core import workflow + +if __name__ == "__main__": + multiprocessing.set_start_method("spawn", force=True) + model, output, extension, multiprocess, elsewhere = sys.argv[1:] + state = workflow.State.make_default(Path(model), output_dir=Path(output)) + state.import_extensions(extension) + state.settings.multiprocess = multiprocess == "yes" + os.chdir(elsewhere) + state.run.all() +""" + + +@pytest.fixture +def model(tmp_path): + root = tmp_path / "model space" + for directory in ("configs", "data", "extensions"): + (root / directory).mkdir(parents=True) + (root / "configs" / "settings.yaml").write_text(SETTINGS) + (root / "extensions" / "__init__.py").write_text(EXTENSION) + (root / "extensions" / "value.py").write_text("VALUE = 42\n") + (root / "extensions" / "settings_checker.py").write_text( + 'print("EXTENSION_SETTINGS_CHECKER_IMPORTED", flush=True)\n' + "EXTENSION_CHECKER_SETTINGS = {}\n" + ) + return root + + +def run_and_check(command, cwd, output, multiprocess): + env = os.environ.copy() + for variable in ( + "MKL_NUM_THREADS", + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "NUMBA_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "NUMEXPR_NUM_THREADS", + ): + env[variable] = "1" + result = subprocess.run( + command, + cwd=cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=120, + ) + assert result.returncode == 0, result.stdout + assert (output / "hello.txt").read_text() == ( + "42:mp_hello" if multiprocess else "42:MainProcess" + ) + return result.stdout + + +@pytest.mark.parametrize("multiprocess", [False, True], ids=["single", "multiprocess"]) +@pytest.mark.parametrize( + "form", ["relative", "absolute", "bare", "dot", "trailing", "working-dir"] +) +def test_cli_extensions(model, tmp_path, multiprocess, form): + output = tmp_path / "output" + cwd = tmp_path + extra = [] + if form == "relative": + extension = os.path.join(model.name, "extensions") + elif form == "absolute": + extension = str(model / "extensions") + elif form == "working-dir": + # A relative -w must not be applied twice after chdir. + extension = "extensions" + extra = ["-w", model.name] + else: + cwd = model + extension = { + "bare": "extensions", + "dot": "./extensions", + "trailing": "extensions" + os.sep, + }[form] + command = [ + sys.executable, + "-m", + "activitysim", + "run", + "-c", + str(model / "configs"), + "-d", + str(model / "data"), + "-o", + str(output), + "--ext", + extension, + *extra, + ] + if multiprocess: + command += ["-m"] + stdout = run_and_check(command, cwd, output, multiprocess) + assert "EXTENSION_SETTINGS_CHECKER_IMPORTED" in stdout + + +@pytest.mark.parametrize("multiprocess", [False, True], ids=["single", "spawn"]) +@pytest.mark.parametrize("form", ["relative", "absolute", "dot", "trailing"]) +def test_api_extensions(model, tmp_path, multiprocess, form): + runner = tmp_path / "api_runner.py" + runner.write_text(API_RUNNER) + output = tmp_path / "output" + elsewhere = tmp_path / "unrelated cwd" + elsewhere.mkdir() + extension = { + "relative": "extensions", + "absolute": str(model / "extensions"), + "dot": "./extensions", + "trailing": "extensions" + os.sep, + }[form] + command = [ + sys.executable, + str(runner), + str(model), + str(output), + extension, + "yes" if multiprocess else "no", + str(elsewhere), + ] + run_and_check(command, tmp_path, output, multiprocess) diff --git a/activitysim/core/extensions.py b/activitysim/core/extensions.py new file mode 100644 index 0000000000..53394770c5 --- /dev/null +++ b/activitysim/core/extensions.py @@ -0,0 +1,32 @@ +"""Shared extension loading for the CLI, workflow states, and workers.""" + +from __future__ import annotations + +import importlib +import os +import sys +from pathlib import Path +from types import ModuleType + + +def resolve_extension(extension: str | os.PathLike, working_dir=None) -> str: + """Freeze the search directory while retaining the Python module name. + + The final path component is a module name (possibly dotted), not a Python + filename. Like Python imports, a module may also be found elsewhere on + sys.path. Store an absolute location so workers need not share the caller's + current directory. abspath normalizes separators and trailing slashes without + resolving symlinks, which could change the name of a package being imported. + """ + return os.path.abspath(Path(working_dir or Path.cwd()) / extension) + + +def import_extension(extension: str | os.PathLike) -> ModuleType: + """Import a normalized extension location, restoring sys.path on failure too.""" + location = Path(extension) + original_path = sys.path[:] + sys.path.insert(0, str(location.parent)) + try: + return importlib.import_module(location.name) + finally: + sys.path[:] = original_path diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 101173321b..d626d315c9 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -3,7 +3,6 @@ from __future__ import annotations import glob -import importlib import logging import multiprocessing import os @@ -928,18 +927,7 @@ def setup_injectables_and_logging(injectables, locutor: bool = True) -> workflow # re-import extension modules to register injectables ext = state.get_injectable("imported_extensions", default=()) - for e in ext: - basepath, extpath = os.path.split(e) - if not basepath: - basepath = "." - sys.path.insert(0, basepath) - try: - importlib.import_module(e) - except ImportError as err: - logger.exception("ImportError") - raise - finally: - del sys.path[0] + state.import_extensions(ext, append=False) state.add_injectable("is_sub_task", True) state.add_injectable("locutor", locutor) diff --git a/activitysim/core/test/test_extensions.py b/activitysim/core/test/test_extensions.py new file mode 100644 index 0000000000..9e34dd788d --- /dev/null +++ b/activitysim/core/test/test_extensions.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import multiprocessing +import os +import sys +import uuid +from pathlib import Path + +import pytest + +from activitysim.core import workflow +from activitysim.core.extensions import import_extension + + +@pytest.fixture +def extension(tmp_path): + name = "extension_" + uuid.uuid4().hex + package = tmp_path / "model space" / name + package.mkdir(parents=True) + (package / "__init__.py").write_text("from .values import VALUE\n") + (package / "values.py").write_text("VALUE = 42\n") + yield package + for key in list(sys.modules): + if key == name or key.startswith(name + "."): + del sys.modules[key] + + +def state_at(directory): + for name in ("configs", "data"): + (directory / name).mkdir(exist_ok=True) + return workflow.State.make_default(directory) + + +@pytest.mark.parametrize( + "form", ["bare", "relative", "absolute", "dot", "trailing", "pathlike", "posix"] +) +def test_api_paths(extension, tmp_path, monkeypatch, form): + state = state_at(extension.parent) + # The state's working directory deliberately differs from the process CWD. + monkeypatch.chdir(tmp_path) + options = { + "bare": extension.name, + "relative": os.path.join("..", extension.parent.name, extension.name), + "absolute": str(extension), + "dot": os.path.join(".", extension.name), + "trailing": str(extension) + os.sep, + "pathlike": Path(extension.name), + "posix": extension.as_posix(), + } + old_path = sys.path[:] + state.import_extensions(options[form]) + assert sys.path == old_path + assert sys.modules[extension.name].VALUE == 42 + assert state.get("imported_extensions") == [str(extension)] + + +def test_api_without_filesystem(extension, monkeypatch): + monkeypatch.chdir(extension.parent) + state = workflow.State() + state.import_extensions(extension.name) + assert state.get("imported_extensions") == [str(extension)] + + +def test_append_replace_and_noop(extension): + state = state_at(extension.parent) + state.import_extensions(extension.name) + before = state.get("imported_extensions") + state.import_extensions([extension.name + ".values"]) + assert before == [str(extension)] # Do not mutate a caller's retained list. + assert len(state.get("imported_extensions")) == 2 + state.import_extensions(None, append=False) + assert len(state.get("imported_extensions")) == 2 + state.import_extensions(extension.name, append=False) + assert state.get("imported_extensions") == [str(extension)] + state.import_extensions([], append=False) + assert state.get("imported_extensions") == [] + + +def test_dotted_name_on_python_path(extension, tmp_path, monkeypatch): + monkeypatch.syspath_prepend(str(extension.parent)) + state = state_at(tmp_path) + state.import_extensions(extension.name + ".values") + assert sys.modules[extension.name + ".values"].VALUE == 42 + + +@pytest.mark.parametrize( + "source,exception", + [ + ("import missing_extension_dependency_1118", ModuleNotFoundError), + ('raise RuntimeError("extension failed")', RuntimeError), + ], +) +def test_failed_import_restores_path_and_registration(extension, source, exception): + (extension / "__init__.py").write_text(source) + state = state_at(extension.parent) + old_path = sys.path[:] + with pytest.raises(exception): + state.import_extensions(extension.name) + assert sys.path == old_path + assert state.get("imported_extensions", []) == [] + + +def test_missing_extension(extension): + state = state_at(extension.parent) + old_path = sys.path[:] + with pytest.raises(ModuleNotFoundError, match="missing_extension_1118"): + state.import_extensions("missing_extension_1118") + assert sys.path == old_path + + +def _worker_import(injectables, cwd, name, connection): + """Use the real worker setup in a fresh process with a different CWD.""" + from activitysim.core.mp_tasks import setup_injectables_and_logging + + os.chdir(cwd) + # fork inherits modules; explicitly require an import from the saved location. + for key in list(sys.modules): + if key == name or key.startswith(name + "."): + del sys.modules[key] + try: + old_path = sys.path[:] + state = setup_injectables_and_logging(injectables) + module = import_extension(state.get("imported_extensions")[0]) + connection.send((module.VALUE, module.__file__, sys.path == old_path)) + finally: + connection.close() + + +@pytest.mark.parametrize("method", multiprocessing.get_all_start_methods()) +def test_worker_after_cwd_changes(extension, tmp_path, method): + state = state_at(extension.parent) + state.import_extensions(extension.name) + elsewhere = tmp_path / "other cwd" + elsewhere.mkdir() + # A same-named module in the worker CWD must not shadow the saved location. + (elsewhere / (extension.name + ".py")).write_text("VALUE = -1\n") + injectables = dict( + configs_dir=[extension.parent / "configs"], + data_dir=[extension.parent / "data"], + output_dir=tmp_path / "worker-output", + imported_extensions=state.get("imported_extensions"), + ) + context = multiprocessing.get_context(method) + receiver, sender = context.Pipe(duplex=False) + process = context.Process( + target=_worker_import, args=(injectables, elsewhere, extension.name, sender) + ) + process.start() + sender.close() + try: + assert receiver.poll(60), "Worker did not return an imported extension" + value, filename, restored = receiver.recv() + process.join(30) + assert process.exitcode == 0 + assert value == 42 + assert Path(filename) == extension / "__init__.py" + assert restored + finally: + if process.is_alive(): + process.terminate() + process.join(10) + receiver.close() diff --git a/activitysim/core/workflow/runner.py b/activitysim/core/workflow/runner.py index 79ecd0ed4f..e20bb7275b 100644 --- a/activitysim/core/workflow/runner.py +++ b/activitysim/core/workflow/runner.py @@ -412,7 +412,14 @@ def all( from activitysim.cli.run import INJECTABLES from activitysim.core import mp_tasks - injectables = {k: self._obj.get_injectable(k) for k in INJECTABLES} + # API callers need not define optional CLI injectables (such as + # cache_dir). Preserve the state's working directory as well: + # its config/data paths need not be relative to the process CWD. + injectables = { + k: value + for k in INJECTABLES + if (value := self._obj.get_injectable(k, None)) is not None + } injectables["settings"] = self._obj.settings # injectables["settings_package"] = state.settings.dict() mp_tasks.run_multiprocess(self._obj, injectables) diff --git a/activitysim/core/workflow/state.py b/activitysim/core/workflow/state.py index de22b0687d..e0d032d67f 100644 --- a/activitysim/core/workflow/state.py +++ b/activitysim/core/workflow/state.py @@ -1,10 +1,8 @@ from __future__ import annotations -import importlib import io import logging import os -import sys import textwrap import warnings from collections.abc import Iterable @@ -21,6 +19,7 @@ import activitysim.core.random from activitysim.core.configuration import FileSystem, NetworkSettings, Settings from activitysim.core.exceptions import StateAccessError, CheckpointNameNotFoundError +from activitysim.core.extensions import import_extension, resolve_extension from activitysim.core.workflow.checkpoint import LAST_CHECKPOINT, Checkpoints from activitysim.core.workflow.chunking import Chunking from activitysim.core.workflow.dataset import Datasets @@ -191,60 +190,37 @@ def _initialize_prng(self, base_seed=None): base_seed = self.settings.rng_base_seed self._context["prng"].set_base_seed(base_seed) - def import_extensions(self, ext: str | Iterable[str] = None, append=True) -> None: - """ - Import one or more extension modules for use with this model. - - This method isn't really necessary for single-process model - runs, as extension modules can be imported in the normal manner - for python. The real reason this methid is here is to support - multiprocessing. The names of extension modules imported with - this method will be saved and passed through to subtask workers, - which will also import the extensions and make them available as - model steps within the workers. + def import_extensions( + self, ext: str | os.PathLike | Iterable[str | os.PathLike] = None, append=True + ) -> None: + """Import extensions and register their locations for multiprocessing. Parameters ---------- - ext : str | Iterable[str] - Names of extension modules to import. They should be module - or package names that can be imported from this state's working - directory. If they need to be imported from elsewhere, the - name should be the relative path to the extension module from - the working directory. + ext : str, path-like, or iterable of these + Package paths or Python module names (including dotted names). + Relative paths are resolved against this state's working directory, + or the current directory if no working directory is configured. + For a single Python file, use the module name without ``.py``. + Absolute import locations are saved so workers can load the same + extensions even if their current directory differs from the caller's. append : bool, default True - Extension names will be appended to the "imported_extensions" list - in this State's context (creating it if needed). Setting this - argument to false will remove references to any existing extensions, - before adding this new extension to the list. + Append to the registered extensions. If false, replace the list. + This does not unload previously imported Python modules. """ if ext is None: return - if isinstance(ext, str): + if isinstance(ext, (str, os.PathLike)): ext = [ext] - if append: - extensions = self.get("imported_extensions", []) - else: - extensions = [] - if self.filesystem.working_dir: + extensions = list(self.get("imported_extensions", [])) if append else [] + try: working_dir = self.filesystem.working_dir - else: - working_dir = Path.cwd() + except StateAccessError: + working_dir = None for e in ext: - basepath, extpath = os.path.split(working_dir.joinpath(e)) - if not basepath: - basepath = "." - sys.path.insert(0, os.path.abspath(basepath)) - try: - importlib.import_module(extpath) - except ImportError: - logger.exception("ImportError") - raise - except Exception as err: - logger.exception(f"Error {err}") - raise - finally: - del sys.path[0] - extensions.append(e) + location = resolve_extension(e, working_dir) + import_extension(location) + extensions.append(location) self.set("imported_extensions", extensions) filesystem: FileSystem = StateAttr(FileSystem) diff --git a/docs/dev-guide/core-workflow-api.md b/docs/dev-guide/core-workflow-api.md index 71cb339c15..1bf67d949e 100644 --- a/docs/dev-guide/core-workflow-api.md +++ b/docs/dev-guide/core-workflow-api.md @@ -38,6 +38,35 @@ State.network_settings ``` +### Importing extensions + +Register custom model packages with `State.import_extensions` before running +the model. Relative package paths are interpreted relative to the state's +working directory, which need not be the Python process's current directory: + +```python +state = State.make_default("/path/to/model") +state.import_extensions("extensions") +state.run.all() +``` + +Absolute package paths, path-like objects, and lists of extensions are also +accepted. Python module names, including dotted names such as `my_package.models`, +are supported; for a single Python file, omit the `.py` suffix. Use a unique +package name for each extension, as Python caches imports by module name. + +The CLI uses the same loader, for example +`activitysim run -w /path/to/model --ext extensions`, or +`activitysim run -c model/configs -d model/data -o output --ext model/extensions`. +Both interfaces record absolute import locations in `imported_extensions` so +multiprocessing workers can reimport the extensions even from another current +directory. Package paths may include `./` or a trailing directory separator. +These rules are the same for single-process and multiprocessing runs. + +`append=False` replaces the registered extension list; it does not unload modules +already imported into the Python process. Importing extension code temporarily +adds its parent directory to `sys.path`, which is restored even if import fails. + ## Basic Context Management From 41c78054229a1aa084e0d991b11a4d09fad23db4 Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Wed, 23 Sep 2026 16:26:13 -0500 Subject: [PATCH 2/4] Use lockfile-compatible Python versions in extension CI --- .github/workflows/extension-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/extension-tests.yml b/.github/workflows/extension-tests.yml index 3cecdcb04f..e73255cef7 100644 --- a/.github/workflows/extension-tests.yml +++ b/.github/workflows/extension-tests.yml @@ -16,7 +16,7 @@ jobs: python: ['3.10'] include: - os: ubuntu-latest - python: '3.12' + python: '3.11' steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 From 42dad03b576ae95861bab411a35ced68241abfdd Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Wed, 23 Sep 2026 16:47:41 -0500 Subject: [PATCH 3/4] Run extension regressions through the existing core workflow --- .github/workflows/extension-tests.yml | 43 --------------------------- 1 file changed, 43 deletions(-) delete mode 100644 .github/workflows/extension-tests.yml diff --git a/.github/workflows/extension-tests.yml b/.github/workflows/extension-tests.yml deleted file mode 100644 index e73255cef7..0000000000 --- a/.github/workflows/extension-tests.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Extension Loading - -on: - push: - pull_request: - workflow_dispatch: - -jobs: - extensions: - name: Extensions (${{ matrix.os }}, Python ${{ matrix.python }}) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.10'] - include: - - os: ubuntu-latest - python: '3.11' - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - with: - version: '0.7.12' - enable-cache: true - cache-dependency-glob: uv.lock - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - name: Install ActivitySim - run: uv sync --locked --python ${{ matrix.python }} - - name: Test parent and worker extension imports - run: >- - uv run --locked --python ${{ matrix.python }} pytest - activitysim/core/test/test_extensions.py - activitysim/cli/test/test_extensions.py - --junitxml=extension-results.xml - - name: Upload test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: extension-results-${{ matrix.os }}-${{ matrix.python }} - path: extension-results.xml From 38ba3c307c56c65cc66bdc4f54ed2b3af62e23fd Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Wed, 23 Sep 2026 19:58:55 -0500 Subject: [PATCH 4/4] Preserve importable extension names for existing model consumers --- activitysim/cli/run.py | 5 ++-- activitysim/cli/test/test_extensions.py | 5 ++++ activitysim/core/mp_tasks.py | 3 ++- activitysim/core/test/test_extensions.py | 32 ++++++++++++++++++------ activitysim/core/workflow/state.py | 9 +++++-- docs/dev-guide/core-workflow-api.md | 8 +++--- 6 files changed, 46 insertions(+), 16 deletions(-) diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index be6d493dda..d7d4ff7587 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -17,7 +17,6 @@ from activitysim.core import chunk, config, mem, timing, tracing, workflow from activitysim.core.configuration import FileSystem, Settings from activitysim.core.run_id import RunId -from activitysim.core.extensions import import_extension from activitysim.abm.models.settings_checker import check_model_settings @@ -33,6 +32,7 @@ "cache_dir", "settings_file_name", "imported_extensions", + "_extension_locations", "run_timestamp", "run_id", ] @@ -389,9 +389,8 @@ def run(args): if extension_names: for ext in extension_names: try: - extension = import_extension(ext) settings_checker_ext = importlib.import_module( - extension.__name__ + ".settings_checker" + ext + ".settings_checker" ) extension_checker_settings.update( settings_checker_ext.EXTENSION_CHECKER_SETTINGS diff --git a/activitysim/cli/test/test_extensions.py b/activitysim/cli/test/test_extensions.py index 0b0ad770d0..25d27929a2 100644 --- a/activitysim/cli/test/test_extensions.py +++ b/activitysim/cli/test/test_extensions.py @@ -46,6 +46,7 @@ def extension_hello(state: workflow.State): """ API_RUNNER = """\ +import importlib import multiprocessing import os import sys @@ -58,6 +59,10 @@ def extension_hello(state: workflow.State): model, output, extension, multiprocess, elsewhere = sys.argv[1:] state = workflow.State.make_default(Path(model), output_dir=Path(output)) state.import_extensions(extension) + # Existing example repositories use this public list as importable names. + for name in state.get_injectable("imported_extensions"): + checker = importlib.import_module(name + ".settings_checker") + assert checker.EXTENSION_CHECKER_SETTINGS == {} state.settings.multiprocess = multiprocess == "yes" os.chdir(elsewhere) state.run.all() diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index d626d315c9..c17882aae9 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -927,7 +927,8 @@ def setup_injectables_and_logging(injectables, locutor: bool = True) -> workflow # re-import extension modules to register injectables ext = state.get_injectable("imported_extensions", default=()) - state.import_extensions(ext, append=False) + locations = state.get_injectable("_extension_locations", default={}) + state.import_extensions([locations.get(e, e) for e in ext], append=False) state.add_injectable("is_sub_task", True) state.add_injectable("locutor", locutor) diff --git a/activitysim/core/test/test_extensions.py b/activitysim/core/test/test_extensions.py index 9e34dd788d..0a8f0c6d92 100644 --- a/activitysim/core/test/test_extensions.py +++ b/activitysim/core/test/test_extensions.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib import multiprocessing import os import sys @@ -9,7 +10,6 @@ import pytest from activitysim.core import workflow -from activitysim.core.extensions import import_extension @pytest.fixture @@ -51,29 +51,41 @@ def test_api_paths(extension, tmp_path, monkeypatch, form): state.import_extensions(options[form]) assert sys.path == old_path assert sys.modules[extension.name].VALUE == 42 - assert state.get("imported_extensions") == [str(extension)] + assert state.get("_extension_locations") == {extension.name: str(extension)} + # Match downstream consumers such as SANDAG's settings-checker discovery. + for name in state.get("imported_extensions"): + assert importlib.import_module(name + ".values").VALUE == 42 + assert state.get("imported_extensions") == [extension.name] def test_api_without_filesystem(extension, monkeypatch): monkeypatch.chdir(extension.parent) state = workflow.State() state.import_extensions(extension.name) - assert state.get("imported_extensions") == [str(extension)] + assert state.get("imported_extensions") == [extension.name] def test_append_replace_and_noop(extension): state = state_at(extension.parent) state.import_extensions(extension.name) before = state.get("imported_extensions") + before_locations = state.get("_extension_locations") state.import_extensions([extension.name + ".values"]) - assert before == [str(extension)] # Do not mutate a caller's retained list. + assert before == [extension.name] # Do not mutate a caller's retained list. assert len(state.get("imported_extensions")) == 2 + assert before_locations == {extension.name: str(extension)} + assert set(state.get("_extension_locations")) == { + extension.name, + extension.name + ".values", + } state.import_extensions(None, append=False) assert len(state.get("imported_extensions")) == 2 state.import_extensions(extension.name, append=False) - assert state.get("imported_extensions") == [str(extension)] + assert state.get("imported_extensions") == [extension.name] + assert state.get("_extension_locations") == {extension.name: str(extension)} state.import_extensions([], append=False) assert state.get("imported_extensions") == [] + assert state.get("_extension_locations") == {} def test_dotted_name_on_python_path(extension, tmp_path, monkeypatch): @@ -120,14 +132,15 @@ def _worker_import(injectables, cwd, name, connection): try: old_path = sys.path[:] state = setup_injectables_and_logging(injectables) - module = import_extension(state.get("imported_extensions")[0]) + module = importlib.import_module(state.get("imported_extensions")[0]) connection.send((module.VALUE, module.__file__, sys.path == old_path)) finally: connection.close() @pytest.mark.parametrize("method", multiprocessing.get_all_start_methods()) -def test_worker_after_cwd_changes(extension, tmp_path, method): +@pytest.mark.parametrize("saved_locations", [True, False]) +def test_worker_after_cwd_changes(extension, tmp_path, method, saved_locations): state = state_at(extension.parent) state.import_extensions(extension.name) elsewhere = tmp_path / "other cwd" @@ -139,7 +152,12 @@ def test_worker_after_cwd_changes(extension, tmp_path, method): data_dir=[extension.parent / "data"], output_dir=tmp_path / "worker-output", imported_extensions=state.get("imported_extensions"), + _extension_locations=state.get("_extension_locations"), ) + if not saved_locations: + # Legacy callers can still supply just the public module-name registry. + injectables.pop("_extension_locations") + injectables["working_dir"] = extension.parent context = multiprocessing.get_context(method) receiver, sender = context.Pipe(duplex=False) process = context.Process( diff --git a/activitysim/core/workflow/state.py b/activitysim/core/workflow/state.py index e0d032d67f..1971bfca90 100644 --- a/activitysim/core/workflow/state.py +++ b/activitysim/core/workflow/state.py @@ -213,15 +213,20 @@ def import_extensions( if isinstance(ext, (str, os.PathLike)): ext = [ext] extensions = list(self.get("imported_extensions", [])) if append else [] + locations = dict(self.get("_extension_locations", {})) if append else {} try: working_dir = self.filesystem.working_dir except StateAccessError: working_dir = None for e in ext: location = resolve_extension(e, working_dir) - import_extension(location) - extensions.append(location) + module = import_extension(location) + # Keep the public registry usable with importlib (e.g. external + # settings checkers). Worker lookup paths are separate metadata. + extensions.append(module.__name__) + locations[module.__name__] = location self.set("imported_extensions", extensions) + self.set("_extension_locations", locations) filesystem: FileSystem = StateAttr(FileSystem) settings: Settings = StateAttr(Settings) diff --git a/docs/dev-guide/core-workflow-api.md b/docs/dev-guide/core-workflow-api.md index 1bf67d949e..4401427178 100644 --- a/docs/dev-guide/core-workflow-api.md +++ b/docs/dev-guide/core-workflow-api.md @@ -58,9 +58,11 @@ package name for each extension, as Python caches imports by module name. The CLI uses the same loader, for example `activitysim run -w /path/to/model --ext extensions`, or `activitysim run -c model/configs -d model/data -o output --ext model/extensions`. -Both interfaces record absolute import locations in `imported_extensions` so -multiprocessing workers can reimport the extensions even from another current -directory. Package paths may include `./` or a trailing directory separator. +Both interfaces retain importable module names in `imported_extensions`, so +existing consumers can use them with `importlib.import_module`, including to +load an extension's `settings_checker` submodule. Absolute import locations are +stored separately in internal worker metadata, allowing multiprocessing workers +to reimport extensions even from another current directory. Package paths may include `./` or a trailing directory separator. These rules are the same for single-process and multiprocessing runs. `append=False` replaces the registered extension list; it does not unload modules