From 9949e04928bac18bf84f78dea3dabea4be702efd Mon Sep 17 00:00:00 2001 From: Michel Lind Date: Tue, 15 Sep 2026 18:28:53 +0100 Subject: [PATCH 1/3] getdeps: add a vendor subcommand `getdeps.py vendor --output-dir DIR project` fetches every third-party dependency of the project and copies its source tree to DIR/, skipping deps satisfied by system packages and the project itself. It also writes DIR/getdeps-vendor.txt listing each vendored project with its fetcher hash (the pinned git revision or the archive sha256). This is the getdeps analogue of `cargo vendor` / `go mod vendor`: distro packagers can ship the result as a vendor tarball next to the project source, as Fedora's vendored-dependency guidelines require, and a follow-up change will let `build` consume such a directory offline. Extracted trees rather than the original archives and clones are vendored so the result is self-contained, reviewable and license-scannable; .git directories are dropped and symlinks are followed for the same reason (subproject fetchers symlink into the scratch dir, which does not exist on an offline builder). Tested on Fedora 44: `getdeps.py --allow-system-packages vendor --no-tests --output-dir /var/tmp/cachelib-vendor cachelib` produced eleven trees (about 330 MB, fbthrift being 216 MB of that) plus getdeps-vendor.txt, with no .git directories or symlinks left behind. Once the Fedora rpm mappings from facebook/CacheLib#488 land that drops to the seven projects Fedora does not package. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Michel Lind --- build/fbcode_builder/getdeps/cli.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/build/fbcode_builder/getdeps/cli.py b/build/fbcode_builder/getdeps/cli.py index 818ee8f94..0265d2a96 100644 --- a/build/fbcode_builder/getdeps/cli.py +++ b/build/fbcode_builder/getdeps/cli.py @@ -192,6 +192,54 @@ def run_project_cmd(self, args, loader, manifest): fetcher.update() +@cmd("vendor", "copy the sources of a project's dependencies into a directory") +class VendorCmd(ProjectCmdBase): + """Populate a directory with one source tree per third-party dependency, + analogous to `cargo vendor`, so that a later build can run without + network access (see --vendor-dir).""" + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--output-dir", + required=True, + help=( + "Directory to populate; each dependency is copied " + "to /" + ), + ) + parser.add_argument( + "--host-type", + help="Vendor deps for this host type rather than the current system", + ) + + def run_project_cmd(self, args, loader, manifest): + os.makedirs(args.output_dir, exist_ok=True) + vendored = [] + for m in loader.manifests_in_dependency_order(): + if m == manifest: + continue + fetcher = loader.create_fetcher(m) + if isinstance(fetcher, SystemPackageFetcher): + # Satisfied by system packages; nothing to vendor + continue + fetcher.update() + dest = os.path.join(args.output_dir, m.name) + if os.path.exists(dest): + shutil.rmtree(dest) + print("Vendoring %s -> %s" % (m.name, dest)) + # Follow symlinks so the result is self-contained: subproject + # fetchers link into the scratch dir, which won't exist offline. + shutil.copytree( + fetcher.get_src_dir(), + dest, + ignore=shutil.ignore_patterns(".git"), + ignore_dangling_symlinks=True, + ) + vendored.append("%s %s\n" % (m.name, fetcher.hash())) + with open(os.path.join(args.output_dir, "getdeps-vendor.txt"), "w") as f: + f.writelines(vendored) + + @cmd("install-system-deps", "Install system packages to satisfy the deps for a project") class InstallSysDepsCmd(ProjectCmdBase): def setup_project_cmd_parser(self, parser): From 8f865f62d02dbbaf4679e909fef60498a5b6b895 Mon Sep 17 00:00:00 2001 From: Michel Lind Date: Tue, 15 Sep 2026 18:33:30 +0100 Subject: [PATCH 2/3] getdeps: add --vendor-dir to build from vendored sources offline `getdeps.py --vendor-dir DIR build project` takes every third-party dependency from DIR/, as populated by `getdeps.py vendor`, instead of cloning or downloading it. The lookup sits in the manifest's fetcher selection after the system-package check, so a dependency that --allow-system-packages resolves to an installed package still wins, and anything not found in DIR raises an error naming the missing project rather than falling back to the network. That single check is the offline guarantee: no GitFetcher or ArchiveFetcher is ever constructed. The vendored tree is wrapped in the existing LocalDirFetcher, the same mechanism --src-dir uses, so no fetcher code changes. Its hash is fixed and it always reports the sources as changed, which means repeated builds against a vendor dir reconfigure their dependencies each time; that is acceptable for the one-shot distro builds this is meant for. --free-up-disk only removed build trees whose sources live under the scratch dir, a guard meant to protect a user's own --src-dir checkout. Vendored sources are ours to clean up after, so the guard now also accepts the vendor dir (stored realpath'd so the prefix comparison matches LocalDirFetcher's realpath'd source paths). Known limitation: patchfiles are applied with `git apply` from the enclosing git top-level, so a vendor dir placed inside another git checkout would mis-apply patches for the (few) manifests that carry them. A distro build directory is not a git checkout, so this does not affect the intended use; making the patch step independent of the surrounding repository is left for a follow-up. Tested on Fedora 44 (aarch64, 4 cores) on a tree that also carried facebook/CacheLib#488, #489 and #490, against the seven-project vendor dir produced by the previous commit (magic_enum, sparsemap, folly, fizz, wangle, mvfst, fbthrift; 276 MB): unshare -rn python3 build/fbcode_builder/getdeps.py \ --allow-system-packages --vendor-dir /var/tmp/cachelib-vendor \ --scratch-path /var/tmp/getdeps-offline-scratch --num-jobs 2 \ --extra-cmake-defines '{"CMAKE_POLICY_VERSION_MINIMUM":"3.5"}' \ build --free-up-disk --no-tests --src-dir=. cachelib `unshare -rn` puts the build in its own network namespace. Inspected from outside while it ran, the build process was in net:[4026532485] versus the shell's net:[4026531833]; `nsenter -n ip -brief link` inside it showed only `lo` DOWN, `getent hosts github.com` failed (exit 2), and `ss -tunap` listed no sockets. The build finished in 72 minutes with exit 0, all eight projects installed, no "Download with" or "Cloning" line in the log, and cachebench linking the system glog, liboqs, libaio and libnuma. Requesting a project that is neither vendored nor allowed from system packages fails with the new error, and `show-source-dir --recursive` resolves every vendored project into the vendor dir. The --free-up-disk fix was verified separately: an offline sparsemap build with the flag left no build tree behind, where the 72-minute run (made before the fix) had left 11 GB. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Michel Lind --- build/fbcode_builder/README.md | 12 ++++++++++++ build/fbcode_builder/getdeps/builder.py | 8 ++++++-- build/fbcode_builder/getdeps/buildopts.py | 6 ++++++ build/fbcode_builder/getdeps/cli.py | 9 +++++++++ build/fbcode_builder/getdeps/manifest.py | 12 ++++++++++++ 5 files changed, 45 insertions(+), 2 deletions(-) diff --git a/build/fbcode_builder/README.md b/build/fbcode_builder/README.md index 62c2f5b56..30c8c1449 100644 --- a/build/fbcode_builder/README.md +++ b/build/fbcode_builder/README.md @@ -17,6 +17,18 @@ repositories that use `fbcode_builder`. Typically this directory is copied into the open source repositories as `build/fbcode_builder/`. +## Vendoring dependencies for offline builds + +Distributions typically require builds to run without network access and to +ship third-party sources alongside the project. `getdeps.py vendor +--output-dir DIR project` copies the source tree of every dependency that is +not satisfied by system packages to `DIR/` and records what it +vendored in `DIR/getdeps-vendor.txt`. A later `getdeps.py --vendor-dir DIR +build project` then takes those trees instead of fetching, and fails if a +dependency is missing from `DIR` rather than downloading it. Pass the same +`--allow-system-packages` and `--no-tests` options to both commands so they +agree on the dependency set. + # Project Configuration Files The `manifests` subdirectory contains configuration files for many different diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index c91de65d2..c49204904 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -226,8 +226,12 @@ def build(self, reconfigure: bool) -> None: self._build(reconfigure=reconfigure) if self.build_opts.free_up_disk: - # don't clean --src-dir=. case as user may want to build again or run tests on the build - if self.src_dir.startswith(self.build_opts.scratch_dir) and os.path.isdir( + # don't clean --src-dir=. case as user may want to build again or + # run tests on the build; vendored sources are ours to clean up after. + managed = [self.build_opts.scratch_dir] + if self.build_opts.vendor_dir: + managed.append(self.build_opts.vendor_dir) + if self.src_dir.startswith(tuple(managed)) and os.path.isdir( self.build_dir ): if os.path.islink(self.build_dir): diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index a872a6f1d..d7086af61 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -60,6 +60,7 @@ def __init__( use_shipit: bool = False, vcvars_path: str | None = None, allow_system_packages: bool = False, + vendor_dir: str | None = None, lfs_path: str | None = None, shared_lib: bool = False, facebook_internal: bool | None = None, @@ -126,6 +127,10 @@ def __init__( self.host_type: HostType = host_type self.use_shipit: bool = use_shipit self.allow_system_packages: bool = allow_system_packages + # realpath so prefix checks against LocalDirFetcher paths line up + self.vendor_dir: str | None = ( + os.path.realpath(vendor_dir) if vendor_dir else None + ) self.lfs_path: str | None = lfs_path self.shared_lib: bool = shared_lib if shared_lib and self.is_windows(): @@ -732,6 +737,7 @@ def setup_build_options( "use_shipit", "vcvars_path", "allow_system_packages", + "vendor_dir", "lfs_path", "shared_lib", "free_up_disk", diff --git a/build/fbcode_builder/getdeps/cli.py b/build/fbcode_builder/getdeps/cli.py index 0265d2a96..7c89b9801 100644 --- a/build/fbcode_builder/getdeps/cli.py +++ b/build/fbcode_builder/getdeps/cli.py @@ -966,6 +966,15 @@ def add_common_arg(*args, **kwargs): action="store_true", default=False, ) + add_common_arg( + "--vendor-dir", + help=( + "Take third party sources from /, as populated " + "by the vendor command, and fail rather than download anything " + "that is missing there" + ), + default=None, + ) add_common_arg( "-v", "--verbose", diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index a43127bbd..99204ce5d 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -32,6 +32,7 @@ from .fetcher import ( ArchiveFetcher, GitFetcher, + LocalDirFetcher, PreinstalledNopFetcher, ShipitTransformerFetcher, SimpleShipitTransformerFetcher, @@ -612,6 +613,17 @@ def _create_fetcher( # pyre-fixme[7]: Expected `Fetcher` but got `SystemPackageFetcher`. return package_fetcher + if build_options.vendor_dir: + vendored = os.path.join(build_options.vendor_dir, self.name) + if not os.path.isdir(vendored): + raise Exception( + f"project {self.name} is not present in " + f"{build_options.vendor_dir}; populate it with " + "`getdeps.py vendor` using the same options" + ) + # pyre-fixme[7]: Expected `Fetcher` but got `LocalDirFetcher`. + return LocalDirFetcher(vendored) + if repo_url: rev = self.get("git", "rev") depth = self.get("git", "depth") From d710e968b9f4a50f2f861f53bd0d157b3b818992 Mon Sep 17 00:00:00 2001 From: Michel Lind Date: Tue, 15 Sep 2026 22:50:55 +0100 Subject: [PATCH 3/3] getdeps: add tests for vendor and --vendor-dir Cover the new behaviour with focused unit tests in the style of the existing builder tests (MagicMock loader and build options, real ManifestParser objects): - `vendor` copies each non-system dependency's fetched tree to /, skips dependencies that resolve to a SystemPackageFetcher, skips the project itself, drops .git, follows symlinks so the result is self-contained, and writes getdeps-vendor.txt with the fetcher hash of every vendored project; - `vendor` replaces a stale tree already present in the output dir; - with --vendor-dir set, a manifest with a download URL resolves to a LocalDirFetcher on /; - with --vendor-dir set and the project absent from it, fetcher creation fails naming the missing project instead of falling back to a network fetcher; - without --vendor-dir the normal ArchiveFetcher is still chosen. Run with `python3 -m unittest getdeps.test.vendor_test` from build/fbcode_builder: 5 tests, all passing. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Michel Lind --- .../getdeps/test/vendor_test.py | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 build/fbcode_builder/getdeps/test/vendor_test.py diff --git a/build/fbcode_builder/getdeps/test/vendor_test.py b/build/fbcode_builder/getdeps/test/vendor_test.py new file mode 100644 index 000000000..93a301b07 --- /dev/null +++ b/build/fbcode_builder/getdeps/test/vendor_test.py @@ -0,0 +1,178 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import argparse +import os +import shutil +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from ..cli import VendorCmd +from ..fetcher import ChangeStatus, LocalDirFetcher, PreinstalledNopFetcher +from ..manifest import ManifestContext, ManifestParser + + +def make_manifest(name: str, extra: str = "") -> ManifestParser: + return ManifestParser(name, f"[manifest]\nname = {name}\n{extra}") + + +def make_ctx() -> ManifestContext: + return ManifestContext( + { + "os": "linux", + "distro": None, + "distro_vers": None, + "fb": "off", + "fbsource": "off", + "test": "off", + } + ) + + +class FakeSourceFetcher: + """Stands in for a Git/Archive fetcher: owns a source tree on disk.""" + + def __init__(self, src_dir: str, hash_value: str) -> None: + self.src_dir = src_dir + self.hash_value = hash_value + self.updated = False + + def update(self) -> ChangeStatus: + self.updated = True + return ChangeStatus() + + def hash(self) -> str: + return self.hash_value + + def get_src_dir(self) -> str: + return self.src_dir + + +class VendorCmdTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.output_dir = os.path.join(self.tmp, "vendor") + + def make_src_tree(self, name: str) -> str: + src = os.path.join(self.tmp, "src", name) + os.makedirs(os.path.join(src, ".git")) + os.makedirs(os.path.join(src, "sub")) + with open(os.path.join(src, ".git", "HEAD"), "w") as f: + f.write("ref: refs/heads/main\n") + with open(os.path.join(src, "sub", "code.cpp"), "w") as f: + f.write("int x;\n") + # a symlink into a directory that won't exist on an offline builder + os.symlink(os.path.join(src, "sub", "code.cpp"), os.path.join(src, "link.cpp")) + return src + + def run_vendor(self, manifests, fetchers) -> None: + loader = MagicMock() + loader.manifests_in_dependency_order.return_value = manifests + loader.create_fetcher.side_effect = lambda m: fetchers[m.name] + args = argparse.Namespace(output_dir=self.output_dir) + VendorCmd().run_project_cmd(args, loader, manifests[-1]) + + def test_vendors_source_deps_and_skips_system_and_top_level(self) -> None: + dep_src = FakeSourceFetcher(self.make_src_tree("depa"), "a" * 40) + top_src = FakeSourceFetcher(self.make_src_tree("top"), "t" * 40) + manifests = [ + make_manifest("depa"), + make_manifest("sysdep"), + make_manifest("top"), + ] + fetchers = { + "depa": dep_src, + "sysdep": PreinstalledNopFetcher(), + "top": top_src, + } + + self.run_vendor(manifests, fetchers) + + self.assertTrue(dep_src.updated, "source dep must be fetched before copying") + self.assertFalse(top_src.updated, "the project itself is not vendored") + self.assertEqual( + sorted(os.listdir(self.output_dir)), ["depa", "getdeps-vendor.txt"] + ) + + vendored = os.path.join(self.output_dir, "depa") + self.assertTrue(os.path.isfile(os.path.join(vendored, "sub", "code.cpp"))) + self.assertFalse(os.path.exists(os.path.join(vendored, ".git"))) + # symlinks are materialised so the tree is self-contained + self.assertTrue(os.path.isfile(os.path.join(vendored, "link.cpp"))) + self.assertFalse(os.path.islink(os.path.join(vendored, "link.cpp"))) + + with open(os.path.join(self.output_dir, "getdeps-vendor.txt")) as f: + self.assertEqual(f.read(), "depa %s\n" % ("a" * 40)) + + def test_replaces_stale_vendored_tree(self) -> None: + stale = os.path.join(self.output_dir, "depa", "stale.txt") + os.makedirs(os.path.dirname(stale)) + with open(stale, "w") as f: + f.write("old\n") + dep_src = FakeSourceFetcher(self.make_src_tree("depa"), "b" * 40) + manifests = [make_manifest("depa"), make_manifest("top")] + fetchers = {"depa": dep_src, "top": FakeSourceFetcher(self.tmp, "t" * 40)} + + self.run_vendor(manifests, fetchers) + + self.assertFalse(os.path.exists(stale)) + self.assertTrue( + os.path.isfile(os.path.join(self.output_dir, "depa", "sub", "code.cpp")) + ) + + +class VendorDirFetcherTest(unittest.TestCase): + """--vendor-dir routes third-party deps through LocalDirFetcher, or fails.""" + + DOWNLOAD_MANIFEST = """ +[download] +url = https://example.com/dep-1.0.tar.gz +sha256 = 0000000000000000000000000000000000000000000000000000000000000000 +""" + + def setUp(self) -> None: + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.build_opts = MagicMock() + self.build_opts.use_shipit = False + self.build_opts.fbsource_dir = None + self.build_opts.allow_system_packages = False + self.build_opts.vendor_dir = os.path.join(self.tmp, "vendor") + patcher = patch( + "getdeps.manifest.ShipitTransformerFetcher.available", return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + + def test_vendored_project_uses_local_dir_fetcher(self) -> None: + vendored = os.path.join(self.build_opts.vendor_dir, "dep") + os.makedirs(vendored) + manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST) + + fetcher = manifest._create_fetcher(self.build_opts, make_ctx()) + + self.assertIsInstance(fetcher, LocalDirFetcher) + self.assertEqual(fetcher.get_src_dir(), os.path.realpath(vendored)) + + def test_missing_vendored_project_fails_instead_of_downloading(self) -> None: + os.makedirs(self.build_opts.vendor_dir) + manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST) + + with self.assertRaisesRegex( + Exception, "project dep is not present in .*vendor" + ): + manifest._create_fetcher(self.build_opts, make_ctx()) + + def test_no_vendor_dir_keeps_normal_fetcher(self) -> None: + self.build_opts.vendor_dir = None + manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST) + + fetcher = manifest._create_fetcher(self.build_opts, make_ctx()) + + self.assertNotIsInstance(fetcher, LocalDirFetcher) + self.assertEqual(fetcher.hash(), "0" * 64)