diff --git a/build/fbcode_builder/README.md b/build/fbcode_builder/README.md index 62c2f5b56..30c8c1449 100644 --- a/build/fbcode_builder/README.md +++ b/build/fbcode_builder/README.md @@ -17,6 +17,18 @@ repositories that use `fbcode_builder`. Typically this directory is copied into the open source repositories as `build/fbcode_builder/`. +## Vendoring dependencies for offline builds + +Distributions typically require builds to run without network access and to +ship third-party sources alongside the project. `getdeps.py vendor +--output-dir DIR project` copies the source tree of every dependency that is +not satisfied by system packages to `DIR/` and records what it +vendored in `DIR/getdeps-vendor.txt`. A later `getdeps.py --vendor-dir DIR +build project` then takes those trees instead of fetching, and fails if a +dependency is missing from `DIR` rather than downloading it. Pass the same +`--allow-system-packages` and `--no-tests` options to both commands so they +agree on the dependency set. + # Project Configuration Files The `manifests` subdirectory contains configuration files for many different diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index c91de65d2..c49204904 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -226,8 +226,12 @@ def build(self, reconfigure: bool) -> None: self._build(reconfigure=reconfigure) if self.build_opts.free_up_disk: - # don't clean --src-dir=. case as user may want to build again or run tests on the build - if self.src_dir.startswith(self.build_opts.scratch_dir) and os.path.isdir( + # don't clean --src-dir=. case as user may want to build again or + # run tests on the build; vendored sources are ours to clean up after. + managed = [self.build_opts.scratch_dir] + if self.build_opts.vendor_dir: + managed.append(self.build_opts.vendor_dir) + if self.src_dir.startswith(tuple(managed)) and os.path.isdir( self.build_dir ): if os.path.islink(self.build_dir): diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index a872a6f1d..d7086af61 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -60,6 +60,7 @@ def __init__( use_shipit: bool = False, vcvars_path: str | None = None, allow_system_packages: bool = False, + vendor_dir: str | None = None, lfs_path: str | None = None, shared_lib: bool = False, facebook_internal: bool | None = None, @@ -126,6 +127,10 @@ def __init__( self.host_type: HostType = host_type self.use_shipit: bool = use_shipit self.allow_system_packages: bool = allow_system_packages + # realpath so prefix checks against LocalDirFetcher paths line up + self.vendor_dir: str | None = ( + os.path.realpath(vendor_dir) if vendor_dir else None + ) self.lfs_path: str | None = lfs_path self.shared_lib: bool = shared_lib if shared_lib and self.is_windows(): @@ -732,6 +737,7 @@ def setup_build_options( "use_shipit", "vcvars_path", "allow_system_packages", + "vendor_dir", "lfs_path", "shared_lib", "free_up_disk", diff --git a/build/fbcode_builder/getdeps/cli.py b/build/fbcode_builder/getdeps/cli.py index 818ee8f94..7c89b9801 100644 --- a/build/fbcode_builder/getdeps/cli.py +++ b/build/fbcode_builder/getdeps/cli.py @@ -192,6 +192,54 @@ def run_project_cmd(self, args, loader, manifest): fetcher.update() +@cmd("vendor", "copy the sources of a project's dependencies into a directory") +class VendorCmd(ProjectCmdBase): + """Populate a directory with one source tree per third-party dependency, + analogous to `cargo vendor`, so that a later build can run without + network access (see --vendor-dir).""" + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--output-dir", + required=True, + help=( + "Directory to populate; each dependency is copied " + "to /" + ), + ) + parser.add_argument( + "--host-type", + help="Vendor deps for this host type rather than the current system", + ) + + def run_project_cmd(self, args, loader, manifest): + os.makedirs(args.output_dir, exist_ok=True) + vendored = [] + for m in loader.manifests_in_dependency_order(): + if m == manifest: + continue + fetcher = loader.create_fetcher(m) + if isinstance(fetcher, SystemPackageFetcher): + # Satisfied by system packages; nothing to vendor + continue + fetcher.update() + dest = os.path.join(args.output_dir, m.name) + if os.path.exists(dest): + shutil.rmtree(dest) + print("Vendoring %s -> %s" % (m.name, dest)) + # Follow symlinks so the result is self-contained: subproject + # fetchers link into the scratch dir, which won't exist offline. + shutil.copytree( + fetcher.get_src_dir(), + dest, + ignore=shutil.ignore_patterns(".git"), + ignore_dangling_symlinks=True, + ) + vendored.append("%s %s\n" % (m.name, fetcher.hash())) + with open(os.path.join(args.output_dir, "getdeps-vendor.txt"), "w") as f: + f.writelines(vendored) + + @cmd("install-system-deps", "Install system packages to satisfy the deps for a project") class InstallSysDepsCmd(ProjectCmdBase): def setup_project_cmd_parser(self, parser): @@ -918,6 +966,15 @@ def add_common_arg(*args, **kwargs): action="store_true", default=False, ) + add_common_arg( + "--vendor-dir", + help=( + "Take third party sources from /, as populated " + "by the vendor command, and fail rather than download anything " + "that is missing there" + ), + default=None, + ) add_common_arg( "-v", "--verbose", diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index a43127bbd..99204ce5d 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -32,6 +32,7 @@ from .fetcher import ( ArchiveFetcher, GitFetcher, + LocalDirFetcher, PreinstalledNopFetcher, ShipitTransformerFetcher, SimpleShipitTransformerFetcher, @@ -612,6 +613,17 @@ def _create_fetcher( # pyre-fixme[7]: Expected `Fetcher` but got `SystemPackageFetcher`. return package_fetcher + if build_options.vendor_dir: + vendored = os.path.join(build_options.vendor_dir, self.name) + if not os.path.isdir(vendored): + raise Exception( + f"project {self.name} is not present in " + f"{build_options.vendor_dir}; populate it with " + "`getdeps.py vendor` using the same options" + ) + # pyre-fixme[7]: Expected `Fetcher` but got `LocalDirFetcher`. + return LocalDirFetcher(vendored) + if repo_url: rev = self.get("git", "rev") depth = self.get("git", "depth") diff --git a/build/fbcode_builder/getdeps/test/vendor_test.py b/build/fbcode_builder/getdeps/test/vendor_test.py new file mode 100644 index 000000000..93a301b07 --- /dev/null +++ b/build/fbcode_builder/getdeps/test/vendor_test.py @@ -0,0 +1,178 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import argparse +import os +import shutil +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from ..cli import VendorCmd +from ..fetcher import ChangeStatus, LocalDirFetcher, PreinstalledNopFetcher +from ..manifest import ManifestContext, ManifestParser + + +def make_manifest(name: str, extra: str = "") -> ManifestParser: + return ManifestParser(name, f"[manifest]\nname = {name}\n{extra}") + + +def make_ctx() -> ManifestContext: + return ManifestContext( + { + "os": "linux", + "distro": None, + "distro_vers": None, + "fb": "off", + "fbsource": "off", + "test": "off", + } + ) + + +class FakeSourceFetcher: + """Stands in for a Git/Archive fetcher: owns a source tree on disk.""" + + def __init__(self, src_dir: str, hash_value: str) -> None: + self.src_dir = src_dir + self.hash_value = hash_value + self.updated = False + + def update(self) -> ChangeStatus: + self.updated = True + return ChangeStatus() + + def hash(self) -> str: + return self.hash_value + + def get_src_dir(self) -> str: + return self.src_dir + + +class VendorCmdTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.output_dir = os.path.join(self.tmp, "vendor") + + def make_src_tree(self, name: str) -> str: + src = os.path.join(self.tmp, "src", name) + os.makedirs(os.path.join(src, ".git")) + os.makedirs(os.path.join(src, "sub")) + with open(os.path.join(src, ".git", "HEAD"), "w") as f: + f.write("ref: refs/heads/main\n") + with open(os.path.join(src, "sub", "code.cpp"), "w") as f: + f.write("int x;\n") + # a symlink into a directory that won't exist on an offline builder + os.symlink(os.path.join(src, "sub", "code.cpp"), os.path.join(src, "link.cpp")) + return src + + def run_vendor(self, manifests, fetchers) -> None: + loader = MagicMock() + loader.manifests_in_dependency_order.return_value = manifests + loader.create_fetcher.side_effect = lambda m: fetchers[m.name] + args = argparse.Namespace(output_dir=self.output_dir) + VendorCmd().run_project_cmd(args, loader, manifests[-1]) + + def test_vendors_source_deps_and_skips_system_and_top_level(self) -> None: + dep_src = FakeSourceFetcher(self.make_src_tree("depa"), "a" * 40) + top_src = FakeSourceFetcher(self.make_src_tree("top"), "t" * 40) + manifests = [ + make_manifest("depa"), + make_manifest("sysdep"), + make_manifest("top"), + ] + fetchers = { + "depa": dep_src, + "sysdep": PreinstalledNopFetcher(), + "top": top_src, + } + + self.run_vendor(manifests, fetchers) + + self.assertTrue(dep_src.updated, "source dep must be fetched before copying") + self.assertFalse(top_src.updated, "the project itself is not vendored") + self.assertEqual( + sorted(os.listdir(self.output_dir)), ["depa", "getdeps-vendor.txt"] + ) + + vendored = os.path.join(self.output_dir, "depa") + self.assertTrue(os.path.isfile(os.path.join(vendored, "sub", "code.cpp"))) + self.assertFalse(os.path.exists(os.path.join(vendored, ".git"))) + # symlinks are materialised so the tree is self-contained + self.assertTrue(os.path.isfile(os.path.join(vendored, "link.cpp"))) + self.assertFalse(os.path.islink(os.path.join(vendored, "link.cpp"))) + + with open(os.path.join(self.output_dir, "getdeps-vendor.txt")) as f: + self.assertEqual(f.read(), "depa %s\n" % ("a" * 40)) + + def test_replaces_stale_vendored_tree(self) -> None: + stale = os.path.join(self.output_dir, "depa", "stale.txt") + os.makedirs(os.path.dirname(stale)) + with open(stale, "w") as f: + f.write("old\n") + dep_src = FakeSourceFetcher(self.make_src_tree("depa"), "b" * 40) + manifests = [make_manifest("depa"), make_manifest("top")] + fetchers = {"depa": dep_src, "top": FakeSourceFetcher(self.tmp, "t" * 40)} + + self.run_vendor(manifests, fetchers) + + self.assertFalse(os.path.exists(stale)) + self.assertTrue( + os.path.isfile(os.path.join(self.output_dir, "depa", "sub", "code.cpp")) + ) + + +class VendorDirFetcherTest(unittest.TestCase): + """--vendor-dir routes third-party deps through LocalDirFetcher, or fails.""" + + DOWNLOAD_MANIFEST = """ +[download] +url = https://example.com/dep-1.0.tar.gz +sha256 = 0000000000000000000000000000000000000000000000000000000000000000 +""" + + def setUp(self) -> None: + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.build_opts = MagicMock() + self.build_opts.use_shipit = False + self.build_opts.fbsource_dir = None + self.build_opts.allow_system_packages = False + self.build_opts.vendor_dir = os.path.join(self.tmp, "vendor") + patcher = patch( + "getdeps.manifest.ShipitTransformerFetcher.available", return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + + def test_vendored_project_uses_local_dir_fetcher(self) -> None: + vendored = os.path.join(self.build_opts.vendor_dir, "dep") + os.makedirs(vendored) + manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST) + + fetcher = manifest._create_fetcher(self.build_opts, make_ctx()) + + self.assertIsInstance(fetcher, LocalDirFetcher) + self.assertEqual(fetcher.get_src_dir(), os.path.realpath(vendored)) + + def test_missing_vendored_project_fails_instead_of_downloading(self) -> None: + os.makedirs(self.build_opts.vendor_dir) + manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST) + + with self.assertRaisesRegex( + Exception, "project dep is not present in .*vendor" + ): + manifest._create_fetcher(self.build_opts, make_ctx()) + + def test_no_vendor_dir_keeps_normal_fetcher(self) -> None: + self.build_opts.vendor_dir = None + manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST) + + fetcher = manifest._create_fetcher(self.build_opts, make_ctx()) + + self.assertNotIsInstance(fetcher, LocalDirFetcher) + self.assertEqual(fetcher.hash(), "0" * 64)