getdeps: vendor subcommand and --vendor-dir for offline builds

build/fbcode_builder/README.md

@@ -17,6 +17,18 @@ repositories that use `fbcode_builder`.  Typically this directory is copied
 into the open source repositories as `build/fbcode_builder/`.
 
 
+## Vendoring dependencies for offline builds
+
+Distributions typically require builds to run without network access and to
+ship third-party sources alongside the project.  `getdeps.py vendor
+--output-dir DIR project` copies the source tree of every dependency that is
+not satisfied by system packages to `DIR/<project>` and records what it
+vendored in `DIR/getdeps-vendor.txt`.  A later `getdeps.py --vendor-dir DIR
+build project` then takes those trees instead of fetching, and fails if a
+dependency is missing from `DIR` rather than downloading it.  Pass the same
+`--allow-system-packages` and `--no-tests` options to both commands so they
+agree on the dependency set.
+
 # Project Configuration Files
 
 The `manifests` subdirectory contains configuration files for many different

build/fbcode_builder/getdeps/builder.py

@@ -226,8 +226,12 @@ def build(self, reconfigure: bool) -> None:
         self._build(reconfigure=reconfigure)
 
         if self.build_opts.free_up_disk:
-            # don't clean --src-dir=. case as user may want to build again or run tests on the build
-            if self.src_dir.startswith(self.build_opts.scratch_dir) and os.path.isdir(
+            # don't clean --src-dir=. case as user may want to build again or
+            # run tests on the build; vendored sources are ours to clean up after.
+            managed = [self.build_opts.scratch_dir]
+            if self.build_opts.vendor_dir:
+                managed.append(self.build_opts.vendor_dir)
+            if self.src_dir.startswith(tuple(managed)) and os.path.isdir(
                 self.build_dir
             ):
                 if os.path.islink(self.build_dir):

build/fbcode_builder/getdeps/buildopts.py

@@ -60,6 +60,7 @@ def __init__(
         use_shipit: bool = False,
         vcvars_path: str | None = None,
         allow_system_packages: bool = False,
+        vendor_dir: str | None = None,
         lfs_path: str | None = None,
         shared_lib: bool = False,
         facebook_internal: bool | None = None,
@@ -126,6 +127,10 @@ def __init__(
         self.host_type: HostType = host_type
         self.use_shipit: bool = use_shipit
         self.allow_system_packages: bool = allow_system_packages
+        # realpath so prefix checks against LocalDirFetcher paths line up
+        self.vendor_dir: str | None = (
+            os.path.realpath(vendor_dir) if vendor_dir else None
+        )
         self.lfs_path: str | None = lfs_path
         self.shared_lib: bool = shared_lib
         if shared_lib and self.is_windows():
@@ -732,6 +737,7 @@ def setup_build_options(
             "use_shipit",
             "vcvars_path",
             "allow_system_packages",
+            "vendor_dir",
             "lfs_path",
             "shared_lib",
             "free_up_disk",

build/fbcode_builder/getdeps/cli.py

@@ -192,6 +192,54 @@ def run_project_cmd(self, args, loader, manifest):
             fetcher.update()
 
 
+@cmd("vendor", "copy the sources of a project's dependencies into a directory")
+class VendorCmd(ProjectCmdBase):
+    """Populate a directory with one source tree per third-party dependency,
+    analogous to `cargo vendor`, so that a later build can run without
+    network access (see --vendor-dir)."""
+
+    def setup_project_cmd_parser(self, parser):
+        parser.add_argument(
+            "--output-dir",
+            required=True,
+            help=(
+                "Directory to populate; each dependency is copied "
+                "to <output-dir>/<project>"
+            ),
+        )
+        parser.add_argument(
+            "--host-type",
+            help="Vendor deps for this host type rather than the current system",
+        )
+
+    def run_project_cmd(self, args, loader, manifest):
+        os.makedirs(args.output_dir, exist_ok=True)
+        vendored = []
+        for m in loader.manifests_in_dependency_order():
+            if m == manifest:
+                continue
+            fetcher = loader.create_fetcher(m)
+            if isinstance(fetcher, SystemPackageFetcher):
+                # Satisfied by system packages; nothing to vendor
+                continue
+            fetcher.update()
+            dest = os.path.join(args.output_dir, m.name)
+            if os.path.exists(dest):
+                shutil.rmtree(dest)
+            print("Vendoring %s -> %s" % (m.name, dest))
+            # Follow symlinks so the result is self-contained: subproject
+            # fetchers link into the scratch dir, which won't exist offline.
+            shutil.copytree(
+                fetcher.get_src_dir(),
+                dest,
+                ignore=shutil.ignore_patterns(".git"),
+                ignore_dangling_symlinks=True,
+            )
+            vendored.append("%s %s\n" % (m.name, fetcher.hash()))
+        with open(os.path.join(args.output_dir, "getdeps-vendor.txt"), "w") as f:
+            f.writelines(vendored)
+
+
 @cmd("install-system-deps", "Install system packages to satisfy the deps for a project")
 class InstallSysDepsCmd(ProjectCmdBase):
     def setup_project_cmd_parser(self, parser):
@@ -918,6 +966,15 @@ def add_common_arg(*args, **kwargs):
         action="store_true",
         default=False,
     )
+    add_common_arg(
+        "--vendor-dir",
+        help=(
+            "Take third party sources from <vendor-dir>/<project>, as populated "
+            "by the vendor command, and fail rather than download anything "
+            "that is missing there"
+        ),
+        default=None,
+    )
     add_common_arg(
         "-v",
         "--verbose",

build/fbcode_builder/getdeps/manifest.py

@@ -32,6 +32,7 @@
 from .fetcher import (
     ArchiveFetcher,
     GitFetcher,
+    LocalDirFetcher,
     PreinstalledNopFetcher,
     ShipitTransformerFetcher,
     SimpleShipitTransformerFetcher,
@@ -612,6 +613,17 @@ def _create_fetcher(
                     # pyre-fixme[7]: Expected `Fetcher` but got `SystemPackageFetcher`.
                     return package_fetcher
 
+        if build_options.vendor_dir:
+            vendored = os.path.join(build_options.vendor_dir, self.name)
+            if not os.path.isdir(vendored):
+                raise Exception(
+                    f"project {self.name} is not present in "
+                    f"{build_options.vendor_dir}; populate it with "
+                    "`getdeps.py vendor` using the same options"
+                )
+            # pyre-fixme[7]: Expected `Fetcher` but got `LocalDirFetcher`.
+            return LocalDirFetcher(vendored)
+
         if repo_url:
             rev = self.get("git", "rev")
             depth = self.get("git", "depth")

build/fbcode_builder/getdeps/test/vendor_test.py

@@ -0,0 +1,178 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+import argparse
+import os
+import shutil
+import tempfile
+import unittest
+from unittest.mock import MagicMock, patch
+
+from ..cli import VendorCmd
+from ..fetcher import ChangeStatus, LocalDirFetcher, PreinstalledNopFetcher
+from ..manifest import ManifestContext, ManifestParser
+
+
+def make_manifest(name: str, extra: str = "") -> ManifestParser:
+    return ManifestParser(name, f"[manifest]\nname = {name}\n{extra}")
+
+
+def make_ctx() -> ManifestContext:
+    return ManifestContext(
+        {
+            "os": "linux",
+            "distro": None,
+            "distro_vers": None,
+            "fb": "off",
+            "fbsource": "off",
+            "test": "off",
+        }
+    )
+
+
+class FakeSourceFetcher:
+    """Stands in for a Git/Archive fetcher: owns a source tree on disk."""
+
+    def __init__(self, src_dir: str, hash_value: str) -> None:
+        self.src_dir = src_dir
+        self.hash_value = hash_value
+        self.updated = False
+
+    def update(self) -> ChangeStatus:
+        self.updated = True
+        return ChangeStatus()
+
+    def hash(self) -> str:
+        return self.hash_value
+
+    def get_src_dir(self) -> str:
+        return self.src_dir
+
+
+class VendorCmdTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+        self.output_dir = os.path.join(self.tmp, "vendor")
+
+    def make_src_tree(self, name: str) -> str:
+        src = os.path.join(self.tmp, "src", name)
+        os.makedirs(os.path.join(src, ".git"))
+        os.makedirs(os.path.join(src, "sub"))
+        with open(os.path.join(src, ".git", "HEAD"), "w") as f:
+            f.write("ref: refs/heads/main\n")
+        with open(os.path.join(src, "sub", "code.cpp"), "w") as f:
+            f.write("int x;\n")
+        # a symlink into a directory that won't exist on an offline builder
+        os.symlink(os.path.join(src, "sub", "code.cpp"), os.path.join(src, "link.cpp"))
+        return src
+
+    def run_vendor(self, manifests, fetchers) -> None:
+        loader = MagicMock()
+        loader.manifests_in_dependency_order.return_value = manifests
+        loader.create_fetcher.side_effect = lambda m: fetchers[m.name]
+        args = argparse.Namespace(output_dir=self.output_dir)
+        VendorCmd().run_project_cmd(args, loader, manifests[-1])
+
+    def test_vendors_source_deps_and_skips_system_and_top_level(self) -> None:
+        dep_src = FakeSourceFetcher(self.make_src_tree("depa"), "a" * 40)
+        top_src = FakeSourceFetcher(self.make_src_tree("top"), "t" * 40)
+        manifests = [
+            make_manifest("depa"),
+            make_manifest("sysdep"),
+            make_manifest("top"),
+        ]
+        fetchers = {
+            "depa": dep_src,
+            "sysdep": PreinstalledNopFetcher(),
+            "top": top_src,
+        }
+
+        self.run_vendor(manifests, fetchers)
+
+        self.assertTrue(dep_src.updated, "source dep must be fetched before copying")
+        self.assertFalse(top_src.updated, "the project itself is not vendored")
+        self.assertEqual(
+            sorted(os.listdir(self.output_dir)), ["depa", "getdeps-vendor.txt"]
+        )
+
+        vendored = os.path.join(self.output_dir, "depa")
+        self.assertTrue(os.path.isfile(os.path.join(vendored, "sub", "code.cpp")))
+        self.assertFalse(os.path.exists(os.path.join(vendored, ".git")))
+        # symlinks are materialised so the tree is self-contained
+        self.assertTrue(os.path.isfile(os.path.join(vendored, "link.cpp")))
+        self.assertFalse(os.path.islink(os.path.join(vendored, "link.cpp")))
+
+        with open(os.path.join(self.output_dir, "getdeps-vendor.txt")) as f:
+            self.assertEqual(f.read(), "depa %s\n" % ("a" * 40))
+
+    def test_replaces_stale_vendored_tree(self) -> None:
+        stale = os.path.join(self.output_dir, "depa", "stale.txt")
+        os.makedirs(os.path.dirname(stale))
+        with open(stale, "w") as f:
+            f.write("old\n")
+        dep_src = FakeSourceFetcher(self.make_src_tree("depa"), "b" * 40)
+        manifests = [make_manifest("depa"), make_manifest("top")]
+        fetchers = {"depa": dep_src, "top": FakeSourceFetcher(self.tmp, "t" * 40)}
+
+        self.run_vendor(manifests, fetchers)
+
+        self.assertFalse(os.path.exists(stale))
+        self.assertTrue(
+            os.path.isfile(os.path.join(self.output_dir, "depa", "sub", "code.cpp"))
+        )
+
+
+class VendorDirFetcherTest(unittest.TestCase):
+    """--vendor-dir routes third-party deps through LocalDirFetcher, or fails."""
+
+    DOWNLOAD_MANIFEST = """
+[download]
+url = https://example.com/dep-1.0.tar.gz
+sha256 = 0000000000000000000000000000000000000000000000000000000000000000
+"""
+
+    def setUp(self) -> None:
+        self.tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+        self.build_opts = MagicMock()
+        self.build_opts.use_shipit = False
+        self.build_opts.fbsource_dir = None
+        self.build_opts.allow_system_packages = False
+        self.build_opts.vendor_dir = os.path.join(self.tmp, "vendor")
+        patcher = patch(
+            "getdeps.manifest.ShipitTransformerFetcher.available", return_value=False
+        )
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_vendored_project_uses_local_dir_fetcher(self) -> None:
+        vendored = os.path.join(self.build_opts.vendor_dir, "dep")
+        os.makedirs(vendored)
+        manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST)
+
+        fetcher = manifest._create_fetcher(self.build_opts, make_ctx())
+
+        self.assertIsInstance(fetcher, LocalDirFetcher)
+        self.assertEqual(fetcher.get_src_dir(), os.path.realpath(vendored))
+
+    def test_missing_vendored_project_fails_instead_of_downloading(self) -> None:
+        os.makedirs(self.build_opts.vendor_dir)
+        manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST)
+
+        with self.assertRaisesRegex(
+            Exception, "project dep is not present in .*vendor"
+        ):
+            manifest._create_fetcher(self.build_opts, make_ctx())
+
+    def test_no_vendor_dir_keeps_normal_fetcher(self) -> None:
+        self.build_opts.vendor_dir = None
+        manifest = make_manifest("dep", self.DOWNLOAD_MANIFEST)
+
+        fetcher = manifest._create_fetcher(self.build_opts, make_ctx())
+
+        self.assertNotIsInstance(fetcher, LocalDirFetcher)
+        self.assertEqual(fetcher.hash(), "0" * 64)