From 6d79a07d93d7f9bfdb42977fe938e269149b3c0b Mon Sep 17 00:00:00 2001 From: Michel Lind Date: Thu, 17 Sep 2026 10:07:40 +0100 Subject: [PATCH] getdeps: add distro_family and recognise Red Hat Enterprise Linux get_linux_type() derives the distro name from NAME in /etc/os-release, which on Red Hat Enterprise Linux yields "red_hat_enterprise". Nothing knows that name: HostType.get_package_manager() returns None for it, so --allow-system-packages silently satisfies nothing and every dependency is built from source, and the manifests with distro=centos_stream selectors do not apply either. RHEL's VERSION_ID also carries a minor ("9.8"), as do AlmaLinux's and Rocky's, so even a matching distro would miss the distro_vers=9 selectors written for CentOS Stream. This matters for EPEL: Fedora's Koji builds EPEL 9 packages in a buildroot with redhat-release (real RHEL), while EPEL 10 buildroots and the local mock configs use centos-stream-release. A getdeps-based EPEL 9 build therefore behaves differently in Koji than in local mock. Nobody noticed because getdeps has not been used for distro packaging before and RHEL is rare among people building from source. Rather than pretend RHEL is CentOS Stream, model what the manifests actually mean. Following Chef's platform_family, introduce a distro_family alongside distro: "rhel" for rhel, centos, centos_stream, alma and rocky; "fedora"; "debian" for debian, ubuntu, pop!_os and mint; "arch". Fedora is deliberately its own family, unlike Ansible's os_family which folds it into RedHat: its base repositories carry many dependencies that the EL family only has in EPEL or not at all (glog, gflags, fmt, fast_float, googletest, benchmark, liboqs, jq), zlib is zlib-ng-compat there, and CentOS Stream 9 needs gcc-toolset; nineteen manifest sections already select on exactly that difference. Concretely: - RHEL reports distro "rhel", the same short name as its os-release ID; - within the rhel family distro_vers is the major version only, so RHEL 9.8, Alma 9.6 and Rocky 9.6 all match distro_vers=9 as CentOS Stream 9 does; Fedora and Ubuntu keep their full versions; - distro_family is available to manifest selectors and drives get_package_manager(), which now returns rpm for the whole rhel family; - `install-system-deps --distro` accepts rhel; - os-release parsing moves into parse_os_release() so it can be unit tested. Existing selectors keep their meaning: distro=centos_stream still matches only CentOS Stream. A follow-up will switch the sections that mean "any EL 9" to distro_family=rhel so that RHEL, Alma and Rocky pick up gcc-toolset, libaio and numactl too. Until then a RHEL host already gains every plain [rpms] mapping and the not(distro=fedora) ones. Tested: `install-system-deps --recursive --dry-run --os-type linux --distro rhel --distro-version 9 cachelib` now produces a dnf command (previously "I don't know how to install any packages on this system"), and unit tests cover the os-release parsing for Fedora, CentOS Stream, RHEL, Alma, Rocky and Ubuntu plus the family and package-manager mapping. Verified against synthetic os-release contents, not a live RHEL host. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Michel Lind --- build/fbcode_builder/getdeps/buildopts.py | 1 + build/fbcode_builder/getdeps/cli.py | 2 +- .../getdeps/getdeps_platform.py | 48 +++++++++++++++++-- build/fbcode_builder/getdeps/manifest.py | 1 + .../getdeps/test/builder_test.py | 2 +- .../getdeps/test/features_test.py | 2 + .../getdeps/test/platform_test.py | 41 +++++++++++++++- .../getdeps/test/vendor_test.py | 1 + 8 files changed, 90 insertions(+), 8 deletions(-) diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index d7086af61e..a0342f5db0 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -266,6 +266,7 @@ def get_context_generator( "os": host_type.ostype, "distro": host_type.distro, "distro_vers": host_type.distrovers, + "distro_family": host_type.distro_family, "fb": "on" if self.facebook_internal else "off", "fbsource": "on" if self.fbsource_dir else "off", "test": "off", diff --git a/build/fbcode_builder/getdeps/cli.py b/build/fbcode_builder/getdeps/cli.py index b719cc1e49..d85d84ff12 100644 --- a/build/fbcode_builder/getdeps/cli.py +++ b/build/fbcode_builder/getdeps/cli.py @@ -314,7 +314,7 @@ def setup_project_cmd_parser(self, parser): parser.add_argument( "--distro", help="Filter to just this distro to run", - choices=["ubuntu", "centos_stream", "fedora"], + choices=["ubuntu", "centos_stream", "fedora", "rhel"], action="store", dest="distro", default=None, diff --git a/build/fbcode_builder/getdeps/getdeps_platform.py b/build/fbcode_builder/getdeps/getdeps_platform.py index b4ba523416..6f009043ac 100644 --- a/build/fbcode_builder/getdeps/getdeps_platform.py +++ b/build/fbcode_builder/getdeps/getdeps_platform.py @@ -18,13 +18,37 @@ def is_windows() -> bool: return sys.platform.startswith("win") +# Distribution families, after Chef's platform_family: distros that share a +# package namespace and release cadence for the purposes of manifests. +# Fedora is deliberately its own family (unlike Ansible's os_family, which +# folds it into RedHat): its base repos carry far more of our dependencies +# than the EL family's do, and manifests already select on that difference. +DISTRO_FAMILIES: dict[str, tuple[str, ...]] = { + "rhel": ("rhel", "centos", "centos_stream", "alma", "rocky"), + "fedora": ("fedora",), + "debian": ("debian", "ubuntu", "pop!_os", "mint"), + "arch": ("arch",), +} + + +def distro_family(distro: str | None) -> str | None: + for family, members in DISTRO_FAMILIES.items(): + if distro in members: + return family + return None + + def get_linux_type() -> tuple[str | None, str | None, str | None]: try: with open("/etc/os-release") as f: data = f.read() except EnvironmentError: return (None, None, None) + return parse_os_release(data) + +def parse_os_release(data: str) -> tuple[str, str | None, str | None]: + """Derive (ostype, distro, distrovers) from the contents of /etc/os-release.""" os_vars: dict[str, str] = {} for line in data.splitlines(): parts = line.split("=", 1) @@ -48,6 +72,17 @@ def get_linux_type() -> tuple[str | None, str | None, str | None]: if version_id: version_id = version_id.lower() + # Red Hat Enterprise Linux's NAME would otherwise become + # "red_hat_enterprise"; use the same short name as its os-release ID. + if name == "red_hat_enterprise": + name = "rhel" + + # The EL family is versioned by major release for packaging purposes, + # and manifests select on distro_vers=9; RHEL, Alma and Rocky report + # a minor too ("9.8") which would never match. + if version_id and distro_family(name) == "rhel": + version_id = version_id.split(".")[0] + return "linux", name, version_id @@ -260,6 +295,10 @@ def is_linux(self) -> bool: def is_freebsd(self) -> bool: return self.ostype == "freebsd" + @property + def distro_family(self) -> str | None: + return distro_family(self.distro) + def as_tuple_string(self) -> str: return "%s-%s-%s" % ( self.ostype, @@ -272,13 +311,12 @@ def get_package_manager(self) -> str | None: return None if self.is_darwin(): return "homebrew" - if self.distro in ("fedora", "centos", "centos_stream", "rocky", "alma"): + family = self.distro_family + if family in ("fedora", "rhel"): return "rpm" - if self.distro is not None and self.distro.startswith( - ("debian", "ubuntu", "pop!_os", "mint") - ): + if family == "debian": return "deb" - if self.distro == "arch": + if family == "arch": return "pacman-package" return None diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 43ece9fa29..23859efead 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -963,6 +963,7 @@ class ManifestContext: "os", "distro", "distro_vers", + "distro_family", "fb", "fbsource", "test", diff --git a/build/fbcode_builder/getdeps/test/builder_test.py b/build/fbcode_builder/getdeps/test/builder_test.py index 7b5fba8cb5..f74e7200a0 100644 --- a/build/fbcode_builder/getdeps/test/builder_test.py +++ b/build/fbcode_builder/getdeps/test/builder_test.py @@ -13,7 +13,6 @@ from ..envfuncs import Env from ..manifest import ManifestContext, ManifestParser - MINIMAL_MANIFEST = """ [manifest] name = test @@ -41,6 +40,7 @@ def make_cmake_builder() -> CMakeBuilder: "os": None, "distro": None, "distro_vers": None, + "distro_family": None, "fb": "off", "fbsource": "off", "test": "off", diff --git a/build/fbcode_builder/getdeps/test/features_test.py b/build/fbcode_builder/getdeps/test/features_test.py index 01072b6b04..b994dcce08 100644 --- a/build/fbcode_builder/getdeps/test/features_test.py +++ b/build/fbcode_builder/getdeps/test/features_test.py @@ -21,6 +21,7 @@ def _ctx_with_features(features: set[str] | None = None) -> ManifestContext: "os": "linux", "distro": None, "distro_vers": None, + "distro_family": None, "fb": "off", "fbsource": "off", "test": "off", @@ -64,6 +65,7 @@ def _make_loader(manifests: dict[str, str]) -> ManifestLoader: "os": "linux", "distro": None, "distro_vers": None, + "distro_family": None, "fb": "off", "fbsource": "off", "test": "off", diff --git a/build/fbcode_builder/getdeps/test/platform_test.py b/build/fbcode_builder/getdeps/test/platform_test.py index 52572bb475..858e2f8ddc 100644 --- a/build/fbcode_builder/getdeps/test/platform_test.py +++ b/build/fbcode_builder/getdeps/test/platform_test.py @@ -6,7 +6,7 @@ import unittest -from ..getdeps_platform import HostType +from ..getdeps_platform import HostType, parse_os_release class PlatformTest(unittest.TestCase): @@ -37,3 +37,42 @@ def test_is_methods(self) -> None: self.assertFalse(p.is_windows()) self.assertFalse(p.is_darwin()) self.assertTrue(p.is_linux()) + + +class OsReleaseTest(unittest.TestCase): + def parse(self, name: str, version_id: str) -> tuple[str, str | None, str | None]: + return parse_os_release(f'NAME="{name}"\nVERSION_ID="{version_id}"\n') + + def test_fedora(self) -> None: + self.assertEqual(self.parse("Fedora Linux", "44"), ("linux", "fedora", "44")) + self.assertEqual(HostType("linux", "fedora", "44").distro_family, "fedora") + + def test_centos_stream(self) -> None: + self.assertEqual( + self.parse("CentOS Stream", "9"), ("linux", "centos_stream", "9") + ) + + def test_rhel_gets_short_name_and_major_version(self) -> None: + # EPEL buildroots run real RHEL, whose NAME would otherwise become + # "red_hat_enterprise" and whose VERSION_ID carries a minor. + ostype, distro, vers = self.parse("Red Hat Enterprise Linux", "9.8") + self.assertEqual((ostype, distro, vers), ("linux", "rhel", "9")) + host = HostType(ostype, distro, vers) + self.assertEqual(host.distro_family, "rhel") + self.assertEqual(host.get_package_manager(), "rpm") + + def test_el_rebuilds_share_family_and_major_version(self) -> None: + self.assertEqual(self.parse("AlmaLinux", "9.6"), ("linux", "alma", "9")) + self.assertEqual(self.parse("Rocky Linux", "9.6"), ("linux", "rocky", "9")) + self.assertEqual(HostType("linux", "alma", "9").distro_family, "rhel") + + def test_debian_family_keeps_full_version(self) -> None: + self.assertEqual(self.parse("Ubuntu", "22.04"), ("linux", "ubuntu", "22.04")) + host = HostType("linux", "ubuntu", "22.04") + self.assertEqual(host.distro_family, "debian") + self.assertEqual(host.get_package_manager(), "deb") + + def test_unknown_distro_has_no_family(self) -> None: + host = HostType("linux", "gentoo", None) + self.assertIsNone(host.distro_family) + self.assertIsNone(host.get_package_manager()) diff --git a/build/fbcode_builder/getdeps/test/vendor_test.py b/build/fbcode_builder/getdeps/test/vendor_test.py index ed571cb6d6..1d96fea615 100644 --- a/build/fbcode_builder/getdeps/test/vendor_test.py +++ b/build/fbcode_builder/getdeps/test/vendor_test.py @@ -35,6 +35,7 @@ def make_ctx() -> ManifestContext: "os": "linux", "distro": None, "distro_vers": None, + "distro_family": None, "fb": "off", "fbsource": "off", "test": "off",