diff --git a/buildscripts/mcext b/buildscripts/mcext new file mode 100755 index 0000000000..60e325fd19 --- /dev/null +++ b/buildscripts/mcext @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Create, refresh and verify McCode external-contribution manifests (*.ext). + +A *.ext manifest records, next to the directory an external contribution is +populated into, where each of its files comes from and what its SHA256 is -- +see cmake/Modules/External.cmake for the format and for the CMake side that +consumes it. This script is the other half: it computes those hashes so they +need not be transcribed by hand, and re-checks them against upstream. + + mcext check [PATH ...] do the recorded hashes still match + upstream? (CI-friendly: non-zero exit on + any mismatch). PATH may be a manifest or + a directory to search; default: the whole + McCode source tree. + mcext update MANIFEST [-v TAG] recompute every sha256 in MANIFEST, after + optionally repointing it at a new + upstream tag. + mcext hash URL [URL ...] print the sha256 of each URL. + +"check" is the one to wire into CI. A mismatch means the bytes behind a fixed +reference changed -- a moved tag, a regenerated release archive, a compromised +host -- and that is exactly what the manifest exists to make visible. +""" + +import argparse +import hashlib +import io +import json +import re +import sys +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +CHUNK = 1 << 16 +USER_AGENT = "mccode-mcext/1.0" + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def fetch(url: str) -> bytes: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=60) as response: + return response.read() + + +class Archive: + """A downloaded release archive, with "strip" leading components dropped.""" + + _cache: dict = {} + + @classmethod + def get(cls, spec: dict) -> "Archive": + """One download per URL, however many manifests share the archive.""" + key = (spec["url"], int(spec.get("strip", 1))) + if key not in cls._cache: + cls._cache[key] = cls(spec) + return cls._cache[key] + + def __init__(self, spec: dict): + self.url = spec["url"] + self.strip = int(spec.get("strip", 1)) + self.blob = fetch(self.url) + self.sha256 = sha256_bytes(self.blob) + if self.url.endswith(".zip"): + self._zip = zipfile.ZipFile(io.BytesIO(self.blob)) + self._tar = None + else: + self._tar = tarfile.open(fileobj=io.BytesIO(self.blob), mode="r:*") + self._zip = None + + def _members(self): + if self._tar is not None: + return [m.name for m in self._tar.getmembers() if m.isfile()] + return [n for n in self._zip.namelist() if not n.endswith("/")] + + def read(self, member: str) -> bytes: + wanted = member.lstrip("./") + for name in self._members(): + stripped = "/".join(name.split("/")[self.strip:]) + if stripped == wanted: + if self._tar is not None: + return self._tar.extractfile(name).read() + return self._zip.read(name) + raise KeyError(f"{member!r} is not in {self.url}") + + +def inherit(entry: dict, defaults: dict, key: str): + value = entry.get(key) + return defaults.get(key) if value is None else value + + +def split_manifest(document): + """Return (entries, contribution-wide defaults) for either manifest form.""" + if isinstance(document, list): + return document, {} + if isinstance(document, dict): + entries = document.get("files") + if not isinstance(entries, list): + raise ValueError('object manifest has no "files" array') + return entries, document + raise ValueError("manifest must be a JSON array or object") + + +def derived_base(entry: dict, defaults: dict): + git = inherit(entry, defaults, "git") + version = inherit(entry, defaults, "version") + if not git or not version: + return None + repo = re.sub(r"\.git$", "", git.rstrip("/")) + match = re.fullmatch(r"https?://github\.com/([^/]+)/([^/]+)", repo) + if not match: + return None + return f"https://raw.githubusercontent.com/{match[1]}/{match[2]}/{version}/" + + +def entry_bytes(entry: dict, defaults: dict, archive): + """Fetch one entry's content; mirrors External.cmake's resolution order.""" + name = entry["name"] + source = entry.get("from", name) + url = inherit(entry, defaults, "url") + base = inherit(entry, defaults, "base") + if url: + return fetch(url), url + if base: + full = base.rstrip("/") + "/" + source + return fetch(full), full + if archive is not None: + return archive.read(source), f"{archive.url}!{source}" + base = derived_base(entry, defaults) + if base: + full = base + source + return fetch(full), full + raise ValueError(f'entry {name!r} names no source') + + +def load(path: Path): + document = json.loads(path.read_text()) + entries, defaults = split_manifest(document) + return document, entries, defaults + + +def open_archive(defaults: dict, entries): + """Download the release archive only if some entry actually needs it.""" + spec = defaults.get("archive") + if not spec: + return None + for entry in entries: + if not inherit(entry, defaults, "url") and not inherit(entry, defaults, "base"): + return Archive.get(spec) + return None + + +def manifests_under(paths): + found = [] + for raw in paths: + path = Path(raw) + if path.is_dir(): + found.extend(sorted(path.rglob("*.ext"))) + else: + found.append(path) + return found + + +def command_check(args) -> int: + roots = args.paths or [Path(__file__).resolve().parent.parent] + manifests = manifests_under(roots) + if not manifests: + print("no *.ext manifests found", file=sys.stderr) + return 1 + failures = 0 + for path in manifests: + try: + _, entries, defaults = load(path) + archive = open_archive(defaults, entries) + except Exception as error: # noqa: BLE001 + print(f"{path}: FAILED to read: {error}") + failures += 1 + continue + if archive is not None and defaults["archive"].get("sha256") not in (None, archive.sha256): + print(f"{path}: archive MISMATCH {defaults['archive']['url']}") + print(f" recorded {defaults['archive']['sha256']}") + print(f" upstream {archive.sha256}") + failures += 1 + for entry in entries: + name = entry.get("name", "") + try: + blob, origin = entry_bytes(entry, defaults, archive) + except Exception as error: # noqa: BLE001 + print(f"{path}: {name}: FAILED: {error}") + failures += 1 + continue + actual = sha256_bytes(blob) + recorded = inherit(entry, defaults, "sha256") + if recorded is None: + print(f"{path}: {name}: no recorded sha256 (upstream is {actual})") + failures += 1 + elif actual != recorded.lower(): + print(f"{path}: {name}: MISMATCH ({origin})") + print(f" recorded {recorded}") + print(f" upstream {actual}") + failures += 1 + elif args.verbose: + print(f"{path}: {name}: ok") + if failures: + print(f"\n{failures} problem(s) across {len(manifests)} manifest(s)") + return 1 + print(f"{len(manifests)} manifest(s) verified against upstream") + return 0 + + +def command_update(args) -> int: + path = Path(args.manifest) + document, entries, defaults = load(path) + + if args.version: + old = defaults.get("version") + if not old: + print(f"{path}: no \"version\" recorded, so there is nothing to repoint; " + "edit the URLs by hand and re-run without --version", file=sys.stderr) + return 1 + + def repoint(holder, *keys): + for key in keys: + if isinstance(holder.get(key), str): + holder[key] = holder[key].replace(old, args.version) + + repoint(document, "version", "base", "url") + if isinstance(document.get("archive"), dict): + repoint(document["archive"], "url") + for entry in entries: + repoint(entry, "version", "base", "url") + if isinstance(entry.get("archive"), dict): + repoint(entry["archive"], "url") + entries, defaults = split_manifest(document) + + archive = open_archive(defaults, entries) + if archive is not None: + defaults["archive"]["sha256"] = archive.sha256 + print(f"archive {archive.url}\n sha256 {archive.sha256}") + for entry in entries: + blob, origin = entry_bytes(entry, defaults, archive) + entry["sha256"] = sha256_bytes(blob) + print(f"{entry['name']}\n {origin}\n sha256 {entry['sha256']}") + + path.write_text(json.dumps(document, indent=2) + "\n") + print(f"\nwrote {path}") + return 0 + + +def command_hash(args) -> int: + for url in args.urls: + print(f"{sha256_bytes(fetch(url))} {url}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="mcext", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + check = sub.add_parser("check", help="verify recorded hashes against upstream") + check.add_argument("paths", nargs="*", type=Path, + help="manifests or directories to search (default: whole source tree)") + check.add_argument("-v", "--verbose", action="store_true", help="also report files that match") + check.set_defaults(run=command_check) + + update = sub.add_parser("update", help="recompute the hashes in one manifest") + update.add_argument("manifest") + update.add_argument("-v", "--version", help="repoint the manifest at this upstream tag first") + update.set_defaults(run=command_update) + + hash_cmd = sub.add_parser("hash", help="print the sha256 of one or more URLs") + hash_cmd.add_argument("urls", nargs="+") + hash_cmd.set_defaults(run=command_hash) + + args = parser.parse_args() + return args.run(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cmake/Modules/ChopperLib.cmake b/cmake/Modules/ChopperLib.cmake deleted file mode 100644 index 80b67ad642..0000000000 --- a/cmake/Modules/ChopperLib.cmake +++ /dev/null @@ -1,45 +0,0 @@ -# Fetches the contributed mcstas-chopper-lib repo (Greg Tucker, -# https://github.com/mcdotstar/mcstas-chopper-lib) at configure time and -# installs its pieces into the three McStas resource locations they belong -# in. This is deliberately *not* built as a CMake subproject: chopper-lib's -# own CMakeLists.txt builds a compiled chopper_lib library plus its CTest -# suite (and requires CMake >= 3.25 to do so), but McStas has no use for a -# compiled library here -- chopper-lib.c/.h are %include-d as raw source -# directly into generated instrument C, exactly like the existing -# mcstas-comps/share/*-lib.c snippets. We therefore only Populate the -# source tree (content_fetch, see fetcher.cmake) and copy/install specific -# files out of it; the 3.25 floor in chopper-lib's own CMakeLists.txt is -# never evaluated. -include(fetcher) - -set( CHOPPERLIB_REPO "https://github.com/mcdotstar/mcstas-chopper-lib.git" CACHE STRING - "Location (URL or local path) of mcstas-chopper-lib sources." ) -set( CHOPPERLIB_VERSION "v4.1.0" CACHE STRING - "Git tag/ref of mcstas-chopper-lib to fetch. Pinned rather than tracking a branch: \ -chopper-lib is young, under active development, and its own README documents breaking \ -field renames between major versions." ) - -content_fetch(chopperlib "${CHOPPERLIB_VERSION}" "${CHOPPERLIB_REPO}") - -# 1) C library snippets, %include-d by components/instruments -> resources/share -# (alongside e.g. monitor_nd-lib.c) -install( FILES "${chopperlib_SOURCE_DIR}/chopper-lib.c" - "${chopperlib_SOURCE_DIR}/chopper-lib.h" - DESTINATION "${DEST_DATADIR_CODEFILES}" ) - -# 2) Components -> resources/contrib -file( GLOB CHOPPERLIB_COMPS "${chopperlib_SOURCE_DIR}/*.comp" ) -install( FILES ${CHOPPERLIB_COMPS} DESTINATION "${DEST_DATADIR_COMPS}/contrib" ) - -# 3) Instruments -> one folder per instrument under resources/examples/Tests_optics, -# matching the Test_Xxx/Test_Xxx.instr convention already used there. -file( GLOB CHOPPERLIB_INSTRUMENTS "${chopperlib_SOURCE_DIR}/*.instr" ) -foreach( chopperlib_instr ${CHOPPERLIB_INSTRUMENTS} ) - get_filename_component( chopperlib_instr_name "${chopperlib_instr}" NAME_WE ) - install( FILES "${chopperlib_instr}" - DESTINATION "${DEST_DATADIR_EXAMPLES}/Tests_optics/${chopperlib_instr_name}" ) -endforeach() -unset( chopperlib_instr ) -unset( chopperlib_instr_name ) - -message( STATUS "mcstas-chopper-lib ${CHOPPERLIB_VERSION}: staged from ${chopperlib_SOURCE_DIR}" ) diff --git a/cmake/Modules/External.cmake b/cmake/Modules/External.cmake new file mode 100644 index 0000000000..f0028953b1 --- /dev/null +++ b/cmake/Modules/External.cmake @@ -0,0 +1,514 @@ +# External contributions -- "*.ext" manifests +# =========================================================================== +# +# Replaces the one-off ChopperLib.cmake with a generic mechanism. +# +# An *external contribution* is a set of files (library snippets, components, +# example instruments, data) that live in somebody else's repository but are +# shipped as part of a McCode installation. Rather than hiding that fact in a +# CMake module under cmake/Modules/, each contribution declares itself with a +# small JSON manifest named ".ext", placed **in the directory the +# files are populated into**. A developer looking for, say, a component in +# mcstas-comps/contrib/ therefore finds either the .comp itself or an .ext +# file naming the upstream repository, release and file hash it comes from. +# +# docs/EXTERNAL-CONTRIBUTIONS.md is the prose version of everything below, and +# buildscripts/mcext computes and re-verifies the hashes a manifest records. +# +# --------------------------------------------------------------------------- +# Manifest format +# --------------------------------------------------------------------------- +# +# The short ("flat") form is a JSON array of file entries, each fully +# self-describing: +# +# [ +# { "name": "chopper-lib.h", +# "git": "https://github.com/mcdotstar/mcstas-chopper-lib.git", +# "url": "https://raw.githubusercontent.com/.../v4.1.0/chopper-lib.h", +# "sha256": "0965f666..." }, +# ... +# ] +# +# The long form is a JSON object holding contribution-wide defaults plus a +# "files" array; every key understood in a file entry may also be given at the +# top level, where it acts as a default for all entries: +# +# { +# "name": "mcstas-chopper-lib", +# "git": "https://github.com/mcdotstar/mcstas-chopper-lib.git", +# "version": "v4.1.0", +# "license": "BSD-3-Clause", +# "base": "https://raw.githubusercontent.com/mcdotstar/mcstas-chopper-lib/v4.1.0/", +# "archive": { "url": "...tar.gz", "sha256": "...", "strip": 1 }, +# "files": [ { "name": "chopper-lib.h", "sha256": "0965f666..." } ] +# } +# +# File-entry keys: +# +# name (required) file name upstream; also the default install name. +# sha256 (required) SHA256 of the file contents. The build fails loudly if +# what arrives does not match, so a moved tag or a rewritten release +# can never silently change what McCode ships. +# as install path relative to the .ext file's own directory. May name +# a subdirectory ("Foo/Foo.instr"), which is how the one-directory- +# per-instrument layout under examples/ is reproduced. Default: name. +# url explicit download URL for this one file. +# from path of the file inside the release archive. Default: name. +# base URL prefix; the file is fetched from "". +# git upstream repository. Informational, but if "base", "url" and +# "archive" are all absent and "git" points at github.com, a raw +# base URL is derived from "git" + "version". +# version upstream tag/ref, used for the derivation above and in messages. +# +# Contribution-wide keys: +# +# files (required in the long form) array of file entries. +# archive { "url", "sha256", "strip" } -- a release tarball/zip. Downloaded +# and unpacked once; entries without "url"/"base" are copied out of +# it. "strip" leading path components are dropped (default 1, which +# matches GitHub's auto-generated source archives). +# license, description, homepage -- recorded in configure output only. +# +# Resolution order for a single file: "url", else "base", else "archive", +# else derived-from-"git". Mixing is fine: a manifest may take most files from +# a release tarball and one from a direct URL. +# +# --------------------------------------------------------------------------- +# Usage +# --------------------------------------------------------------------------- +# +# include( External ) +# mccode_install_externals( DIRECTORY "contrib" DESTINATION "${DEST_DATADIR_COMPS}/contrib" ) +# +# i.e. one call mirroring each install( DIRECTORY ... ) already present, so +# the staged external files land exactly where the in-tree files of that +# directory do. +# +# mccode_install_externals( +# DIRECTORY # absolute, or relative to CMAKE_CURRENT_SOURCE_DIR +# DESTINATION # install destination, as for install( FILES ) +# [ STAGE ] # where fetched files are assembled; default +# # ${CMAKE_CURRENT_BINARY_DIR}/externals/ +# [ COMPONENT ] # passed through to install() +# [ NO_RECURSE ] # only look for *.ext directly in +# [ OUTPUT_VARIABLE ] # staged file paths, set in the caller's scope +# ) +# +# --------------------------------------------------------------------------- +# Cache and offline builds +# --------------------------------------------------------------------------- +# +# Downloads are content-addressed under MCCODE_EXTERNALS_CACHE, so a file +# shared by several manifests is fetched once and survives a wiped build +# directory if the cache is pointed somewhere persistent. Distribution +# packagers who may not fetch during a build have two options: pre-populate +# that cache, or drop the files into MCCODE_EXTERNALS_LOCAL (searched by file +# name, still hash-verified) and set MCCODE_EXTERNALS_OFFLINE=ON so that any +# attempt to reach the network is a hard error rather than a silent download. + +include_guard( GLOBAL ) + +# This module needs string( JSON ) (CMake 3.19) and file( ARCHIVE_EXTRACT ) +# (3.18); McCode's cmake_minimum_required was raised to 3.19 to match. + +option( ENABLE_EXTERNALS + "Populate external (*.ext) contributions from their upstream sources" ON ) +option( MCCODE_EXTERNALS_OFFLINE + "Never download: resolve every external file from MCCODE_EXTERNALS_LOCAL or the cache" OFF ) +option( MCCODE_EXTERNALS_ALLOW_UNVERIFIED + "Permit external file entries that carry no sha256 (strongly discouraged)" OFF ) +set( MCCODE_EXTERNALS_CACHE "${CMAKE_BINARY_DIR}/externals-cache" CACHE PATH + "Content-addressed download cache for external (*.ext) contributions." ) +set( MCCODE_EXTERNALS_LOCAL "" CACHE PATH + "Directory of pre-fetched external contribution files, searched by name before downloading." ) +set( MCCODE_EXTERNALS_TIMEOUT "60" CACHE STRING + "Per-file download timeout, in seconds, for external (*.ext) contributions." ) + +mark_as_advanced( MCCODE_EXTERNALS_ALLOW_UNVERIFIED MCCODE_EXTERNALS_TIMEOUT ) + + +# --- internal helpers ------------------------------------------------------ + +# Fetch a JSON member, yielding "" rather than an error for absent keys. +function( _mcext_get out_var json ) + string( JSON value ERROR_VARIABLE error GET "${json}" ${ARGN} ) + if ( error ) + set( ${out_var} "" PARENT_SCOPE ) + else() + set( ${out_var} "${value}" PARENT_SCOPE ) + endif() +endfunction() + +# Take the entry value if present, otherwise the contribution-wide default. +function( _mcext_inherit out_var entry defaults key ) + _mcext_get( value "${entry}" ${key} ) + if ( value STREQUAL "" ) + _mcext_get( value "${defaults}" ${key} ) + endif() + set( ${out_var} "${value}" PARENT_SCOPE ) +endfunction() + +# Place (hash ) in the cache and return its path in out_var. +#