From 25a27a0d6f69683035db975d094101244e05da65 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 14 Nov 2026 11:13:07 +0000 Subject: [PATCH 1/6] test(native): adversarial wire, privilege, network and jail suites Property and root e2e tests for the helper's request parser, caller gate, setuid install, socket pins, lock dir, per-VM network and firewall, fork_delta, path types, dm serde, elfsum stamping, guest-agent exec and planted jail entries. --- native/Cargo.lock | 1 + native/guest-agent/Cargo.toml | 5 + native/guest-agent/tests/exec_properties.rs | 97 ++++ .../tests/stamped_slot_runtime.rs | 79 +++ .../tests/jail_root_planted.rs | 158 +++++ native/hyperrs-fs/tests/path_properties.rs | 212 +++++++ .../tests/dm_serde_properties.rs | 83 +++ native/suidhelper/Cargo.toml | 44 ++ native/suidhelper/src/net/firewall.rs | 125 ++++ native/suidhelper/src/net/vm.rs | 255 +++++++- native/suidhelper/tests/caller_authorize.rs | 143 +++++ native/suidhelper/tests/cli_request.rs | 387 ++++++++++++ native/suidhelper/tests/e2e/caller_gate.rs | 265 +++++++++ native/suidhelper/tests/e2e/claim_privacy.rs | 298 ++++++++++ .../suidhelper/tests/e2e/fork_delta_live.rs | 314 ++++++++++ native/suidhelper/tests/e2e/jailer.rs | 93 +++ .../suidhelper/tests/e2e/lock_dir_hostile.rs | 172 ++++++ native/suidhelper/tests/e2e/setuid_support.rs | 263 +++++++++ native/suidhelper/tests/e2e/socket_pin.rs | 373 ++++++++++++ native/suidhelper/tests/e2e/vm_net.rs | 549 ++++++++++++++++++ native/suidhelper/tests/e2e/vm_net_traffic.rs | 504 ++++++++++++++++ native/suidhelper/tests/net_addressing.rs | 150 +++++ .../tests/request_wire_properties.rs | 269 +++++++++ 23 files changed, 4838 insertions(+), 1 deletion(-) create mode 100644 native/guest-agent/tests/exec_properties.rs create mode 100644 native/hyperrs-elfsum/tests/stamped_slot_runtime.rs create mode 100644 native/hyperrs-firecracker/tests/jail_root_planted.rs create mode 100644 native/hyperrs-fs/tests/path_properties.rs create mode 100644 native/hyperrs-linux/tests/dm_serde_properties.rs create mode 100644 native/suidhelper/tests/caller_authorize.rs create mode 100644 native/suidhelper/tests/cli_request.rs create mode 100644 native/suidhelper/tests/e2e/caller_gate.rs create mode 100644 native/suidhelper/tests/e2e/claim_privacy.rs create mode 100644 native/suidhelper/tests/e2e/fork_delta_live.rs create mode 100644 native/suidhelper/tests/e2e/lock_dir_hostile.rs create mode 100644 native/suidhelper/tests/e2e/setuid_support.rs create mode 100644 native/suidhelper/tests/e2e/socket_pin.rs create mode 100644 native/suidhelper/tests/e2e/vm_net.rs create mode 100644 native/suidhelper/tests/e2e/vm_net_traffic.rs create mode 100644 native/suidhelper/tests/net_addressing.rs create mode 100644 native/suidhelper/tests/request_wire_properties.rs diff --git a/native/Cargo.lock b/native/Cargo.lock index 29b4af0f..6b232153 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -972,6 +972,7 @@ dependencies = [ "nix 0.29.0", "pid1", "pretty_assertions", + "proptest", "prost", "rstest", "thiserror 1.0.69", diff --git a/native/guest-agent/Cargo.toml b/native/guest-agent/Cargo.toml index 65018537..edd3a668 100644 --- a/native/guest-agent/Cargo.toml +++ b/native/guest-agent/Cargo.toml @@ -30,9 +30,14 @@ tonic-prost-build = "0.14" [dev-dependencies] pretty_assertions = "1" +proptest = "1" rstest = "0.28" tokio-stream = "0.1" [[test]] name = "exec" path = "tests/exec.rs" + +[[test]] +name = "exec_properties" +path = "tests/exec_properties.rs" diff --git a/native/guest-agent/tests/exec_properties.rs b/native/guest-agent/tests/exec_properties.rs new file mode 100644 index 00000000..971346e7 --- /dev/null +++ b/native/guest-agent/tests/exec_properties.rs @@ -0,0 +1,97 @@ +//! Laws of the guest side of `Exec` (`exec::run`), which the host's +//! `Hyper.exec/3` contract rests on: +//! +//! - argv round-trips: the program sees exactly the argument vector sent, +//! with no shell splitting, globbing or re-quoting; +//! - stdout and stderr are each captured byte-for-byte, arbitrary bytes and +//! outputs larger than a pipe buffer on both streams at once included; +//! - the exit status is the program's own `0..=255` code; +//! - a non-empty env *replaces* the environment: the program sees exactly +//! that map and nothing inherited. + +use hyper_guest_agent::exec::run; +use proptest::collection::{btree_map, vec}; +use proptest::prelude::*; +use std::collections::BTreeMap; +use std::io::Write; + +fn sh(script: &str, args: &[String]) -> Vec { + let mut argv = vec!["/bin/sh".into(), "-c".into(), script.into(), "sh".into()]; + argv.extend_from_slice(args); + argv +} + +fn arg() -> impl Strategy { + any::().prop_map(|s| s.replace('\0', "")) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(64))] + + #[test] + fn argv_reaches_the_program_unchanged(args in vec(arg(), 0..8)) { + let (code, out, _) = run(&sh(r#"for a; do printf '%s\0' "$a"; done"#, &args), &BTreeMap::new(), None); + prop_assert_eq!(code, 0); + let expected: Vec = args.iter().flat_map(|a| a.bytes().chain([0])).collect(); + prop_assert_eq!(out, expected); + } + + #[test] + fn exit_status_is_the_programs_own(n in 0u8..=255) { + let (code, _, _) = run(&sh(&format!("exit {n}"), &[]), &BTreeMap::new(), None); + prop_assert_eq!(code, i32::from(n)); + } + + #[test] + fn non_empty_env_is_exactly_what_the_program_sees( + env in btree_map("[A-Z_][A-Z0-9_]{0,12}", "[ -~]{0,24}", 1..8) + ) { + let (code, out, _) = run(&["/usr/bin/env".into()], &env, None); + prop_assert_eq!(code, 0); + let seen: BTreeMap = std::str::from_utf8(&out) + .unwrap() + .lines() + .map(|l| { + let (k, v) = l.split_once('=').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + prop_assert_eq!(seen, env); + } +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(16))] + + #[test] + fn both_streams_are_captured_byte_for_byte( + stdout in vec(any::(), 0..300_000), + stderr in vec(any::(), 0..300_000), + ) { + let dir = std::env::temp_dir(); + let tag = format!("{}-{}", std::process::id(), rand_tag()); + let out_path = dir.join(format!("ga-out-{tag}")); + let err_path = dir.join(format!("ga-err-{tag}")); + std::fs::File::create(&out_path).unwrap().write_all(&stdout).unwrap(); + std::fs::File::create(&err_path).unwrap().write_all(&stderr).unwrap(); + + let args = [ + out_path.to_string_lossy().into_owned(), + err_path.to_string_lossy().into_owned(), + ]; + let (code, out, err) = + run(&sh(r#"cat "$1" & cat "$2" >&2; wait"#, &args), &BTreeMap::new(), None); + let _ = std::fs::remove_file(&out_path); + let _ = std::fs::remove_file(&err_path); + + prop_assert_eq!(code, 0); + prop_assert!(out == stdout, "stdout differs: {} vs {} bytes", out.len(), stdout.len()); + prop_assert!(err == stderr, "stderr differs: {} vs {} bytes", err.len(), stderr.len()); + } +} + +fn rand_tag() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + NEXT.fetch_add(1, Ordering::Relaxed) +} diff --git a/native/hyperrs-elfsum/tests/stamped_slot_runtime.rs b/native/hyperrs-elfsum/tests/stamped_slot_runtime.rs new file mode 100644 index 00000000..14fe362a --- /dev/null +++ b/native/hyperrs-elfsum/tests/stamped_slot_runtime.rs @@ -0,0 +1,79 @@ +//! The end-to-end contract the helper's install-time verification rests on: +//! the digest `Elf::stamp` writes into a built executable is exactly what +//! `checksum!()` reports when that executable runs, and a verifier that +//! re-zeroes `.note.sum` and BLAKE3-hashes the file recomputes it. Laws: +//! * runtime report == stamped digest (the stamper patched the slot the +//! loader maps, and the read is not constant-folded to the link-time zero); +//! * re-zero + rehash of the stamped file == stamped digest; +//! * an unstamped executable reports all zeroes. +#![cfg(all(target_os = "linux", feature = "stamp"))] + +use hyperrs_elfsum::Elf; +use pretty_assertions::assert_eq; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::Command; + +const CHILD: &str = "HYPER_ELFSUM_REPORT_SLOT"; +const TEST: &str = "a_stamped_executable_reports_its_own_stamp_at_runtime"; + +fn hex(d: &[u8]) -> String { + d.iter().map(|b| format!("{b:02x}")).collect() +} + +fn report(exe: &Path) -> String { + let out = Command::new(exe) + .args(["--exact", TEST, "--nocapture", "--test-threads=1"]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let stdout = String::from_utf8(out.stdout).unwrap(); + stdout + .split_once("slot=") + .and_then(|(_, rest)| rest.get(..64)) + .unwrap_or_else(|| panic!("child printed no slot: {stdout}")) + .to_string() +} + +#[test] +fn a_stamped_executable_reports_its_own_stamp_at_runtime() { + if std::env::var_os(CHILD).is_some() { + println!("slot={}", hyperrs_elfsum::checksum!().hex()); + return; + } + let dir = std::env::temp_dir().join(format!("elfsum-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let original = fs::read("/proc/self/exe").unwrap(); + + let unstamped = dir.join("unstamped"); + fs::write(&unstamped, &original).unwrap(); + fs::set_permissions(&unstamped, fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(report(&unstamped), "0".repeat(64)); + + let mut bytes = original.clone(); + let digest = Elf::read(&mut bytes[..]).unwrap().stamp(); + let stamped = dir.join("stamped"); + fs::write(&stamped, &bytes).unwrap(); + fs::set_permissions(&stamped, fs::Permissions::from_mode(0o755)).unwrap(); + + assert_eq!(report(&stamped), hex(&digest)); + + let changed: Vec = (0..bytes.len()) + .filter(|&i| bytes[i] != original[i]) + .collect(); + let (first, last) = (changed[0], *changed.last().unwrap()); + assert!( + last - first < digest.len(), + "stamp touched bytes outside one slot" + ); + let at = (0..=bytes.len() - digest.len()) + .find(|&i| bytes[i..i + digest.len()] == digest) + .unwrap(); + let mut rezeroed = bytes.clone(); + rezeroed[at..at + digest.len()].fill(0); + assert_eq!(*blake3::hash(&rezeroed).as_bytes(), digest); + + fs::remove_dir_all(&dir).unwrap(); +} diff --git a/native/hyperrs-firecracker/tests/jail_root_planted.rs b/native/hyperrs-firecracker/tests/jail_root_planted.rs new file mode 100644 index 00000000..06fbdbf0 --- /dev/null +++ b/native/hyperrs-firecracker/tests/jail_root_planted.rs @@ -0,0 +1,158 @@ +//! The jail `root/` is owned by the jail uid, so a Firecracker a hostile guest has taken over can +//! plant any entry there before the helper stages into it. Law: a name the jail uid planted where +//! the helper stages `vmlinux` or `rootfs` is never written through, retyped or re-owned. + +use assert_fs::fixture::ChildPath; +use assert_fs::prelude::*; +use assert_fs::TempDir; +use hyperrs_firecracker::jailer::chroot::{JailLayout, JailRoot, KernelImage}; +use hyperrs_firecracker::VmId; +use hyperrs_fs::dir::DirError; +use hyperrs_fs::file::{self, Any, IsBlockDevice, OpenFlags, Owner, SafeFile}; +use hyperrs_fs::path::{IsAbsolute, SafePathBuf, StrictComponents}; +use nix::errno::Errno; +use nix::unistd::{getgid, getuid}; +use pretty_assertions::assert_eq; +use rstest::{fixture, rstest}; +use std::assert_matches; +use std::fs; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; + +type Strict = SafePathBuf; + +fn strict(p: impl AsRef) -> Strict { + p.as_ref().to_path_buf().try_into().unwrap() +} + +fn vm() -> VmId { + "hostile".parse().unwrap() +} + +struct Jail { + base: TempDir, + root: ChildPath, +} + +impl Jail { + fn open_root(&self) -> JailRoot { + JailLayout::new(&strict(self.base.path()), &vm()) + .open_root() + .unwrap() + } +} + +#[fixture] +fn jail() -> Jail { + let base = TempDir::new().unwrap(); + let root = base.child("firecracker/hostile/root"); + root.create_dir_all().unwrap(); + Jail { base, root } +} + +fn any_block_device() -> Option { + fs::read_dir("/dev") + .ok()? + .flatten() + .map(|e| e.path()) + .find(|p| { + fs::symlink_metadata(p) + .map(|m| m.file_type().is_block_device()) + .unwrap_or(false) + }) +} + +fn me() -> Owner { + Owner { + uid: getuid(), + gid: getgid(), + } +} + +fn is_root() -> bool { + nix::unistd::geteuid().is_root() +} + +#[rstest] +fn add_rootfs_over_a_planted_regular_file_refuses_and_leaves_it(jail: Jail) { + let Some(dev) = any_block_device() else { + eprintln!( + "SKIP add_rootfs_over_a_planted_regular_file_refuses_and_leaves_it: no block device" + ); + return; + }; + let planted = jail.root.child("rootfs"); + planted.write_str("planted").unwrap(); + fs::set_permissions(&planted, fs::Permissions::from_mode(0o644)).unwrap(); + let device = SafeFile::::open(&strict(&dev), OpenFlags::PATH).unwrap(); + + let err = jail.open_root().add_rootfs(&device, me()).unwrap_err(); + + assert_matches!(err, DirError::Open(file::ValidationError::WrongFileType)); + planted.assert("planted"); + assert_eq!(fs::metadata(&planted).unwrap().mode() & 0o7777, 0o644); +} + +#[rstest] +fn add_rootfs_over_a_planted_symlink_refuses_and_never_touches_its_target(jail: Jail) { + let Some(dev) = any_block_device() else { + eprintln!("SKIP add_rootfs_over_a_planted_symlink: no block device"); + return; + }; + let victim = jail.base.child("victim"); + victim.write_str("victim").unwrap(); + fs::set_permissions(&victim, fs::Permissions::from_mode(0o640)).unwrap(); + jail.root.child("rootfs").symlink_to_file(&victim).unwrap(); + let device = SafeFile::::open(&strict(&dev), OpenFlags::PATH).unwrap(); + + let err = jail.open_root().add_rootfs(&device, me()).unwrap_err(); + + assert_matches!( + err, + DirError::Open(file::ValidationError::Open(Errno::ELOOP)) + ); + victim.assert("victim"); + assert_eq!(fs::metadata(&victim).unwrap().mode() & 0o7777, 0o640); +} + +/// Root-only: unprivileged `linkat(AT_EMPTY_PATH)` is `ENOENT` before 6.10, so neither the +/// hard-link nor the `EXDEV` copy path would be reached. +#[rstest] +fn stage_kernel_never_writes_through_a_planted_vmlinux_symlink_as_root( + jail: Jail, + #[values(false, true)] cross_device: bool, +) { + if !is_root() { + eprintln!("SKIP stage_kernel_never_writes_through_a_planted_vmlinux_symlink_as_root"); + return; + } + let src_dir = if cross_device { + match TempDir::new_in("/dev/shm") { + Ok(d) => d, + Err(_) => { + eprintln!("SKIP cross-device staging: no /dev/shm"); + return; + } + } + } else { + TempDir::new_in(jail.base.path()).unwrap() + }; + let src = src_dir.child("vmlinux"); + src.write_str("kernel").unwrap(); + fs::set_permissions(&src, fs::Permissions::from_mode(0o644)).unwrap(); + let victim = jail.base.child("victim"); + victim.write_str("victim").unwrap(); + jail.root.child("vmlinux").symlink_to_file(&victim).unwrap(); + + let err = jail + .open_root() + .stage_kernel(&KernelImage::open(&strict(&src)).unwrap()) + .unwrap_err(); + + assert_eq!(err.errno(), Some(Errno::EEXIST), "{err:?}"); + victim.assert("victim"); + assert!(fs::symlink_metadata(jail.root.join("vmlinux")) + .unwrap() + .file_type() + .is_symlink()); +} diff --git a/native/hyperrs-fs/tests/path_properties.rs b/native/hyperrs-fs/tests/path_properties.rs new file mode 100644 index 00000000..9f4a7863 --- /dev/null +++ b/native/hyperrs-fs/tests/path_properties.rs @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: MIT +//! Contract of the strict path types every confinement check in the helper is built on. +//! +//! Laws under test, each against an independent byte-level oracle: +//! - **Component oracle.** A byte string is a [`StrictComponent`] iff it is non-empty, not `.`/`..`, +//! at most `NAME_MAX` bytes, and holds no `/`, NUL or ASCII whitespace. Non-UTF-8 bytes are fine. +//! - **Path oracle.** A raw byte path is a strict absolute [`SafePathBuf`] iff it starts with `/` +//! and either is exactly `/` or every `/`-separated segment passes the component oracle. Parsing +//! through `&str`, `PathBuf` and `&Path` all agree. +//! - **Already normal.** An accepted path is a fixed point of std's component normalization, so no +//! later `Path::components`/`join` can reinterpret it. +//! - **`strip_base` is component-prefix.** It returns exactly the trailing segments when the base's +//! segments are a proper prefix, `EqualsBase` on equality, `NotUnderBase` otherwise (including a +//! byte-prefix that is not a component prefix, e.g. `/ab` under `/a`). +//! - **`split_last` inverts `join`.** For any non-root path, `parent.join(name)` rebuilds it, the +//! parent is itself strict, and `name` is `file_name()`; `/` has no split. + +use hyperrs_fs::path::{ + ComponentError, IsAbsolute, RelativeError, SafePath, SafePathBuf, StrictComponent, + StrictComponents, ValidationError, NAME_MAX, +}; +use proptest::prelude::*; +use std::assert_matches; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::{Component, Path, PathBuf}; + +type Strict = SafePathBuf; + +fn oracle_component(bytes: &[u8]) -> bool { + !bytes.is_empty() + && bytes != b"." + && bytes != b".." + && bytes.len() <= NAME_MAX + && !bytes + .iter() + .any(|b| matches!(b, b'/' | 0 | b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r')) +} + +fn oracle_path(raw: &[u8]) -> bool { + match raw { + [b'/'] => true, + [b'/', rest @ ..] => rest.split(|&b| b == b'/').all(oracle_component), + _ => false, + } +} + +fn byte() -> impl Strategy { + prop_oneof![ + 6 => b'a'..=b'z', + 2 => Just(b'.'), + 1 => Just(b'/'), + 1 => Just(0u8), + 1 => prop::sample::select(vec![b' ', b'\t', b'\n', 0x0b, 0x0c, b'\r']), + 1 => 0x80u8..=0xff, + 1 => Just(b'-'), + ] +} + +fn component_bytes() -> impl Strategy> { + prop_oneof![ + 8 => prop::collection::vec(byte(), 0..6), + 1 => prop::collection::vec(b'a'..=b'z', NAME_MAX - 2..NAME_MAX + 3), + 1 => prop::sample::select(vec![b".".to_vec(), b"..".to_vec(), b"...".to_vec(), b".a".to_vec()]), + ] +} + +fn raw_path() -> impl Strategy> { + ( + any::(), + prop::collection::vec(component_bytes(), 0..5), + any::(), + ) + .prop_map(|(absolute, segments, trailing)| { + let mut raw = if absolute { b"/".to_vec() } else { Vec::new() }; + raw.extend(segments.join(&b'/')); + if trailing { + raw.push(b'/'); + } + raw + }) +} + +fn strict_raw(raw: &[u8]) -> Result { + PathBuf::from(OsStr::from_bytes(raw)).try_into() +} + +fn seg() -> impl Strategy { + prop::sample::select(vec!["a", "b", "ab", "a.b", "-"]) +} + +fn abs(segments: &[&str]) -> Strict { + format!("/{}", segments.join("/")).parse().unwrap() +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(2048))] + + #[test] + fn component_accepts_exactly_the_oracle(bytes in component_bytes()) { + let got = <&StrictComponent>::try_from(OsStr::from_bytes(&bytes)); + prop_assert_eq!(got.is_ok(), oracle_component(&bytes), "{:?}", bytes); + if let Ok(name) = got { + prop_assert_eq!(name.as_os_str().as_bytes(), &bytes[..]); + let owned = name.to_owned(); + prop_assert_eq!(owned.as_os_str().as_bytes(), &bytes[..]); + } + } + + #[test] + fn path_accepts_exactly_the_oracle(raw in raw_path()) { + let got = strict_raw(&raw); + prop_assert_eq!(got.is_ok(), oracle_path(&raw), "{:?}", String::from_utf8_lossy(&raw)); + if raw.first() != Some(&b'/') { + prop_assert!(matches!(got, Err(ValidationError::NotAbsolute))); + } + let borrowed: Result<&SafePath, _> = + Path::new(OsStr::from_bytes(&raw)).try_into(); + prop_assert_eq!(borrowed.is_ok(), got.is_ok()); + if let Ok(utf8) = std::str::from_utf8(&raw) { + prop_assert_eq!(utf8.parse::().is_ok(), got.is_ok()); + } + } + + #[test] + fn accepted_paths_are_already_normal(raw in raw_path()) { + if let Ok(path) = strict_raw(&raw) { + let normal: PathBuf = path.as_path().components().collect(); + prop_assert_eq!(normal.as_os_str().as_bytes(), &raw[..]); + prop_assert!(path + .as_path() + .components() + .all(|c| matches!(c, Component::RootDir | Component::Normal(_)))); + } + } + + #[test] + fn strip_base_is_a_component_prefix( + base in prop::collection::vec(seg(), 0..3), + path in prop::collection::vec(seg(), 0..5), + ) { + let got = abs(&path).strip_base(&abs(&base)).map(|rel| { + rel.segments() + .map(|s| s.as_os_str().to_string_lossy().into_owned()) + .collect::>() + }); + if base == path { + prop_assert!(matches!(got, Err(RelativeError::EqualsBase)), "{got:?}"); + } else if path.starts_with(&base) { + let want: Vec = path[base.len()..].iter().map(|s| s.to_string()).collect(); + prop_assert_eq!(got.unwrap(), want); + } else { + prop_assert!(matches!(got, Err(RelativeError::NotUnderBase)), "{got:?}"); + } + } + + #[test] + fn split_last_inverts_join(path in prop::collection::vec(seg(), 0..5)) { + let full = abs(&path); + match full.split_last() { + None => prop_assert!(path.is_empty()), + Some((parent, name)) => { + let rebuilt = parent.join(name); + let want_parent = abs(&path[..path.len() - 1]); + prop_assert_eq!(rebuilt.as_path(), full.as_path()); + prop_assert_eq!(parent.as_path(), want_parent.as_path()); + prop_assert!(strict_raw(parent.as_path().as_os_str().as_bytes()).is_ok()); + prop_assert_eq!(Some(name), full.file_name()); + prop_assert_eq!(name.as_os_str(), OsStr::new(path[path.len() - 1])); + } + } + } + + #[test] + fn split_last_inverts_join_on_relative_paths( + base in prop::collection::vec(seg(), 0..3), + rest in prop::collection::vec(seg(), 2..5), + ) { + let full_segments: Vec<&str> = base.iter().chain(rest.iter()).copied().collect(); + let full = abs(&full_segments); + let base = abs(&base); + let rel = full.strip_base(&base).unwrap(); + let (parent, name) = rel.split_last().unwrap(); + let rebuilt = parent.join(name); + prop_assert_eq!(rebuilt.as_path(), rel.as_path()); + prop_assert_eq!(rel.file_name(), Some(name)); + } +} + +#[test] +fn name_max_is_the_exact_component_length_boundary() { + let at = "a".repeat(NAME_MAX); + let over = "a".repeat(NAME_MAX + 1); + assert!(<&StrictComponent>::try_from(at.as_str()).is_ok()); + assert_matches!( + <&StrictComponent>::try_from(over.as_str()), + Err(ComponentError::TooLong(n)) if n == NAME_MAX + 1 + ); + assert!(format!("/{at}/{at}").parse::().is_ok()); + assert_matches!( + format!("/{at}/{over}").parse::(), + Err(ValidationError::Component(ComponentError::TooLong(_))) + ); +} + +#[test] +fn byte_prefix_that_is_not_a_component_prefix_is_not_under_base() { + assert_matches!( + abs(&["ab", "c"]).strip_base(&abs(&["a"])), + Err(RelativeError::NotUnderBase) + ); +} diff --git a/native/hyperrs-linux/tests/dm_serde_properties.rs b/native/hyperrs-linux/tests/dm_serde_properties.rs new file mode 100644 index 00000000..f4fb06bd --- /dev/null +++ b/native/hyperrs-linux/tests/dm_serde_properties.rs @@ -0,0 +1,83 @@ +//! The helper's JSON requests deserialize straight into these types, so serde +//! must be exactly as strict as the checked constructors. Laws: +//! * a JSON integer is a `ThinId` iff it is in `0..=ThinId::MAX`, and it +//! keeps its value; any non-integer JSON is refused; +//! * a JSON string deserializes to a `DmName`, `LoopDevNum` or `Origin` +//! exactly when `FromStr` accepts it, and to the same value. +#![cfg(feature = "serde")] + +use hyperrs_linux::dm::{DmName, Origin, ThinId}; +use hyperrs_linux::loopdev::LoopDevNum; +use proptest::prelude::*; +use serde::de::DeserializeOwned; +use serde_json::{json, Value}; +use std::fmt::Debug; +use std::str::FromStr; +use test_strategy::proptest; + +#[proptest] +fn a_json_integer_is_a_thin_id_exactly_when_it_fits_24_bits( + #[strategy(prop_oneof![ + 0i64..=i64::from(ThinId::MAX), + (i64::from(ThinId::MAX) + 1)..=i64::MAX, + i64::MIN..0, + ])] + n: i64, +) { + let got = serde_json::from_value::(json!(n)); + if (0..=i64::from(ThinId::MAX)).contains(&n) { + prop_assert_eq!(u64::from(got.unwrap()), n as u64); + } else { + prop_assert!(got.is_err(), "accepted {}", n); + } +} + +#[proptest] +fn non_integer_json_is_never_a_thin_id( + #[strategy(prop_oneof![ + (0u32..1000).prop_map(|n| json!(f64::from(n) + 0.5)), + (0u32..1000).prop_map(|n| json!(n.to_string())), + Just(json!(null)), + Just(json!(true)), + Just(json!([1])), + Just(json!({"id": 1})), + ])] + v: Value, +) { + prop_assert!( + serde_json::from_value::(v.clone()).is_err(), + "accepted {}", + v + ); +} + +fn agrees_with_from_str(s: &str) -> Result<(), TestCaseError> +where + T: DeserializeOwned + FromStr + PartialEq + Debug, + T::Err: Debug, +{ + let wire = serde_json::from_value::(json!(s)); + match s.parse::() { + Ok(parsed) => prop_assert_eq!(wire.unwrap(), parsed), + Err(_) => prop_assert!(wire.is_err(), "wire accepted {:?} that FromStr refuses", s), + } + Ok(()) +} + +fn text() -> impl Strategy { + prop_oneof![ + "[a-z0-9#+.:=@_-]{0,12}", + "hyper-[a-z0-9]{0,8}", + "/dev/loop[0-9+-]{0,11}", + "/dev/mapper/[a-z0-9./-]{0,10}", + "/[a-z./]{0,12}", + "\\PC{0,8}", + ] +} + +#[proptest(ProptestConfig::with_cases(1024))] +fn string_wire_types_deserialize_exactly_as_from_str(#[strategy(text())] s: String) { + agrees_with_from_str::(&s)?; + agrees_with_from_str::(&s)?; + agrees_with_from_str::(&s)?; +} diff --git a/native/suidhelper/Cargo.toml b/native/suidhelper/Cargo.toml index 189ccf56..e3d80f2c 100644 --- a/native/suidhelper/Cargo.toml +++ b/native/suidhelper/Cargo.toml @@ -27,6 +27,18 @@ required-features = ["insecure_mode"] name = "config_seam" path = "tests/config_seam.rs" +[[test]] +name = "caller_authorize" +path = "tests/caller_authorize.rs" + +[[test]] +name = "cli_request" +path = "tests/cli_request.rs" + +[[test]] +name = "request_wire_properties" +path = "tests/request_wire_properties.rs" + [[test]] name = "e2e_config" path = "tests/e2e/config.rs" @@ -35,6 +47,18 @@ path = "tests/e2e/config.rs" name = "e2e_firewall" path = "tests/e2e/firewall.rs" +[[test]] +name = "e2e_vm_net" +path = "tests/e2e/vm_net.rs" + +[[test]] +name = "e2e_vm_net_traffic" +path = "tests/e2e/vm_net_traffic.rs" + +[[test]] +name = "e2e_caller_gate" +path = "tests/e2e/caller_gate.rs" + [[test]] name = "e2e_argv" path = "tests/e2e/argv.rs" @@ -47,6 +71,18 @@ path = "tests/e2e/jailer.rs" name = "e2e_chroot_jail" path = "tests/e2e/chroot_jail.rs" +[[test]] +name = "e2e_socket_pin" +path = "tests/e2e/socket_pin.rs" + +[[test]] +name = "e2e_lock_dir_hostile" +path = "tests/e2e/lock_dir_hostile.rs" + +[[test]] +name = "e2e_claim_privacy" +path = "tests/e2e/claim_privacy.rs" + [[test]] name = "e2e_version" path = "tests/e2e/version.rs" @@ -59,6 +95,10 @@ path = "tests/e2e/thin.rs" name = "e2e_fork_delta" path = "tests/e2e/fork_delta.rs" +[[test]] +name = "e2e_fork_delta_live" +path = "tests/e2e/fork_delta_live.rs" + [[test]] name = "e2e_img" path = "tests/e2e/img.rs" @@ -71,6 +111,10 @@ path = "tests/config_uid_gid_range.rs" name = "config_paths" path = "tests/config_paths.rs" +[[test]] +name = "net_addressing" +path = "tests/net_addressing.rs" + [[test]] name = "dm_backing" path = "tests/dm/backing.rs" diff --git a/native/suidhelper/src/net/firewall.rs b/native/suidhelper/src/net/firewall.rs index e0265422..d5d88571 100644 --- a/native/suidhelper/src/net/firewall.rs +++ b/native/suidhelper/src/net/firewall.rs @@ -194,3 +194,128 @@ mod tests { assert!(serde_json::from_value::(policy).is_err()); } } + +#[cfg(test)] +mod refusal_properties { + //! Laws over arbitrary policies as the helper's JSON delivers them: + //! - a policy is accepted exactly when every rule names `tcp` or `udp` over a + //! non-empty range without port 0 and each direction holds at most `Rules::MAX`; + //! - an accepted policy renders one destination-port match per rule, in order, over + //! exactly the ports it named. + + use super::{Firewall, Rules}; + use hyperrs_linux::net::nft::{L4Proto, Match}; + use proptest::prelude::*; + use serde_json::{json, Value}; + use test_strategy::proptest; + + #[derive(Debug, Clone)] + struct Raw { + protocol: &'static str, + first: u16, + last: u16, + } + + impl Raw { + fn valid(&self) -> bool { + matches!(self.protocol, "tcp" | "udp") && self.first != 0 && self.first <= self.last + } + + fn json(&self) -> Value { + json!({"protocol": self.protocol, "ports": [self.first, self.last]}) + } + + fn rendered(&self) -> Match { + let proto = if self.protocol == "tcp" { + L4Proto::Tcp + } else { + L4Proto::Udp + }; + Match::Dport(proto, self.first..=self.last) + } + } + + fn port() -> impl Strategy { + prop_oneof![Just(0), Just(1), Just(u16::MAX), any::()] + } + + fn raw() -> impl Strategy { + ( + prop_oneof![ + 8 => Just("tcp"), + 8 => Just("udp"), + 1 => Just("icmp"), + 1 => Just("TCP"), + 1 => Just(""), + ], + port(), + port(), + ) + .prop_map(|(protocol, first, last)| Raw { + protocol, + first, + last, + }) + } + + fn direction() -> impl Strategy> { + prop_oneof![ + 4 => prop::collection::vec(raw(), 0..8), + 1 => (Rules::MAX - 2..=Rules::MAX + 2).prop_flat_map(|len| { + prop::collection::vec( + (1u16..=u16::MAX).prop_map(|p| Raw { + protocol: "tcp", + first: p, + last: p, + }), + len, + ) + }), + ] + } + + #[proptest] + fn a_policy_is_accepted_exactly_when_every_rule_and_direction_is_valid( + #[strategy(direction())] inbound: Vec, + #[strategy(direction())] outbound_blocked: Vec, + ) { + let ok = |rules: &[Raw]| rules.len() <= Rules::MAX && rules.iter().all(Raw::valid); + let parsed = serde_json::from_value::(json!({ + "inbound": inbound.iter().map(Raw::json).collect::>(), + "outbound_blocked": outbound_blocked.iter().map(Raw::json).collect::>(), + })); + + prop_assert_eq!(parsed.is_ok(), ok(&inbound) && ok(&outbound_blocked)); + if let Ok(firewall) = parsed { + prop_assert_eq!( + firewall.inbound().collect::>(), + inbound.iter().map(Raw::rendered).collect::>() + ); + prop_assert_eq!( + firewall.outbound_blocked().collect::>(), + outbound_blocked + .iter() + .map(Raw::rendered) + .collect::>() + ); + } + } + + #[proptest] + fn ports_outside_u16_are_refused( + #[strategy(prop_oneof![-70_000i64..0, 65_536i64..200_000])] bad: i64, + first: bool, + ) { + let ports = if first { + json!([bad, 80]) + } else { + json!([80, bad]) + }; + let policy = json!({ + "inbound": [{"protocol": "tcp", "ports": ports}], + "outbound_blocked": [], + }); + + prop_assert!(serde_json::from_value::(policy).is_err()); + } +} diff --git a/native/suidhelper/src/net/vm.rs b/native/suidhelper/src/net/vm.rs index 83ed58dd..0ecd6c0a 100644 --- a/native/suidhelper/src/net/vm.rs +++ b/native/suidhelper/src/net/vm.rs @@ -564,6 +564,10 @@ mod tests { impl Lab { fn new(test: &str) -> Self { + Self::with_blocked(test, &Blocked::default()) + } + + fn with_blocked(test: &str, blocked: &Blocked) -> Self { let guest = ScratchNs::new(&format!("{test}-guest")); let vm = ScratchNs::new(&format!("{test}-vm")); let host = ScratchNs::new(&format!("{test}-host")); @@ -630,7 +634,7 @@ mod tests { GUEST_V6.parse().unwrap(), Uid::from_raw(FIRECRACKER), "198.51.100.1".parse().unwrap(), - &Blocked::default(), + blocked, &HOST_ADDRS.map(|addr| addr.parse().unwrap()), ), VmNet::firewall_table(&firewall), @@ -870,4 +874,253 @@ mod tests { assert_eq!(uplink.proxy_neighs().unwrap(), vec![live]); } + + mod hostile_guest { + //! A guest that forges, impersonates and probes, against the namespace's tables. + //! + //! Laws: + //! - a TCP segment whose source port's high byte reads as a neighbour-discovery + //! type is not neighbour discovery: it never reaches a namespace service; + //! - the namespace's link-local and uplink addresses serve the guest nothing; + //! - the DNS redirect never carries a forged source to the resolver; + //! - the jailed Firecracker reaches not even its own guest; + //! - with `[net] blocked` empty, the host's addresses and the transit range stay + //! unreachable from the guest; + //! - a router advertisement from the guest never reaches the namespace, even on a + //! `tap0` that would otherwise act on it. + + use super::{Lab, Probe, DROPPED_AFTER, FIRECRACKER, NAMESPACE}; + use crate::net::Blocked; + use pretty_assertions::assert_eq; + use rstest::rstest; + use socket2::{Domain, Protocol, Socket, Type}; + use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; + use std::process::Command; + use std::time::Duration; + + const SOLICIT_SPORT: u16 = 135 << 8; + const ADVERT_SPORT: u16 = 136 << 8; + const ROUTER_ADVERT: u8 = 134; + const RA_PREFIX: &str = "2001:db8:bad::"; + + fn skip_unless_root(test: &str) -> bool { + if nix::unistd::geteuid().is_root() { + return false; + } + eprintln!("SKIP {test}: needs root"); + true + } + + fn lab_name(parts: &str) -> String { + let mut name: String = parts + .to_lowercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect(); + name.truncate(40); + name + } + + fn eth0(lab: &Lab) -> u32 { + lab.guest + .get() + .rtnl() + .unwrap() + .link(&Lab::name("eth0")) + .unwrap() + .unwrap() + .index() + .get() + } + + /// Link-local destinations are scoped to the guest's `eth0`. + fn target(lab: &Lab, to: &str) -> SocketAddr { + let to: SocketAddr = to.parse().unwrap(); + match to { + SocketAddr::V6(v6) if v6.ip().is_unicast_link_local() => { + SocketAddrV6::new(*v6.ip(), v6.port(), 0, eth0(lab)).into() + } + to => to, + } + } + + fn guest_connects(lab: &Lab, src: Option<&str>, sport: u16, to: &str) -> bool { + let to = target(lab, to); + lab.guest + .get() + .scoped(|| { + let socket = Socket::new(Domain::for_address(to), Type::STREAM, None).unwrap(); + socket.set_reuse_address(true).unwrap(); + let src: IpAddr = match (src, to) { + (Some(src), _) => src.parse().unwrap(), + (None, SocketAddr::V4(_)) => IpAddr::from([0, 0, 0, 0]), + (None, SocketAddr::V6(_)) => Ipv6Addr::UNSPECIFIED.into(), + }; + socket.bind(&SocketAddr::new(src, sport).into()).unwrap(); + socket.connect_timeout(&to.into(), DROPPED_AFTER).is_ok() + }) + .unwrap() + } + + #[rstest] + #[case::solicit_lookalike_to_a_namespace_service( + None, + SOLICIT_SPORT, + "172.16.0.1:7074", + false + )] + #[case::advert_lookalike_to_a_namespace_service( + None, + ADVERT_SPORT, + "172.16.0.1:7074", + false + )] + #[case::solicit_lookalike_over_link_local(None, SOLICIT_SPORT, "[fe80::1]:7074", false)] + #[case::advert_lookalike_over_link_local(None, ADVERT_SPORT, "[fe80::1]:7074", false)] + #[case::link_local_to_a_namespace_service(None, 0, "[fe80::1]:7074", false)] + #[case::uplink_address_to_a_namespace_service(None, 0, "10.200.0.2:7074", false)] + #[case::forged_source_through_the_dns_redirect( + Some("192.0.2.9"), + 0, + "172.16.0.1:53", + false + )] + #[case::forged_ipv6_source_to_the_host( + Some("2001:db8::1:0:0:6"), + SOLICIT_SPORT, + "[2001:db8:ffff::1]:7071", + false + )] + #[case::lookalike_port_still_reaches_the_internet( + None, + SOLICIT_SPORT, + "198.51.100.1:7071", + true + )] + fn a_hostile_guest_reaches_only_what_policy_allows_as_root( + #[case] src: Option<&str>, + #[case] sport: u16, + #[case] to: &str, + #[case] reachable: bool, + ) { + if skip_unless_root("hostile guest") { + return; + } + let lab = Lab::new(&lab_name(&format!("hg-{src:?}-{sport}-{to}"))); + + assert_eq!(guest_connects(&lab, src, sport, to), reachable); + } + + #[rstest] + #[case::as_firecracker(FIRECRACKER, false)] + #[case::as_root(0, true)] + fn the_jailed_firecracker_cannot_reach_its_own_guest_as_root( + #[case] uid: u32, + #[case] reachable: bool, + ) { + if skip_unless_root("namespace to guest") { + return; + } + let lab = Lab::new(&format!("hg-fc-{uid}")); + + assert_eq!( + lab.connects(Probe::Namespace(uid), "172.16.0.2:7071".parse().unwrap()), + reachable + ); + } + + #[rstest] + #[case::the_host_veth("10.200.0.1:7071", false)] + #[case::a_host_address("198.51.100.2:7071", false)] + #[case::a_host_address_over_ipv6("[2001:db8:ffff::2]:7071", false)] + #[case::the_internet("198.51.100.1:7071", true)] + fn with_nothing_blocked_the_guest_still_never_reaches_the_host_as_root( + #[case] to: &str, + #[case] reachable: bool, + ) { + if skip_unless_root("empty blocked") { + return; + } + let lab = Lab::with_blocked( + &lab_name(&format!("hg-nb-{to}")), + &Blocked::try_from(Vec::new()).unwrap(), + ); + + assert_eq!(guest_connects(&lab, None, 0, to), reachable); + } + + /// An RA from `fe80::2` on `eth0` announcing `RA_PREFIX/64` and itself as a default + /// router. + fn send_router_advert(lab: &Lab) { + let index = eth0(lab); + lab.guest + .get() + .scoped(|| { + let socket = + Socket::new(Domain::IPV6, Type::RAW, Some(Protocol::ICMPV6)).unwrap(); + socket.set_multicast_hops_v6(255).unwrap(); + socket.set_unicast_hops_v6(255).unwrap(); + socket.set_multicast_if_v6(index).unwrap(); + let src = SocketAddrV6::new("fe80::2".parse().unwrap(), 0, 0, index); + socket.bind(&src.into()).unwrap(); + let mut ra = vec![ROUTER_ADVERT, 0, 0, 0, 64, 0]; + ra.extend_from_slice(&1800u16.to_be_bytes()); + ra.extend_from_slice(&[0; 8]); + ra.extend_from_slice(&[3, 4, 64, 0xc0]); + ra.extend_from_slice(&86_400u32.to_be_bytes()); + ra.extend_from_slice(&14_400u32.to_be_bytes()); + ra.extend_from_slice(&[0; 4]); + ra.extend_from_slice(&RA_PREFIX.parse::().unwrap().octets()); + let all_nodes = SocketAddrV6::new("ff02::1".parse().unwrap(), 0, 0, index); + socket.send_to(&ra, &all_nodes.into()).unwrap(); + }) + .unwrap(); + } + + fn namespace_routes(lab: &Lab) -> String { + let out = lab + .vm + .get() + .scoped(|| Command::new("ip").args(["-6", "route", "show"]).output()) + .unwrap() + .unwrap(); + String::from_utf8(out.stdout).unwrap() + } + + #[rstest] + fn a_router_advert_from_the_guest_never_reaches_the_namespace_as_root() { + if skip_unless_root("guest router advert") { + return; + } + let lab = Lab::new("hg-ra"); + lab.vm + .get() + .scoped(|| { + std::fs::write("/proc/sys/net/ipv6/conf/tap0/accept_ra", "2").unwrap(); + }) + .unwrap(); + + send_router_advert(&lab); + std::thread::sleep(Duration::from_secs(1)); + + let routes = namespace_routes(&lab); + assert!( + !routes.contains("proto ra") && !routes.contains(RA_PREFIX.trim_end_matches(':')), + "{routes}" + ); + } + + #[rstest] + fn a_namespace_service_is_reachable_from_the_namespace_itself_as_root() { + if skip_unless_root("namespace service control") { + return; + } + let lab = Lab::new("hg-ctl"); + + assert!(lab.connects( + Probe::Namespace(0), + format!("127.0.0.1:{NAMESPACE}").parse().unwrap() + )); + } + } } diff --git a/native/suidhelper/tests/caller_authorize.rs b/native/suidhelper/tests/caller_authorize.rs new file mode 100644 index 00000000..ada45274 --- /dev/null +++ b/native/suidhelper/tests/caller_authorize.rs @@ -0,0 +1,143 @@ +//! The caller gate ([`Caller::authorize`]) against an independent model of the #156 contract: +//! "privileged ops run only for root and members of the group the helper is installed with". +//! +//! Laws: +//! - oracle: a caller is authorized iff its uid is 0, or the install group is readable, is not +//! gid 0, and is the caller's primary or a supplementary gid; +//! - the refusal names the caller's uid and the install group, and a root install group or an +//! unreadable install refuses every non-root caller with exactly that cause; +//! - membership is a set property: reordering or repeating supplementary gids never changes +//! the verdict; +//! - root bypass is about the uid alone: gid 0 or membership of gid 0 grants nothing. + +use hyper_suidhelper::caller::{Caller, CallerError, NodeGroup}; +use nix::errno::Errno; +use nix::unistd::{Gid, Uid}; +use proptest::prelude::*; +use test_strategy::proptest; + +fn gid() -> impl Strategy { + prop_oneof![0u32..4, 990u32..994, any::()] +} + +fn installed() -> impl Strategy> { + prop_oneof![ + 8 => gid().prop_map(Ok), + 1 => prop_oneof![Just(Errno::EACCES), Just(Errno::ENOENT), Just(Errno::ELOOP)].prop_map(Err), + ] +} + +fn caller(uid: u32, gid: u32, groups: &[u32]) -> Caller { + Caller { + uid: Uid::from_raw(uid), + gid: Gid::from_raw(gid), + groups: groups.iter().copied().map(Gid::from_raw).collect(), + } +} + +fn group(installed: Result) -> Result { + installed + .map_err(CallerError::Stat) + .and_then(|g| Gid::from_raw(g).try_into()) +} + +fn model( + uid: u32, + gid: u32, + groups: &[u32], + installed: Result, +) -> Result<(), CallerError> { + if uid == 0 { + return Ok(()); + } + match installed { + Err(errno) => Err(CallerError::Stat(errno)), + Ok(0) => Err(CallerError::RootGroup), + Ok(g) if gid == g || groups.contains(&g) => Ok(()), + Ok(g) => Err(CallerError::NotInGroup { + uid: Uid::from_raw(uid), + group: Gid::from_raw(g), + }), + } +} + +#[proptest] +fn authorize_agrees_with_the_model( + #[strategy(prop_oneof![Just(0u32), 1u32..3, 1000u32..1003, any::()])] uid: u32, + #[strategy(gid())] gid: u32, + #[strategy(proptest::collection::vec(gid(), 0..40))] groups: Vec, + #[strategy(installed())] installed: Result, +) { + prop_assert_eq!( + caller(uid, gid, &groups).authorize(group(installed)), + model(uid, gid, &groups, installed) + ); +} + +#[proptest] +fn a_member_anywhere_in_a_long_group_list_is_authorized( + #[strategy(1u32..)] uid: u32, + #[strategy(1u32..)] node: u32, + #[strategy(proptest::collection::vec(any::(), 0..2_000))] others: Vec, + at: usize, +) { + let mut groups: Vec = others.into_iter().filter(|&g| g != node).collect(); + let primary = groups.pop().unwrap_or(node.wrapping_add(1).max(1)); + prop_assume!(primary != node); + prop_assert!(caller(uid, primary, &groups) + .authorize(group(Ok(node))) + .is_err()); + groups.insert(at % (groups.len() + 1), node); + prop_assert_eq!( + caller(uid, primary, &groups).authorize(group(Ok(node))), + Ok(()) + ); +} + +#[proptest] +fn reordering_or_repeating_supplementary_groups_never_changes_the_verdict( + #[strategy(1u32..)] uid: u32, + #[strategy(gid())] gid: u32, + #[strategy(proptest::collection::vec(gid(), 1..20))] groups: Vec, + #[strategy(gid())] node: u32, + rotate: usize, + repeat: usize, +) { + let verdict = caller(uid, gid, &groups).authorize(group(Ok(node))); + let mut shuffled = groups.clone(); + shuffled.rotate_left(rotate % groups.len()); + shuffled.reverse(); + shuffled.push(groups[repeat % groups.len()]); + prop_assert_eq!( + caller(uid, gid, &shuffled).authorize(group(Ok(node))), + verdict + ); +} + +#[proptest] +fn the_root_group_grants_a_non_root_uid_nothing( + #[strategy(1u32..)] uid: u32, + #[strategy(gid())] node: u32, +) { + let verdict = caller(uid, 0, &[0]).authorize(group(Ok(node))); + if node == 0 { + prop_assert_eq!(verdict, Err(CallerError::RootGroup)); + } else { + prop_assert_eq!( + verdict, + Err(CallerError::NotInGroup { + uid: Uid::from_raw(uid), + group: Gid::from_raw(node) + }) + ); + } +} + +#[proptest] +fn only_gid_zero_is_refused_as_an_install_group(raw: u32) { + let parsed = NodeGroup::try_from(Gid::from_raw(raw)); + prop_assert_eq!(parsed.is_err(), raw == 0); + if raw == 0 { + prop_assert_eq!(parsed, Err(CallerError::RootGroup)); + } +} diff --git a/native/suidhelper/tests/cli_request.rs b/native/suidhelper/tests/cli_request.rs new file mode 100644 index 00000000..ea1e086d --- /dev/null +++ b/native/suidhelper/tests/cli_request.rs @@ -0,0 +1,387 @@ +//! Hostile-request refusal laws for the helper's single JSON argument, the one input every +//! local caller controls before the privilege gate runs. Every valid request in +//! `fixtures/requests.json` is mutated at every leaf and every nested object; each law says the +//! mutated request is always refused as `RequestError::Json`, never accepted and never a crash: +//! +//! - nesting deeper than the parser's recursion limit, at the top level or in any field; +//! - any top-level value that is not an object; +//! - an `op` that is not byte-for-byte a known op (case, whitespace, separators, homoglyphs); +//! - a string field carrying NUL, ASCII whitespace, a traversal or a leading `-`, which would +//! split a dmsetup table or argv, escape a confined path, or read as a flag; +//! - an integer field given as a negative, a fraction, a string, a bool, a list or an exponent; +//! - a bool field given as anything but a JSON bool; +//! - an unknown key in any nested object, not just the top level, so a misspelled limit or +//! firewall key can never be silently dropped. + +use hyper_suidhelper::cli::{Request, RequestError}; +use proptest::prelude::*; +use rstest::rstest; +use serde_json::{json, Value}; +use std::ffi::OsString; +use test_strategy::proptest; + +const PARSER_DEPTH_LIMIT: usize = 128; +const MAX_ARG_STRLEN: usize = 128 * 1024; +const PLACEHOLDER: &str = "__hostile_placeholder__"; + +fn parse_raw(raw: &str) -> Result { + Request::from_args([OsString::from(raw)].into_iter()) +} + +fn parse(request: &Value) -> Result { + parse_raw(&request.to_string()) +} + +fn refused(raw: &str) -> bool { + matches!(parse_raw(raw), Err(RequestError::Json(_))) +} + +fn valid_requests() -> Vec { + let fixture: serde_json::Map = + serde_json::from_str(include_str!("fixtures/requests.json")).unwrap(); + fixture.into_iter().map(|(_, request)| request).collect() +} + +fn pointers(value: &Value, at: String, out: &mut Vec<(String, Value)>) { + out.push((at.clone(), value.clone())); + match value { + Value::Object(map) => { + for (key, child) in map { + pointers(child, format!("{at}/{key}"), out); + } + } + Value::Array(items) => { + for (idx, child) in items.iter().enumerate() { + pointers(child, format!("{at}/{idx}"), out); + } + } + _ => {} + } +} + +fn every_node() -> Vec<(Value, String, Value)> { + valid_requests() + .into_iter() + .flat_map(|request| { + let mut nodes = Vec::new(); + pointers(&request, String::new(), &mut nodes); + nodes + .into_iter() + .filter(|(at, _)| !at.is_empty()) + .map(move |(at, node)| (request.clone(), at, node)) + }) + .collect() +} + +fn replaced(request: &Value, at: &str, with: Value) -> Value { + let mut request = request.clone(); + *request.pointer_mut(at).unwrap() = with; + request +} + +fn op_node(at: &str) -> bool { + at == "/op" +} + +#[test] +fn every_fixture_request_is_accepted_unmutated() { + for request in valid_requests() { + assert!(parse(&request).is_ok(), "{request}"); + } +} + +#[rstest] +#[case::arrays("[", "]")] +#[case::objects("{\"k\":", "}")] +#[case::unclosed_arrays("[", "")] +fn nesting_past_the_parser_limit_is_refused_in_every_field( + #[case] open: &str, + #[case] close: &str, + #[values(PARSER_DEPTH_LIMIT + 1, 4_096, MAX_ARG_STRLEN / 8)] depth: usize, +) { + let deep = format!("{}0{}", open.repeat(depth), close.repeat(depth)); + let top_level = deep.clone(); + assert!(refused(&top_level), "top-level nesting depth {depth}"); + for (request, at, _) in every_node() { + if op_node(&at) { + continue; + } + let marked = replaced(&request, &at, json!(PLACEHOLDER)).to_string(); + let raw = marked.replace(&format!("\"{PLACEHOLDER}\""), &deep); + assert!(refused(&raw), "depth {depth} at {at} of {request}"); + } +} + +fn non_object() -> impl Strategy { + let leaf = prop_oneof![ + Just(Value::Null), + any::().prop_map(Value::from), + any::().prop_map(Value::from), + any::() + .prop_filter("finite", |f| f.is_finite()) + .prop_map(Value::from), + ".{0,16}".prop_map(Value::from), + ]; + let nested = leaf.clone().prop_recursive(3, 16, 4, |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..4).prop_map(Value::Array), + prop::collection::btree_map("op|id|[a-z]{1,4}", inner, 0..3) + .prop_map(|map| Value::Array(vec![Value::Object(map.into_iter().collect())])), + ] + }); + prop_oneof![leaf, nested] +} + +#[proptest] +fn a_top_level_value_that_is_not_an_object_is_refused(#[strategy(non_object())] request: Value) { + prop_assert!(refused(&request.to_string()), "{request}"); +} + +const OPS: &[&str] = &[ + "vm_launch", + "vm_claim_api_socket", + "vm_claim_vsock_socket", + "vm_teardown", + "vm_firewall", + "img_activate", + "dm_remove", + "thin_pool_create", + "thin_volume_create", + "thin_destroy", + "thin_fork_delta", + "host_init", + "version", +]; + +fn near_miss(op: &str, how: usize) -> String { + match how { + 0 => op.to_uppercase(), + 1 => format!(" {op}"), + 2 => format!("{op}\n"), + 3 => op.replace('_', "-"), + 4 => op.replacen('o', "\u{43e}", 1), + 5 => format!("{op}\0"), + 6 => op.replace('_', ""), + _ => { + let mut chars: Vec = op.chars().collect(); + chars[0] = chars[0].to_ascii_uppercase(); + chars.into_iter().collect() + } + } +} + +#[proptest] +fn an_op_that_is_not_exactly_a_known_op_is_refused( + #[strategy(0..OPS.len())] which: usize, + #[strategy(0usize..8)] how: usize, +) { + let canonical = valid_requests() + .into_iter() + .find(|request| request["op"] == OPS[which]) + .unwrap(); + let op = near_miss(OPS[which], how); + prop_assume!(op != OPS[which]); + let request = replaced(&canonical, "/op", json!(op)); + prop_assert!(refused(&request.to_string()), "{request}"); +} + +#[proptest] +fn an_arbitrary_op_name_is_refused(#[strategy("[ -~]{0,24}")] op: String) { + prop_assume!(!OPS.contains(&op.as_str())); + let request = json!({ "op": op }); + prop_assert!(refused(&request.to_string()), "{request}"); +} + +const SPLITTING_BYTES: &[&str] = &["\0", " ", "\t", "\n", "\r", "\u{b}", "\u{c}"]; + +fn spliced(original: &str, fragment: &str, cut: usize) -> String { + format!("{}{fragment}{}", &original[..cut], &original[cut..]) +} + +fn char_boundaries(s: &str) -> impl Iterator + '_ { + s.char_indices().map(|(idx, _)| idx).chain([s.len()]) +} + +fn traversals(original: &str) -> Vec { + let after_separators = original + .match_indices('/') + .map(|(idx, _)| spliced(original, "../", idx + 1)); + after_separators + .chain([ + format!("{original}/.."), + format!("../{original}"), + "..".to_owned(), + ]) + .collect() +} + +#[test] +fn a_splitting_byte_anywhere_in_any_string_field_is_refused() { + for (request, at, node) in every_node() { + let Value::String(original) = node else { + continue; + }; + for fragment in SPLITTING_BYTES { + for cut in char_boundaries(&original) { + let hostile = spliced(&original, fragment, cut); + let mutated = replaced(&request, &at, json!(hostile)); + assert!( + refused(&mutated.to_string()), + "accepted {hostile:?} at {at}: {mutated}" + ); + } + } + } +} + +#[test] +fn a_traversal_component_in_any_string_field_is_refused() { + for (request, at, node) in every_node() { + let Value::String(original) = node else { + continue; + }; + for hostile in traversals(&original) { + let mutated = replaced(&request, &at, json!(hostile)); + assert!( + refused(&mutated.to_string()), + "accepted {hostile:?} at {at}: {mutated}" + ); + } + } +} + +#[test] +fn a_leading_dash_on_any_string_field_is_refused() { + for (request, at, node) in every_node() { + let Value::String(original) = node else { + continue; + }; + for hostile in [ + format!("-{original}"), + format!("--{original}"), + "-".to_owned(), + ] { + let mutated = replaced(&request, &at, json!(hostile)); + assert!( + refused(&mutated.to_string()), + "accepted {hostile:?} at {at}" + ); + } + } +} + +#[test] +fn an_integer_field_in_any_other_json_shape_is_refused() { + let shapes = ["-1", "1.5", "\"1\"", "true", "1e3", "[1]", "{}"]; + for (request, at, node) in every_node() { + if !node.is_u64() { + continue; + } + for shape in shapes { + let marked = replaced(&request, &at, json!(PLACEHOLDER)).to_string(); + let raw = marked.replace(&format!("\"{PLACEHOLDER}\""), shape); + assert!(refused(&raw), "accepted {shape} at {at}: {raw}"); + } + } +} + +#[test] +fn a_bool_field_in_any_other_json_shape_is_refused() { + for (request, at, node) in every_node() { + if !node.is_boolean() { + continue; + } + for shape in [ + json!("true"), + json!(1), + json!(0), + Value::Null, + json!([true]), + ] { + let mutated = replaced(&request, &at, shape.clone()); + assert!(refused(&mutated.to_string()), "accepted {shape} at {at}"); + } + } +} + +#[test] +fn a_nested_object_or_list_in_the_wrong_shape_is_refused() { + for (request, at, node) in every_node() { + let wrong: &[Value] = match node { + Value::Object(_) => &[json!([]), json!("x"), json!(1)], + Value::Array(_) => &[json!({}), json!("x"), json!(1), Value::Null], + _ => continue, + }; + for shape in wrong { + let mutated = replaced(&request, &at, shape.clone()); + assert!( + refused(&mutated.to_string()), + "accepted {shape} at {at}: {mutated}" + ); + } + } +} + +#[proptest] +fn an_unknown_key_in_any_nested_object_is_refused( + #[strategy("[a-z_]{1,12}")] key: String, + #[strategy(any::())] which: prop::sample::Index, +) { + let objects: Vec<(Value, String, Value)> = every_node() + .into_iter() + .filter(|(_, _, node)| node.is_object()) + .collect(); + prop_assume!(!objects.is_empty()); + let (request, at, node) = which.get(&objects).clone(); + prop_assume!(!node.as_object().unwrap().contains_key(&key)); + let mut mutated = request.clone(); + mutated.pointer_mut(&at).unwrap()[&key] = json!(1); + prop_assert!( + refused(&mutated.to_string()), + "accepted {key} at {at}: {mutated}" + ); +} + +#[test] +fn every_nested_object_is_mutated_by_the_unknown_key_law() { + let nested: Vec = every_node() + .into_iter() + .filter(|(_, _, node)| node.is_object()) + .map(|(_, at, _)| at) + .collect(); + for expected in [ + "/limits", + "/limits/cpu_max", + "/firewall", + "/firewall/inbound/0", + ] { + assert!( + nested.iter().any(|at| at == expected), + "{expected} not covered: {nested:?}" + ); + } +} + +#[test] +fn a_misspelled_nested_key_never_silently_drops_a_limit() { + let launch = valid_requests() + .into_iter() + .find(|request| request["op"] == "vm_launch") + .unwrap(); + for (at, typo) in [ + ("/limits", json!({"memroy_max": 1})), + ( + "/limits", + json!({"memory_max": 1, "cpu": {"quota_us": 1, "period_us": 1}}), + ), + ("/limits/cpu_max", json!({"quota_us": 1, "period": 1})), + ("/firewall", json!({"inbound": [], "outbound_block": []})), + ( + "/firewall/inbound/0", + json!({"protocol": "tcp", "port": [22, 22]}), + ), + ] { + let mutated = replaced(&launch, at, typo.clone()); + assert!(refused(&mutated.to_string()), "accepted {typo} at {at}"); + } +} diff --git a/native/suidhelper/tests/e2e/caller_gate.rs b/native/suidhelper/tests/e2e/caller_gate.rs new file mode 100644 index 00000000..a03ea7f3 --- /dev/null +++ b/native/suidhelper/tests/e2e/caller_gate.rs @@ -0,0 +1,265 @@ +//! The #156 caller gate, end to end through the built binary: privileged ops run only for root +//! and members of the group the helper file belongs to. +//! +//! Each case installs a copy of the helper `root:` mode 0711, so a non-root caller cannot +//! read it and the kernel execs it non-dumpable, as it would a setuid binary, and runs it under an +//! exact uid, primary gid and supplementary list. The probe op is `vm_claim_api_socket`: once +//! past the gate its first act is raising euid, which a non-setuid copy cannot do, so a caller +//! the gate let through fails with `raising euid` and touches nothing. +//! +//! Laws: +//! - a non-root caller passes iff the install group is its primary gid or anywhere among its +//! supplementary gids; +//! - the refusal comes before the config is opened: a refused caller learns nothing about it, +//! while an admitted caller is the one who meets the config checks; +//! - `version` answers anyone and never reads the config; +//! - the gate keys on the inode that runs, not the path it was invoked by; +//! - root is admitted whatever its gids, even beside a root install group. +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use nix::unistd::{chown, setgroups, setresgid, setresuid, Gid, Uid}; +use pretty_assertions::assert_eq; +use proptest::prelude::*; +use rstest::rstest; +use serde_json::{json, Value}; +use std::fs; +use std::os::unix::fs::{symlink, PermissionsExt}; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use test_strategy::proptest; + +mod support; +use support::{is_root, BIN}; + +const NOBODY: u32 = 65_534; +const NODE: u32 = 54_321; +const PROBE: &str = r#"{"op":"vm_claim_api_socket","id":"vgate"}"#; +const PAST_THE_GATE: &str = "raising euid to root"; +const NOT_IN_GROUP: &str = "is not in the helper's group"; + +struct Install { + dir: TempDir, + bin: PathBuf, +} + +impl Install { + fn new(group: u32) -> Self { + let dir = TempDir::new().unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let bin = dir.path().join("hyper-suidhelper"); + fs::copy(BIN, &bin).unwrap(); + chown(&bin, Some(Uid::from_raw(0)), Some(Gid::from_raw(group))).unwrap(); + fs::set_permissions(&bin, fs::Permissions::from_mode(0o711)).unwrap(); + Self { dir, bin } + } + + fn config(&self, body: &str, owner: u32) -> PathBuf { + let path = self.dir.path().join("config.toml"); + fs::write(&path, body).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + chown( + &path, + Some(Uid::from_raw(owner)), + Some(Gid::from_raw(owner)), + ) + .unwrap(); + path + } + + fn absent_config(&self) -> PathBuf { + self.dir.path().join("absent.toml") + } +} + +fn run_as(bin: &Path, config: &Path, uid: u32, gid: u32, groups: &[u32], request: &str) -> Output { + let groups: Vec = groups.iter().copied().map(Gid::from_raw).collect(); + let mut cmd = Command::new(bin); + cmd.arg(request) + .env_clear() + .env("HYPER_SETUIDHELPER_IS_INSECURE_MODE", "1") + .env("HYPER_SETUIDHELPER_CONFIG_PATH", config) + .current_dir("/"); + // SAFETY: the hook runs in the forked child before exec and only makes id syscalls. + unsafe { + cmd.pre_exec(move || { + let (gid, uid) = (Gid::from_raw(gid), Uid::from_raw(uid)); + setgroups(&groups)?; + setresgid(gid, gid, gid)?; + setresuid(uid, uid, uid)?; + Ok(()) + }); + } + cmd.output().unwrap() +} + +fn stderr(out: &Output) -> String { + String::from_utf8_lossy(&out.stderr).into_owned() +} + +fn gids() -> impl Strategy { + prop_oneof![ + Just(0u32), + Just(NOBODY), + 1u32..60_000, + 60_000u32..4_000_000_000 + ] + .prop_filter("not the node group", |&g| g != NODE) +} + +#[proptest(cases = 24)] +fn a_caller_passes_iff_it_holds_the_install_group_as_root( + #[strategy(gids())] primary: u32, + #[strategy(proptest::collection::vec(gids(), 0..64))] mut groups: Vec, + member: bool, + at: usize, +) { + if !is_root() { + eprintln!("SKIP caller gate property: needs root to install the helper and switch ids"); + return Ok(()); + } + if member { + groups.insert(at % (groups.len() + 1), NODE); + } + let install = Install::new(NODE); + + let out = run_as( + &install.bin, + &install.absent_config(), + NOBODY, + primary, + &groups, + PROBE, + ); + + let stderr = stderr(&out); + prop_assert_eq!(out.status.code(), Some(2), "{}", stderr); + if member { + prop_assert!(stderr.contains(PAST_THE_GATE), "member refused: {}", stderr); + } else { + prop_assert!( + stderr.contains(NOT_IN_GROUP), + "non-member admitted: {}", + stderr + ); + prop_assert!( + stderr.contains(&format!("uid {NOBODY} ")) && stderr.contains(&NODE.to_string()), + "{}", + stderr + ); + } +} + +const UNTRUSTED_CONFIG: &str = "not owned by root:root"; + +#[rstest] +#[case::a_non_member_is_refused_without_seeing_the_config(&[], NOT_IN_GROUP, UNTRUSTED_CONFIG)] +#[case::a_member_is_the_one_who_meets_the_config_check(&[NODE], UNTRUSTED_CONFIG, PAST_THE_GATE)] +fn the_gate_refuses_before_the_config_is_opened_as_root( + #[case] groups: &[u32], + #[case] refusal: &str, + #[case] unreached: &str, +) { + if !is_root() { + eprintln!("SKIP gate-before-config: needs root"); + return; + } + let install = Install::new(NODE); + let untrusted = install.config("work_dir = \"/srv/hyper\"\n", NOBODY); + + let out = run_as(&install.bin, &untrusted, NOBODY, NOBODY, groups, PROBE); + + let stderr = stderr(&out); + assert_eq!(out.status.code(), Some(2), "{stderr}"); + assert!(stderr.contains(refusal), "{stderr}"); + assert!(!stderr.contains(unreached), "{stderr}"); +} + +#[rstest] +#[case::untrusted_config(true)] +#[case::malformed_config(false)] +fn version_answers_a_non_member_without_reading_the_config_as_root(#[case] untrusted: bool) { + if !is_root() { + eprintln!("SKIP version for non-members: needs root"); + return; + } + let install = Install::new(NODE); + let config = if untrusted { + install.config("work_dir = \"/srv/hyper\"\n", NOBODY) + } else { + install.config("this is = not = toml", 0) + }; + + let out = run_as( + &install.bin, + &config, + NOBODY, + NOBODY, + &[], + r#"{"op":"version"}"#, + ); + + assert_eq!(out.status.code(), Some(0), "{}", stderr(&out)); + let report: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(report["version"], env!("CARGO_PKG_VERSION")); +} + +#[rstest] +#[case::member(&[NODE], PAST_THE_GATE)] +#[case::non_member(&[], NOT_IN_GROUP)] +fn the_gate_keys_on_the_running_inode_not_the_invoked_path_as_root( + #[case] groups: &[u32], + #[case] expected: &str, +) { + if !is_root() { + eprintln!("SKIP gate via symlink: needs root"); + return; + } + let install = Install::new(NODE); + let elsewhere = TempDir::new().unwrap(); + fs::set_permissions(elsewhere.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let link = elsewhere.path().join("hyper-suidhelper"); + symlink(&install.bin, &link).unwrap(); + chown( + elsewhere.path(), + Some(Uid::from_raw(NOBODY)), + Some(Gid::from_raw(NOBODY)), + ) + .unwrap(); + + let out = run_as( + &link, + &install.absent_config(), + NOBODY, + NOBODY, + groups, + PROBE, + ); + + assert!(stderr(&out).contains(expected), "{}", stderr(&out)); +} + +#[rstest] +#[case::root_install_group(0)] +#[case::node_install_group(NODE)] +fn root_is_admitted_whatever_its_gids_as_root(#[case] group: u32) { + if !is_root() { + eprintln!("SKIP root bypass: needs root"); + return; + } + let install = Install::new(group); + + let out = run_as( + &install.bin, + &install.absent_config(), + 0, + NOBODY, + &[NOBODY], + PROBE, + ); + + assert_eq!(out.status.code(), Some(0), "{}", stderr(&out)); + let reply: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(reply, json!({"result": "pending"})); +} diff --git a/native/suidhelper/tests/e2e/claim_privacy.rs b/native/suidhelper/tests/e2e/claim_privacy.rs new file mode 100644 index 00000000..49bf09c4 --- /dev/null +++ b/native/suidhelper/tests/e2e/claim_privacy.rs @@ -0,0 +1,298 @@ +//! `vm_claim_*_socket` through a real setuid install, called by a non-root member of the helper's +//! group the way the node calls it (#135: "keep VM sockets private"). +//! +//! Laws: +//! - the pin is the jail socket's own inode, handed to the *calling* user and group (never to the +//! effective root the setuid bit grants), mode `0600`; +//! - once claimed, only that caller and root can `connect()` to the VM's API or vsock socket, by +//! the pin or by the jail path: another member of the helper's group, another VM's jail uid and +//! `nobody` are all refused `EACCES` by the socket itself, even through a traversable dir; +//! - a socket dir that is a symlink is refused (`ELOOP`) by both claim and teardown, and nothing +//! is linked into or unlinked from the symlink's target; +//! - a socket dir that is not a directory is a refusal, never a crash. +#![cfg(feature = "insecure_mode")] + +use nix::unistd::{Gid, Uid}; +use pretty_assertions::assert_eq; +use rstest::rstest; +use serde_json::json; +use std::fs; +use std::os::unix::fs::{symlink, FileTypeExt, MetadataExt, PermissionsExt}; +use std::os::unix::net::UnixListener; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::Command; + +mod setuid_support; +mod support; +use setuid_support::{run_as, Caller, Install, MEMBER, OTHER_MEMBER, ROOT}; +use support::is_root; + +const ID: &str = "v1"; + +const SOCKETS: [(&str, &str); 2] = [ + ("vm_claim_api_socket", "api.socket"), + ("vm_claim_vsock_socket", "vsock.sock"), +]; + +const OTHER_VMS_JAIL: Caller = Caller { + uid: 900_003, + gid: 900_003, + groups: &[], +}; + +const NOBODY: Caller = Caller { + uid: 65_534, + gid: 65_534, + groups: &[], +}; + +struct Vm { + install: Install, + config: PathBuf, + srv: PathBuf, + _listeners: Vec, +} + +impl Vm { + fn new() -> Self { + let install = Install::new(); + let srv = install.path().join("srv"); + let root = srv.join("jails/firecracker").join(ID).join("root"); + fs::create_dir_all(&root).unwrap(); + let socks = srv.join("socks"); + fs::create_dir_all(&socks).unwrap(); + chown(&socks, MEMBER); + fs::set_permissions(&socks, fs::Permissions::from_mode(0o755)).unwrap(); + let listeners = SOCKETS + .iter() + .map(|(_, socket)| { + let path = root.join(socket); + let listener = UnixListener::bind(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o777)).unwrap(); + listener + }) + .collect(); + let config = install.config(&srv); + Self { + install, + config, + srv, + _listeners: listeners, + } + } + + fn jail_socket(&self, socket: &str) -> PathBuf { + self.srv + .join("jails/firecracker") + .join(ID) + .join("root") + .join(socket) + } + + fn pin(&self, socket: &str) -> PathBuf { + self.srv.join("socks").join(format!("fc-{ID}-{socket}")) + } +} + +fn chown(path: &Path, caller: Caller) { + nix::unistd::chown( + path, + Some(Uid::from_raw(caller.uid)), + Some(Gid::from_raw(caller.gid)), + ) + .unwrap(); +} + +/// `connect()` to `path` as `caller`: `0` on success, otherwise the errno. +fn connect_as(caller: Caller, path: &Path) -> i32 { + let Caller { uid, gid, groups } = caller; + let mut cmd = Command::new("python3"); + cmd.args([ + "-I", + "-c", + "import socket,sys\ns=socket.socket(socket.AF_UNIX)\ntry:\n s.connect(sys.argv[1]); print(0)\nexcept OSError as e:\n print(e.errno)", + ]) + .arg(path) + .env_clear(); + // SAFETY: the hook runs in the forked child before exec and makes only raw id syscalls. + unsafe { + cmd.pre_exec(move || { + let groups: Vec = groups.iter().copied().map(Gid::from_raw).collect(); + nix::unistd::setgroups(&groups)?; + nix::unistd::setresgid(Gid::from_raw(gid), Gid::from_raw(gid), Gid::from_raw(gid))?; + nix::unistd::setresuid(Uid::from_raw(uid), Uid::from_raw(uid), Uid::from_raw(uid))?; + Ok(()) + }) + }; + let out = cmd.output().unwrap(); + String::from_utf8_lossy(&out.stdout) + .trim() + .parse() + .unwrap_or_else(|_| panic!("connect probe failed: {out:?}")) +} + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn a_member_claim_hands_the_socket_to_the_caller_alone_as_root(#[case] (op, socket): (&str, &str)) { + if !is_root() { + eprintln!("SKIP setuid claim: needs root to install setuid"); + return; + } + let vm = Vm::new(); + + let out = run_as( + &vm.install, + &vm.config, + MEMBER, + &json!({"op": op, "id": ID}), + ); + + let pin = fs::symlink_metadata(vm.pin(socket)).unwrap(); + let jail = fs::symlink_metadata(vm.jail_socket(socket)).unwrap(); + assert_eq!( + ( + out.code, + out.stdout.trim(), + pin.file_type().is_socket(), + (pin.dev(), pin.ino()), + (pin.uid(), pin.gid()), + pin.mode() & 0o7777, + ), + ( + Some(0), + r#"{"result":"claimed"}"#, + true, + (jail.dev(), jail.ino()), + (MEMBER.uid, MEMBER.gid), + 0o600, + ), + "{}", + out.stderr + ); +} + +const EACCES: i32 = 13; + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn once_claimed_only_the_caller_and_root_can_connect_as_root(#[case] (op, socket): (&str, &str)) { + if !is_root() { + eprintln!("SKIP setuid claim privacy: needs root to install setuid"); + return; + } + let vm = Vm::new(); + run_as( + &vm.install, + &vm.config, + MEMBER, + &json!({"op": op, "id": ID}), + ); + + let callers = [ + ("caller", MEMBER), + ("root", ROOT), + ("other member", OTHER_MEMBER), + ("other VM's jail uid", OTHER_VMS_JAIL), + ("nobody", NOBODY), + ]; + let got: Vec<(&str, i32, i32)> = callers + .iter() + .map(|&(who, caller)| { + ( + who, + connect_as(caller, &vm.pin(socket)), + connect_as(caller, &vm.jail_socket(socket)), + ) + }) + .collect(); + + assert_eq!( + got, + vec![ + ("caller", 0, 0), + ("root", 0, 0), + ("other member", EACCES, EACCES), + ("other VM's jail uid", EACCES, EACCES), + ("nobody", EACCES, EACCES), + ] + ); +} + +#[rstest] +#[case::claim_api(json!({"op": "vm_claim_api_socket", "id": ID}))] +#[case::claim_vsock(json!({"op": "vm_claim_vsock_socket", "id": ID}))] +#[case::teardown(json!({"op": "vm_teardown", "id": ID}))] +fn a_symlinked_socket_dir_is_refused_and_its_target_untouched_as_root( + #[case] request: serde_json::Value, +) { + if !is_root() { + eprintln!("SKIP symlinked socket dir: needs root to install setuid"); + return; + } + let vm = Vm::new(); + let elsewhere = vm.install.path().join("elsewhere"); + fs::create_dir(&elsewhere).unwrap(); + chown(&elsewhere, MEMBER); + let decoy = elsewhere.join(format!("fc-{ID}-vsock.sock")); + fs::write(&decoy, b"decoy").unwrap(); + fs::remove_dir(vm.srv.join("socks")).unwrap(); + symlink(&elsewhere, vm.srv.join("socks")).unwrap(); + + let out = run_as(&vm.install, &vm.config, MEMBER, &request); + + let mut left: Vec = fs::read_dir(&elsewhere) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + assert_eq!( + ( + out.code, + out.stderr.contains("ELOOP"), + left, + fs::read(&decoy).unwrap() + ), + ( + Some(2), + true, + vec![format!("fc-{ID}-vsock.sock")], + b"decoy".to_vec() + ), + "{request}: {}", + out.stderr + ); +} + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn a_socket_dir_that_is_a_file_is_refused_not_crashed_as_root(#[case] (op, _): (&str, &str)) { + if !is_root() { + eprintln!("SKIP socket dir file: needs root to install setuid"); + return; + } + let vm = Vm::new(); + fs::remove_dir(vm.srv.join("socks")).unwrap(); + fs::write(vm.srv.join("socks"), b"not a dir").unwrap(); + + let out = run_as( + &vm.install, + &vm.config, + MEMBER, + &json!({"op": op, "id": ID}), + ); + + assert_eq!( + ( + out.code, + out.stdout.as_str(), + out.stderr.contains("socket dir") + ), + (Some(2), "", true), + "{}", + out.stderr + ); +} diff --git a/native/suidhelper/tests/e2e/fork_delta_live.rs b/native/suidhelper/tests/e2e/fork_delta_live.rs new file mode 100644 index 00000000..0d9436c9 --- /dev/null +++ b/native/suidhelper/tests/e2e/fork_delta_live.rs @@ -0,0 +1,314 @@ +//! `thin_fork_delta` against a real dm-thin pool, driven through the helper +//! binary. The oracle is the parent itself: replaying the produced COW store +//! as a dm-snapshot over the origin must read back byte-for-byte as the parent. +//! Laws: +//! * for ANY set of whole- and partial-block writes over a non-zero origin +//! (including none at all), replay == parent; +//! * nothing outlives a call: no `hyper-scratch-*`/`hyper-copy-*` device, +//! the scratch thin id is free for reuse, the metadata snapshot is +//! released, and the parent is still live and unchanged; +//! * `written_chunks` is zero exactly when nothing diverged and never more +//! than the number of diverged blocks (each block fits one copy chunk); +//! * two forks of different parents of one pool racing each other both +//! produce their own parent's exact divergence. +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use pretty_assertions::assert_eq; +use proptest::prelude::*; +use proptest::strategy::ValueTree; +use proptest::test_runner::{Config as RunnerConfig, TestRunner}; +use serde_json::json; +use std::collections::BTreeSet; +use std::fs; +use std::io::{Seek, SeekFrom, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; + +mod support; +use support::{is_root, run, stdout_json, DmDevice, LoopDevice, MapperDevice}; + +const BLOCK_SECTORS: u64 = 128; +const BLOCK_BYTES: u64 = BLOCK_SECTORS * 512; +const BLOCKS: u64 = 32; +const SIZE: u64 = BLOCKS * BLOCK_BYTES; +const SCRATCH_ID: u32 = 900; + +fn system_bin(name: &str) -> Option { + let out = Command::new("sh") + .args(["-c", &format!("command -v {name}")]) + .output() + .ok()?; + fs::canonicalize(String::from_utf8(out.stdout).ok()?.trim()).ok() +} + +fn sparse(path: &Path, bytes: u64) { + fs::File::create(path).unwrap().set_len(bytes).unwrap(); +} + +struct Live { + origin: MapperDevice, + pool: DmDevice, + _data: LoopDevice, + meta: LoopDevice, + config: PathBuf, + base: PathBuf, + _dir: TempDir, + tag: String, +} + +impl Live { + fn new(tag: &str) -> Option { + let (dmsetup, thin_dump) = (system_bin("dmsetup")?, system_bin("thin_dump")?); + let dir = TempDir::new().unwrap(); + let base = fs::canonicalize(dir.path()).unwrap(); + let thin_dump_copy = base.join("thin_dump"); + fs::copy(&thin_dump, &thin_dump_copy).unwrap(); + fs::set_permissions(&thin_dump_copy, fs::Permissions::from_mode(0o755)).unwrap(); + let config = base.join("config.toml"); + fs::write( + &config, + format!( + "work_dir = \"{}\"\n[tools]\ndmsetup = \"{}\"\nthin_dump = \"{}\"\n", + base.display(), + dmsetup.display(), + thin_dump_copy.display() + ), + ) + .unwrap(); + fs::set_permissions(&config, fs::Permissions::from_mode(0o644)).unwrap(); + + sparse(&base.join("meta.img"), 8 << 20); + sparse(&base.join("data.img"), 64 << 20); + let origin_bytes: Vec = (0..SIZE).map(|i| (i % 241) as u8 | 0x80).collect(); + fs::write(base.join("origin.img"), origin_bytes).unwrap(); + let meta = LoopDevice::attach(&base.join("meta.img"))?; + let data = LoopDevice::attach(&base.join("data.img"))?; + let tag = format!("hyper-e2e-fdl-{tag}-{}", std::process::id()); + let pool = DmDevice::create( + &format!("{tag}-pool"), + &format!( + "0 {} thin-pool {} {} {BLOCK_SECTORS} 0", + (64u64 << 20) / 512, + meta.path(), + data.path() + ), + )?; + let origin = MapperDevice::linear(&format!("{tag}-origin"), &base.join("origin.img"))?; + Some(Self { + origin, + pool, + _data: data, + meta, + config, + base, + _dir: dir, + tag, + }) + } + + fn pool_name(&self) -> String { + format!("{}-pool", self.tag) + } + + fn parent(&self, id: u32) -> DmDevice { + assert!(self.pool.message(&format!("create_thin {id}"))); + DmDevice::create( + &format!("{}-parent-{id}", self.tag), + &format!( + "0 {} thin {} {id} {}", + SIZE / 512, + self.pool.path(), + self.origin.path() + ), + ) + .expect("parent thin") + } + + fn fork(&self, parent: &DmDevice, scratch_id: u32, cow: &Path) -> assert_cmd::assert::Assert { + sparse(cow, 8 << 20); + run( + &self.config, + json!({ + "op": "thin_fork_delta", + "pool": self.pool_name(), + "meta": self.meta.path(), + "parent": parent.path().trim_start_matches("/dev/mapper/"), + "scratch_id": scratch_id, + "origin": self.origin.path(), + "cow": cow, + "chunk_sectors": 8, + }), + ) + } + + fn replay(&self, cow: &Path) -> Vec { + let lo = LoopDevice::attach(cow).expect("cow loop"); + let replay = DmDevice::create( + &format!("{}-replay", self.tag), + &format!( + "0 {} snapshot {} {} P 8", + SIZE / 512, + self.origin.path(), + lo.path() + ), + ) + .expect("the cow store replays as a snapshot"); + fs::read(replay.path()).unwrap() + } + + fn assert_nothing_outlived(&self, scratch_id: u32) { + let listed = Command::new("dmsetup").arg("ls").output().unwrap(); + let listed = String::from_utf8_lossy(&listed.stdout); + for prefix in ["hyper-scratch-", "hyper-copy-"] { + let leftover = format!("{prefix}{scratch_id}"); + assert!( + !listed + .lines() + .any(|l| l.split_whitespace().next() == Some(&leftover)), + "{leftover} outlived the call:\n{listed}" + ); + } + assert!( + self.pool.message("reserve_metadata_snap"), + "the metadata snapshot was left reserved" + ); + assert!(self.pool.message("release_metadata_snap")); + assert!( + self.pool.message(&format!("create_thin {scratch_id}")), + "scratch id {scratch_id} was not freed" + ); + assert!(self.pool.message(&format!("delete {scratch_id}"))); + } +} + +#[derive(Debug, Clone)] +enum Divergence { + Block { + block: u64, + fill: u8, + }, + Partial { + block: u64, + offset: u64, + len: u64, + fill: u8, + }, +} + +impl Divergence { + fn block(&self) -> u64 { + match *self { + Divergence::Block { block, .. } | Divergence::Partial { block, .. } => block, + } + } + + fn apply(&self, dev: &str) { + let (at, len, fill) = match *self { + Divergence::Block { block, fill } => (block * BLOCK_BYTES, BLOCK_BYTES, fill), + Divergence::Partial { + block, + offset, + len, + fill, + } => (block * BLOCK_BYTES + offset * 512, len * 512, fill), + }; + let mut f = fs::OpenOptions::new().write(true).open(dev).unwrap(); + f.seek(SeekFrom::Start(at)).unwrap(); + f.write_all(&vec![fill; len as usize]).unwrap(); + f.sync_all().unwrap(); + } +} + +fn writes() -> impl Strategy> { + let one = prop_oneof![ + (0..BLOCKS, any::()).prop_map(|(block, fill)| Divergence::Block { block, fill }), + (0..BLOCKS, 0u64..BLOCK_SECTORS, any::()).prop_flat_map(|(block, offset, fill)| { + (1..=BLOCK_SECTORS - offset).prop_map(move |len| Divergence::Partial { + block, + offset, + len, + fill, + }) + }), + ]; + prop::collection::vec(one, 0..10) +} + +macro_rules! require { + ($live:expr) => { + match (is_root(), $live) { + (true, Some(live)) => live, + _ => { + eprintln!("SKIP: needs root, dmsetup, thin_dump and dm-thin"); + return; + } + } + }; +} + +#[test] +fn fork_delta_replays_any_divergence_exactly_and_leaves_nothing_behind_as_root() { + let live = require!(Live::new("prop")); + let mut runner = TestRunner::new(RunnerConfig { + cases: 8, + failure_persistence: None, + ..RunnerConfig::default() + }); + let mut cases: Vec> = vec![Vec::new()]; + for _ in 0..7 { + cases.push(writes().new_tree(&mut runner).unwrap().current()); + } + + for (n, case) in cases.iter().enumerate() { + let parent = live.parent(10 + n as u32); + for w in case { + w.apply(&parent.path()); + } + let before = fs::read(parent.path()).unwrap(); + let cow = live.base.join(format!("cow-{n}.img")); + + let out = live.fork(&parent, SCRATCH_ID, &cow).success(); + + let diverged: BTreeSet = case.iter().map(Divergence::block).collect(); + let written = stdout_json(&out)["written_chunks"].as_u64().unwrap(); + assert_eq!(written == 0, diverged.is_empty(), "{case:?}: {written}"); + assert!(written <= diverged.len() as u64, "{case:?}: {written}"); + assert!( + live.replay(&cow) == before, + "replay differs from the parent: {case:?}" + ); + assert!( + fs::read(parent.path()).unwrap() == before, + "the fork changed the parent: {case:?}" + ); + live.assert_nothing_outlived(SCRATCH_ID); + } +} + +#[test] +fn racing_forks_of_two_parents_each_get_their_own_divergence_as_root() { + let live = require!(Live::new("race")); + let (a, b) = (live.parent(20), live.parent(21)); + for block in [0, 7, BLOCKS - 1] { + Divergence::Block { block, fill: 0x0a }.apply(&a.path()); + } + for block in [3, 7, 19] { + Divergence::Block { block, fill: 0x0b }.apply(&b.path()); + } + let (want_a, want_b) = (fs::read(a.path()).unwrap(), fs::read(b.path()).unwrap()); + let (cow_a, cow_b) = (live.base.join("cow-a.img"), live.base.join("cow-b.img")); + + std::thread::scope(|s| { + let ta = s.spawn(|| live.fork(&a, 901, &cow_a).try_success().is_ok()); + let tb = s.spawn(|| live.fork(&b, 902, &cow_b).try_success().is_ok()); + assert!(ta.join().unwrap(), "fork of a failed under contention"); + assert!(tb.join().unwrap(), "fork of b failed under contention"); + }); + + assert!(live.replay(&cow_a) == want_a, "a's replay is not a"); + assert!(live.replay(&cow_b) == want_b, "b's replay is not b"); + live.assert_nothing_outlived(901); + live.assert_nothing_outlived(902); +} diff --git a/native/suidhelper/tests/e2e/jailer.rs b/native/suidhelper/tests/e2e/jailer.rs index 2bf96344..f70ab22d 100644 --- a/native/suidhelper/tests/e2e/jailer.rs +++ b/native/suidhelper/tests/e2e/jailer.rs @@ -417,3 +417,96 @@ fn teardown_waits_for_the_vm_lock_an_in_flight_launch_holds_as_root(launch: Opti String::from_utf8_lossy(&out.stderr) ); } + +fn vm_lock_dir() -> SafeDir { + let locks = PathBuf::from("/run/hyper-suidhelper/locks"); + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&locks) + .unwrap(); + fs::set_permissions("/run/hyper-suidhelper", fs::Permissions::from_mode(0o755)).unwrap(); + let locks: SafePathBuf = locks.try_into().unwrap(); + SafeFile::open(&locks, OpenFlags::RDONLY).unwrap() +} + +#[rstest] +fn launch_waits_for_a_teardown_holding_the_vm_lock_before_touching_the_jail_as_root( + launch: Option, +) { + let launch = require!(launch); + let cfg = launch.config(""); + let stale = launch.plant_stale_jail(); + let dir = vm_lock_dir(); + let name = format!("vm-{}", launch.id); + let held = FileLock::acquire( + &dir, + name.as_str().try_into().unwrap(), + Duration::from_secs(5), + ) + .unwrap(); + + let request = launch.request(json!({})).to_string(); + let mut child = launch + .ns + .get() + .scoped(|| { + support::command(&cfg, &[&request]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() + }) + .unwrap(); + thread::sleep(Duration::from_millis(1500)); + let while_held = ( + child.try_wait().unwrap().is_none(), + stale.exists(), + launch.recorded_argv().is_none(), + ); + drop(held); + let out = child.wait_with_output().unwrap(); + + assert_eq!( + ( + while_held, + out.status.code(), + stale.exists(), + launch.recorded_argv().is_some() + ), + ((true, true, true), Some(RECORDER_EXIT), false, true), + "{}", + String::from_utf8_lossy(&out.stderr) + ); +} + +#[rstest] +fn a_launch_that_never_gets_the_vm_lock_stages_nothing_and_never_execs_as_root( + launch: Option, +) { + let launch = require!(launch); + let cfg = launch.config(""); + let stale = launch.plant_stale_jail(); + let dir = vm_lock_dir(); + let name = format!("vm-{}", launch.id); + let _held = FileLock::acquire( + &dir, + name.as_str().try_into().unwrap(), + Duration::from_secs(5), + ) + .unwrap(); + + let out = launch.run(&cfg, launch.request(json!({}))).failure(); + + assert_eq!( + ( + out.get_output().status.code() == Some(RECORDER_EXIT), + stale.exists(), + launch.recorded_argv().is_none(), + Path::new("/run/netns").join(launch.netns()).exists(), + ), + (false, true, true, false), + "{}", + String::from_utf8_lossy(&out.get_output().stderr) + ); +} diff --git a/native/suidhelper/tests/e2e/lock_dir_hostile.rs b/native/suidhelper/tests/e2e/lock_dir_hostile.rs new file mode 100644 index 00000000..6fe66bbc --- /dev/null +++ b/native/suidhelper/tests/e2e/lock_dir_hostile.rs @@ -0,0 +1,172 @@ +//! The per-VM lock that serializes `vm_launch` and `vm_teardown` (#135) lives in the fixed +//! `/run/hyper-suidhelper/locks`. Whoever controls that directory controls the serialization, so +//! the helper must only ever lock inside a directory chain only root can write. Driven through a +//! real setuid install by a non-root member of its group, each case in a private mount namespace +//! whose `/run` is a fresh tmpfs, so the host's own lock dir is never touched. +//! +//! Laws: +//! - on an empty `/run`, the first member caller (even with umask 000) creates +//! `/run/hyper-suidhelper` as `root:root 0755` and `locks` as `root:root 0700`, and a teardown +//! leaves no lock file behind; +//! - a `/run/hyper-suidhelper` or `locks` that is not root-owned, is group- or world-writable, is a +//! symlink, or is not a directory, and a lock name that is a symlink, are all refused with a lock +//! error: the teardown then touches nothing (the jail and socket pins survive) and nothing is +//! created at a symlink's target. +#![cfg(feature = "insecure_mode")] + +use pretty_assertions::assert_eq; +use rstest::rstest; +use serde_json::json; +use std::fs; +use std::path::PathBuf; + +mod setuid_support; +mod support; +use setuid_support::{run_in_fresh_run, Install, MEMBER}; +use support::is_root; + +const ID: &str = "v1"; + +struct Vm { + install: Install, + config: PathBuf, + jail_marker: PathBuf, + pin: PathBuf, +} + +fn vm() -> Vm { + let install = Install::new(); + let srv = install.path().join("srv"); + let root = srv.join("jails/firecracker").join(ID).join("root"); + fs::create_dir_all(&root).unwrap(); + let jail_marker = root.join("marker"); + fs::write(&jail_marker, b"still here").unwrap(); + let socks = srv.join("socks"); + fs::create_dir_all(&socks).unwrap(); + nix::unistd::chown( + &socks, + Some(nix::unistd::Uid::from_raw(MEMBER.uid)), + Some(nix::unistd::Gid::from_raw(MEMBER.gid)), + ) + .unwrap(); + let pin = socks.join(format!("fc-{ID}-api.socket")); + fs::write(&pin, b"").unwrap(); + let config = install.config(&srv); + Vm { + install, + config, + jail_marker, + pin, + } +} + +#[test] +fn a_member_on_a_fresh_run_creates_a_root_only_lock_dir_as_root() { + if !is_root() { + eprintln!("SKIP fresh lock dir: needs root to install setuid"); + return; + } + let vm = vm(); + + let out = run_in_fresh_run( + &vm.install, + &vm.config, + MEMBER, + "umask 000", + &json!({"op": "vm_teardown", "id": ID}), + &[ + "/run/hyper-suidhelper", + "/run/hyper-suidhelper/locks", + "/run/hyper-suidhelper/locks/vm-v1", + ], + ); + + assert_eq!( + ( + out.code, + out.stdout.trim(), + out.state.as_str(), + vm.jail_marker.exists(), + vm.pin.exists(), + ), + ( + Some(0), + r#"{"result":"tore_down"}"#, + "/run/hyper-suidhelper directory 755 0 0\n/run/hyper-suidhelper/locks directory 700 0 0\n", + false, + false, + ), + "{}", + out.stderr + ); +} + +#[rstest] +#[case::member_owned_helper_dir( + "mkdir -m 755 /run/hyper-suidhelper; chown 62010:62001 /run/hyper-suidhelper", + "" +)] +#[case::world_writable_sticky_helper_dir("mkdir -m 1777 /run/hyper-suidhelper", "")] +#[case::group_writable_helper_dir("mkdir -m 775 /run/hyper-suidhelper", "")] +#[case::helper_dir_is_a_file("touch /run/hyper-suidhelper", "")] +#[case::helper_dir_symlinked_to_a_member_dir( + "mkdir -m 755 /run/attacker; mkdir -m 700 /run/attacker/locks; chown 62010 /run/attacker; \ + ln -s /run/attacker /run/hyper-suidhelper", + "/run/attacker/locks/vm-v1" +)] +#[case::member_owned_locks( + "mkdir -m 755 /run/hyper-suidhelper; mkdir -m 700 /run/hyper-suidhelper/locks; \ + chown 62010 /run/hyper-suidhelper/locks", + "" +)] +#[case::group_writable_locks( + "mkdir -m 755 /run/hyper-suidhelper; mkdir -m 730 /run/hyper-suidhelper/locks", + "" +)] +#[case::locks_symlinked_to_a_root_dir( + "mkdir -m 755 /run/hyper-suidhelper; mkdir -m 700 /run/elsewhere; \ + ln -s /run/elsewhere /run/hyper-suidhelper/locks", + "/run/elsewhere/vm-v1" +)] +#[case::lock_name_symlinked( + "mkdir -m 755 /run/hyper-suidhelper; mkdir -m 700 /run/hyper-suidhelper/locks; \ + ln -s /run/victim /run/hyper-suidhelper/locks/vm-v1", + "/run/victim" +)] +fn a_lock_dir_root_does_not_solely_control_is_refused_and_teardown_touches_nothing_as_root( + #[case] setup: &str, + #[case] never_created: &str, +) { + if !is_root() { + eprintln!("SKIP hostile lock dir: needs root to install setuid"); + return; + } + let vm = vm(); + let observe: Vec<&str> = [never_created] + .into_iter() + .filter(|p| !p.is_empty()) + .collect(); + + let out = run_in_fresh_run( + &vm.install, + &vm.config, + MEMBER, + setup, + &json!({"op": "vm_teardown", "id": ID}), + &observe, + ); + + assert_eq!( + ( + out.code, + out.stdout.as_str(), + out.stderr.contains("lock"), + vm.jail_marker.exists(), + vm.pin.exists(), + out.state.as_str(), + ), + (Some(2), "", true, true, true, ""), + "{setup}: {}", + out.stderr + ); +} diff --git a/native/suidhelper/tests/e2e/setuid_support.rs b/native/suidhelper/tests/e2e/setuid_support.rs new file mode 100644 index 00000000..49efa488 --- /dev/null +++ b/native/suidhelper/tests/e2e/setuid_support.rs @@ -0,0 +1,263 @@ +//! A real setuid-root install of the static insecure-mode helper, driven by a non-root member of +//! its group, optionally inside a private mount namespace whose `/run` is a fresh tmpfs, so the +//! helper's fixed `/run/hyper-suidhelper` lock dir can be staged hostile without touching the +//! host's. + +// Each [[test]] target compiles its own copy of this module and not every target uses every item. +#![allow(dead_code)] + +use assert_fs::TempDir; +use nix::unistd::{Gid, Uid}; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR"); + +pub const INSTALL_GROUP: u32 = 62_001; + +#[derive(Clone, Copy, Debug)] +pub struct Caller { + pub uid: u32, + pub gid: u32, + pub groups: &'static [u32], +} + +pub const MEMBER: Caller = Caller { + uid: 62_010, + gid: INSTALL_GROUP, + groups: &[], +}; + +pub const OTHER_MEMBER: Caller = Caller { + uid: 62_011, + gid: 62_020, + groups: &[INSTALL_GROUP], +}; + +pub const ROOT: Caller = Caller { + uid: 0, + gid: 0, + groups: &[0], +}; + +fn home_of(uid: u32) -> Option { + nix::unistd::User::from_uid(Uid::from_raw(uid)) + .ok() + .flatten() + .map(|user| user.dir) +} + +/// The insecure-mode helper statically linked for musl, the only kind that runs setuid. +pub fn static_insecure_helper() -> PathBuf { + let target = format!("{}-unknown-linux-musl", std::env::consts::ARCH); + let target_dir = Path::new(MANIFEST_DIR).join("../target/setuid-e2e/insecure"); + let mut cargo = Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned())); + cargo + .current_dir(MANIFEST_DIR) + .args([ + "build", + "--quiet", + "-p", + "hyper-suidhelper", + "--bin", + "hyper-suidhelper", + "--features", + "insecure_mode", + "--target", + &target, + "--target-dir", + ]) + .arg(&target_dir); + for coverage_or_target in [ + "RUSTFLAGS", + "CARGO_ENCODED_RUSTFLAGS", + "RUSTC_WRAPPER", + "RUSTC_WORKSPACE_WRAPPER", + "LLVM_PROFILE_FILE", + "CARGO_TARGET_DIR", + "CARGO_BUILD_TARGET_DIR", + "CARGO_LLVM_COV", + "CARGO_LLVM_COV_TARGET_DIR", + ] { + cargo.env_remove(coverage_or_target); + } + let invoker = std::env::var("SUDO_UID") + .ok() + .zip(std::env::var("SUDO_GID").ok()) + .and_then(|(uid, gid)| Some((uid.parse::().ok()?, gid.parse::().ok()?))); + if let Some((uid, gid)) = invoker { + if let Some(home) = home_of(uid) { + cargo.env("HOME", home); + } + cargo.uid(uid).gid(gid); + } + assert!( + cargo.status().expect("spawning cargo").success(), + "building the static insecure helper failed" + ); + target_dir.join(&target).join("debug/hyper-suidhelper") +} + +/// A tempdir every caller can traverse, holding a `root:` 4755 copy of the helper. +pub struct Install { + pub dir: TempDir, + pub bin: PathBuf, +} + +impl Install { + pub fn new() -> Self { + let dir = TempDir::new().unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let bin = dir.path().join("hyper-suidhelper"); + fs::copy(static_insecure_helper(), &bin).unwrap(); + nix::unistd::chown( + &bin, + Some(Uid::from_raw(0)), + Some(Gid::from_raw(INSTALL_GROUP)), + ) + .unwrap(); + fs::set_permissions(&bin, fs::Permissions::from_mode(0o4755)).unwrap(); + Self { dir, bin } + } + + pub fn path(&self) -> &Path { + self.dir.path() + } + + /// A world-readable, root-owned config naming `work_dir` and a cgroup that does not exist. + pub fn config(&self, work_dir: &Path) -> PathBuf { + let path = self.path().join("config.toml"); + fs::write( + &path, + format!( + "work_dir = \"{}\"\n[jails]\ncgroup = \"hyper-e2e-does-not-exist\"\n", + work_dir.display() + ), + ) + .unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + path + } +} + +/// What one helper invocation left behind. +#[derive(Debug)] +pub struct Outcome { + pub code: Option, + pub stdout: String, + pub stderr: String, + /// `stat -c '%n %F %a %u %g'` of every path the script listed, as seen inside the namespace. + pub state: String, +} + +/// Run `bin` with `request` as `caller` (via `setpriv`, so the setuid bit does the elevating), +/// inside a private mount namespace whose `/run` is an empty tmpfs on which `setup` (a shell +/// snippet, run as root) stages whatever it likes first. `observe` lists paths to `stat` inside +/// the namespace after the helper exits. +pub fn run_in_fresh_run( + install: &Install, + config: &Path, + caller: Caller, + setup: &str, + request: &serde_json::Value, + observe: &[&str], +) -> Outcome { + let out = install.path().join(format!( + "out-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&out).unwrap(); + let groups = caller + .groups + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + let as_caller = if caller.uid == 0 { + String::new() + } else if groups.is_empty() { + format!( + "setpriv --reuid {} --regid {} --clear-groups", + caller.uid, caller.gid + ) + } else { + format!( + "setpriv --reuid {} --regid {} --groups {groups}", + caller.uid, caller.gid + ) + }; + let script = format!( + r#"set -e +mount -t tmpfs -o mode=0755 hyper-e2e-run /run +{setup} +set +e +{as_caller} env -i HYPER_SETUIDHELPER_IS_INSECURE_MODE=1 HYPER_SETUIDHELPER_CONFIG_PATH="$2" "$1" "$3" >"$4/stdout" 2>"$4/stderr" +echo $? >"$4/rc" +for p in {observe}; do stat -c '%n %F %a %u %g' "$p" 2>/dev/null; done >"$4/state" +exit 0 +"#, + observe = observe + .iter() + .map(|p| format!("'{p}'")) + .collect::>() + .join(" "), + ); + let status = Command::new("unshare") + .args(["--mount", "--propagation", "private", "--", "/bin/sh", "-c"]) + .arg(&script) + .arg("sh") + .arg(&install.bin) + .arg(config) + .arg(request.to_string()) + .arg(&out) + .env_clear() + .env("PATH", "/usr/sbin:/usr/bin:/sbin:/bin") + .status() + .unwrap(); + assert!(status.success(), "the namespace script itself failed"); + let read = |name: &str| fs::read_to_string(out.join(name)).unwrap_or_default(); + Outcome { + code: read("rc").trim().parse().ok(), + stdout: read("stdout"), + stderr: read("stderr"), + state: read("state"), + } +} + +/// Run `bin` with `request` as `caller` against the host's own `/run`. +pub fn run_as( + install: &Install, + config: &Path, + caller: Caller, + request: &serde_json::Value, +) -> Outcome { + let Caller { uid, gid, groups } = caller; + let mut cmd = Command::new(&install.bin); + cmd.arg(request.to_string()) + .env_clear() + .env("HYPER_SETUIDHELPER_IS_INSECURE_MODE", "1") + .env("HYPER_SETUIDHELPER_CONFIG_PATH", config); + // SAFETY: the hook runs in the forked child before exec and makes only raw id syscalls. + unsafe { + cmd.pre_exec(move || { + let groups: Vec = groups.iter().copied().map(Gid::from_raw).collect(); + nix::unistd::setgroups(&groups)?; + nix::unistd::setresgid(Gid::from_raw(gid), Gid::from_raw(gid), Gid::from_raw(gid))?; + nix::unistd::setresuid(Uid::from_raw(uid), Uid::from_raw(uid), Uid::from_raw(uid))?; + Ok(()) + }) + }; + let out = cmd.output().unwrap(); + Outcome { + code: out.status.code(), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + state: String::new(), + } +} diff --git a/native/suidhelper/tests/e2e/socket_pin.rs b/native/suidhelper/tests/e2e/socket_pin.rs new file mode 100644 index 00000000..f646268d --- /dev/null +++ b/native/suidhelper/tests/e2e/socket_pin.rs @@ -0,0 +1,373 @@ +//! The socket-pin contract of #135, observed through the binary: `vm_claim_*_socket` hard-links +//! the VM's own socket into `/socks/fc--`, and `vm_teardown` removes exactly +//! that VM's pins under the VM's lock. +//! +//! Laws: +//! - claim is idempotent: re-claiming leaves one pin, still the jail socket's inode, `0600` and +//! owned by the caller; +//! - claim never follows what already sits at the pin name: a planted symlink is replaced, its +//! target untouched; +//! - claim refuses anything at the jail socket name that is not a socket, and then pins nothing; +//! - teardown is scoped: tearing down VM `a` removes `a`'s two pins and jail and nothing else in +//! the socket dir, even for a VM `b` whose id has `a` as a dash-separated prefix; +//! - teardown is idempotent and leaves no lock file behind, and concurrent teardowns of one VM all +//! succeed (they serialize on the lock rather than time out or trip over each other). +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use pretty_assertions::assert_eq; +use proptest::prelude::*; +use rstest::rstest; +use serde_json::json; +use std::fs; +use std::os::unix::fs::{symlink, FileTypeExt, MetadataExt, PermissionsExt}; +use std::os::unix::net::UnixListener; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +mod support; +use support::{is_root, run, stdout_json}; + +const SOCKETS: [(&str, &str); 2] = [ + ("vm_claim_api_socket", "api.socket"), + ("vm_claim_vsock_socket", "vsock.sock"), +]; + +const LOCKS: &str = "/run/hyper-suidhelper/locks"; + +struct Work { + tmp: TempDir, + srv: PathBuf, + config: PathBuf, + listeners: Vec, +} + +impl Work { + fn new() -> Self { + let tmp = TempDir::new().unwrap(); + let srv = tmp.path().join("srv"); + fs::create_dir_all(srv.join("socks")).unwrap(); + let config = tmp.path().join("config.toml"); + fs::write( + &config, + format!( + "work_dir = \"{}\"\n[jails]\ncgroup = \"hyper-e2e-does-not-exist\"\n", + srv.display() + ), + ) + .unwrap(); + fs::set_permissions(&config, fs::Permissions::from_mode(0o644)).unwrap(); + Self { + tmp, + srv, + config, + listeners: Vec::new(), + } + } + + fn jail(&self, id: &str) -> PathBuf { + let root = self.srv.join("jails/firecracker").join(id).join("root"); + fs::create_dir_all(&root).unwrap(); + root + } + + fn socks(&self) -> PathBuf { + self.srv.join("socks") + } + + fn pin(&self, id: &str, socket: &str) -> PathBuf { + self.socks().join(format!("fc-{id}-{socket}")) + } + + fn boot(&mut self, id: &str) { + let jail = self.jail(id); + for (_, socket) in SOCKETS { + self.listeners + .push(UnixListener::bind(jail.join(socket)).unwrap()); + } + } + + fn claim_all(&self, id: &str) { + for (op, _) in SOCKETS { + let out = run(&self.config, json!({"op": op, "id": id})).success(); + assert_eq!(stdout_json(&out)["result"], "claimed", "{op} {id}"); + } + } + + fn teardown(&self, id: &str) { + let out = run(&self.config, json!({"op": "vm_teardown", "id": id})).success(); + assert_eq!(stdout_json(&out)["result"], "tore_down"); + } +} + +fn unique(tag: &str) -> String { + format!("{tag}{}", std::process::id()) +} + +fn ino(path: &Path) -> (u64, u64) { + let meta = fs::symlink_metadata(path).unwrap(); + (meta.dev(), meta.ino()) +} + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn reclaiming_keeps_one_private_pin_on_the_jail_socket_as_root(#[case] (op, socket): (&str, &str)) { + if !is_root() { + eprintln!("SKIP claim idempotence: needs root"); + return; + } + let mut work = Work::new(); + let id = unique("idem"); + work.boot(&id); + + for _ in 0..3 { + let out = run(&work.config, json!({"op": op, "id": id})).success(); + assert_eq!(stdout_json(&out)["result"], "claimed"); + } + + let pin = work.pin(&id, socket); + let meta = fs::symlink_metadata(&pin).unwrap(); + assert_eq!( + ( + ino(&pin), + meta.file_type().is_socket(), + meta.mode() & 0o7777, + meta.uid(), + meta.nlink(), + ), + ( + ino(&work.jail(&id).join(socket)), + true, + 0o600, + nix::unistd::getuid().as_raw(), + 2, + ) + ); + work.teardown(&id); +} + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn claim_replaces_a_planted_symlink_pin_without_following_it_as_root( + #[case] (op, socket): (&str, &str), +) { + if !is_root() { + eprintln!("SKIP claim symlink pin: needs root"); + return; + } + let mut work = Work::new(); + let id = unique("sym"); + work.boot(&id); + let victim = work.tmp.path().join("victim"); + fs::write(&victim, b"keep").unwrap(); + fs::set_permissions(&victim, fs::Permissions::from_mode(0o644)).unwrap(); + let victim_before = fs::symlink_metadata(&victim).unwrap(); + symlink(&victim, work.pin(&id, socket)).unwrap(); + + run(&work.config, json!({"op": op, "id": id})).success(); + + let victim_after = fs::symlink_metadata(&victim).unwrap(); + assert_eq!( + ( + fs::read(&victim).unwrap(), + victim_after.mode(), + victim_after.uid(), + victim_after.nlink(), + ino(&work.pin(&id, socket)), + ), + ( + b"keep".to_vec(), + victim_before.mode(), + victim_before.uid(), + 1, + ino(&work.jail(&id).join(socket)), + ) + ); + work.teardown(&id); +} + +#[derive(Debug, Clone, Copy)] +enum Planted { + Regular, + Directory, + Symlink, + Fifo, +} + +fn plant(at: &Path, what: Planted, decoy: &Path) { + match what { + Planted::Regular => fs::write(at, b"x").unwrap(), + Planted::Directory => fs::create_dir(at).unwrap(), + Planted::Symlink => symlink(decoy, at).unwrap(), + Planted::Fifo => { + nix::unistd::mkfifo(at, nix::sys::stat::Mode::from_bits_truncate(0o600)).unwrap() + } + } +} + +#[rstest] +#[case::api_regular(SOCKETS[0], Planted::Regular)] +#[case::api_directory(SOCKETS[0], Planted::Directory)] +#[case::api_symlink_to_a_live_socket(SOCKETS[0], Planted::Symlink)] +#[case::vsock_fifo(SOCKETS[1], Planted::Fifo)] +#[case::vsock_symlink_to_a_live_socket(SOCKETS[1], Planted::Symlink)] +fn claim_of_a_non_socket_is_refused_and_pins_nothing_as_root( + #[case] (op, socket): (&str, &str), + #[case] planted: Planted, +) { + if !is_root() { + eprintln!("SKIP claim non-socket: needs root"); + return; + } + let work = Work::new(); + let id = unique("bad"); + let decoy = work.tmp.path().join("decoy.sock"); + let _decoy = UnixListener::bind(&decoy).unwrap(); + plant(&work.jail(&id).join(socket), planted, &decoy); + + let out = run(&work.config, json!({"op": op, "id": id})).failure(); + + assert_eq!( + ( + String::from_utf8_lossy(&out.get_output().stdout).contains("claimed"), + work.pin(&id, socket).symlink_metadata().is_ok(), + ), + (false, false), + "{planted:?}" + ); + work.teardown(&id); +} + +fn id_pair() -> impl Strategy { + const ID: &str = "[a-zA-Z0-9][a-zA-Z0-9-]{0,12}"; + let suffix = prop_oneof![ + Just("api".to_owned()), + Just("vsock".to_owned()), + Just("api-socket".to_owned()), + "[a-z0-9]{1,6}", + ]; + (ID, suffix, ID, any::()) + .prop_map(|(a, suffix, other, prefixed)| { + let b = if prefixed { + format!("{a}-{suffix}") + } else { + other + }; + (a, b) + }) + .prop_filter("distinct ids", |(a, b)| a != b) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(10))] + + #[test] + fn teardown_removes_exactly_its_own_pins_and_jail_as_root((a, b) in id_pair()) { + if !is_root() { + eprintln!("SKIP teardown scoping: needs root"); + return Ok(()); + } + let tag = std::process::id(); + let (a, b) = (format!("{a}{tag}"), format!("{b}{tag}")); + prop_assume!(a != b); + let mut work = Work::new(); + work.boot(&a); + work.boot(&b); + work.claim_all(&a); + work.claim_all(&b); + let decoys = [ + format!("grpc-{a}.sock"), + format!("fc-{a}-api.socket.old"), + format!("fc-{a}-other.sock"), + format!("fc-{a}"), + ]; + for decoy in &decoys { + fs::write(work.socks().join(decoy), b"").unwrap(); + } + + work.teardown(&a); + + let mut left: Vec = fs::read_dir(work.socks()) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + let mut expected: Vec = decoys + .iter() + .cloned() + .chain(SOCKETS.iter().map(|(_, s)| format!("fc-{b}-{s}"))) + .collect(); + expected.sort(); + prop_assert_eq!(left, expected); + prop_assert!(!work.srv.join("jails/firecracker").join(&a).exists()); + prop_assert!(work.jail(&b).join("api.socket").exists()); + work.teardown(&b); + } +} + +#[rstest] +fn teardown_is_idempotent_and_leaves_no_lock_file_as_root() { + if !is_root() { + eprintln!("SKIP teardown idempotence: needs root"); + return; + } + let mut work = Work::new(); + let id = unique("twice"); + work.boot(&id); + work.claim_all(&id); + + work.teardown(&id); + work.teardown(&id); + fs::remove_dir(work.socks()).unwrap(); + work.teardown(&id); + + assert!( + !Path::new(LOCKS).join(format!("vm-{id}")).exists(), + "teardown must remove its lock file" + ); +} + +#[rstest] +fn concurrent_teardowns_of_one_vm_all_succeed_as_root() { + if !is_root() { + eprintln!("SKIP concurrent teardown: needs root"); + return; + } + let mut work = Work::new(); + let id = unique("race"); + work.boot(&id); + work.claim_all(&id); + let request = json!({"op": "vm_teardown", "id": id}).to_string(); + + let children: Vec<_> = (0..8) + .map(|_| { + support::command(&work.config, &[&request]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() + }) + .collect(); + let outcomes: Vec<(bool, String)> = children + .into_iter() + .map(|child| { + let out = child.wait_with_output().unwrap(); + ( + out.status.success() && String::from_utf8_lossy(&out.stdout).contains("tore_down"), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) + }) + .collect(); + + let failed: Vec<&String> = outcomes + .iter() + .filter(|(ok, _)| !ok) + .map(|(_, e)| e) + .collect(); + assert!(failed.is_empty(), "teardowns failed: {failed:?}"); + assert!(!work.pin(&id, "api.socket").exists()); + assert!(!work.srv.join("jails/firecracker").join(&id).exists()); + assert!(!Path::new(LOCKS).join(format!("vm-{id}")).exists()); +} diff --git a/native/suidhelper/tests/e2e/vm_net.rs b/native/suidhelper/tests/e2e/vm_net.rs new file mode 100644 index 00000000..f3a021f0 --- /dev/null +++ b/native/suidhelper/tests/e2e/vm_net.rs @@ -0,0 +1,549 @@ +//! The network `vm_launch` builds and `vm_teardown` removes, observed from a scratch host +//! namespace with `ip` and `nft` rather than the helper's own rtnetlink code. +//! +//! Laws: +//! - teardown removes everything a launch added to the host: the `vm-veth-`, the +//! guest's `/128` route, its proxy-NDP entry and the VM's namespace; a second teardown +//! changes nothing; +//! - tearing one VM down leaves every other VM's network as it was; +//! - a relaunch replaces the VM's previous network, whatever NetId it held; +//! - a launch that fails while networking (a NetId another VM holds, a missing uplink) +//! leaves no network of its own behind and never damages the VM holding the NetId; +//! - teardown without `[net]` still unlinks the VM, and the next teardown with `[net]` +//! sweeps the proxy entry it could not find; +//! - helpers launching and tearing down distinct VMs at once end in exactly the network +//! of the VMs left running; +//! - `vm_firewall` replaces the port policy whole and leaves the namespace's own table be; +//! - a policy at the per-direction cap, on the edge ports, is one `nft` accepts. +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use pretty_assertions::assert_eq; +use rstest::rstest; +use serde_json::{json, Value}; +use std::collections::BTreeSet; +use std::fs; +use std::net::Ipv6Addr; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicU32, Ordering}; + +mod support; +use hyperrs_linux::net::netns::NetNs; +use support::{is_root, MapperDevice, ScratchNs}; + +const RECORDER_EXIT: i32 = 7; +const REFUSED_EXIT: i32 = 2; +const NODE_PREFIX: &str = "2001:db8::/64"; + +static HOSTS: AtomicU32 = AtomicU32::new(0); + +/// The cookbook's VM address for `net_id`, `:1:0:0:`. +fn guest(net_id: u16) -> Ipv6Addr { + Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 1, 0, 0, net_id) +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct HostNet { + veths: BTreeSet, + guest_routes: BTreeSet, + proxies: BTreeSet, +} + +impl HostNet { + fn of(vms: &[u16]) -> Self { + Self { + veths: vms.iter().map(|id| format!("vm-veth-{id}")).collect(), + guest_routes: vms.iter().copied().map(guest).collect(), + proxies: vms.iter().copied().map(guest).collect(), + } + } +} + +struct Host { + ns: ScratchNs, + tmp: TempDir, + work: PathBuf, + firecracker: PathBuf, + jailer: PathBuf, + kernel: PathBuf, + rootfs: MapperDevice, + tag: u32, +} + +impl Drop for Host { + fn drop(&mut self) { + let Ok(entries) = fs::read_dir("/run/netns") else { + return; + }; + let mine = format!("hyper-vm{}x{}", std::process::id(), self.tag); + for entry in entries.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with(&mine) { + let _ = NetNs::delete(name.as_str().try_into().unwrap()); + } + } + } +} + +impl Host { + fn new() -> Option { + if !is_root() { + return None; + } + let tmp = TempDir::new().unwrap(); + let base = fs::canonicalize(tmp.path()).unwrap(); + let work = base.join("srv"); + fs::create_dir_all(work.join("jails")).unwrap(); + let firecracker = base.join("firecracker"); + fs::write(&firecracker, "#!/bin/true\n").unwrap(); + let jailer = base.join("jailer"); + fs::copy(env!("CARGO_BIN_EXE_fake-jailer"), &jailer).unwrap(); + fs::set_permissions(&jailer, fs::Permissions::from_mode(0o755)).unwrap(); + let kernel = work.join("vmlinux-src"); + fs::write(&kernel, "kernel image").unwrap(); + let backing = base.join("rootfs.img"); + fs::write(&backing, vec![0u8; 1024 * 1024]).unwrap(); + let tag = HOSTS.fetch_add(1, Ordering::Relaxed); + let rootfs = MapperDevice::linear( + &format!("hyper-e2e-vmnet-{}-{tag}", std::process::id()), + &backing, + )?; + Some(Self { + ns: ScratchNs::new(&format!("vmnet{tag}")), + tmp, + work, + firecracker, + jailer, + kernel, + rootfs, + tag, + }) + } + + fn id(&self, vm: &str) -> String { + format!("vm{}x{}{vm}", std::process::id(), self.tag) + } + + fn config(&self, net: Option<&str>) -> PathBuf { + let p = self + .tmp + .path() + .join(format!("config-{}.toml", net.unwrap_or("none"))); + let net = net.map_or(String::new(), |uplink| { + format!("[net]\nuplink = \"{uplink}\"\nipv6_prefix = \"{NODE_PREFIX}\"\n") + }); + let body = format!( + "work_dir = \"{}\"\n[tools]\njailer = \"{}\"\nfirecracker = \"{}\"\n{net}", + self.work.display(), + self.jailer.display(), + self.firecracker.display(), + ); + fs::write(&p, body).unwrap(); + fs::set_permissions(&p, fs::Permissions::from_mode(0o644)).unwrap(); + p + } + + fn launch(&self, cfg: &Path, vm: &str, net_id: u16, firewall: Value) -> i32 { + let request = json!({ + "op": "vm_launch", + "id": self.id(vm), + "uid": 900_001, + "gid": 900_002, + "kernel": self.kernel, + "device": self.rootfs.path(), + "limits": {}, + "net_id": net_id, + "firewall": firewall, + }); + self.ns + .run(cfg, request) + .get_output() + .status + .code() + .unwrap() + } + + /// Run every request at once, each in its own helper, and return their exit codes. + fn concurrently(&self, cfg: &Path, requests: &[Value]) -> Vec { + let children: Vec<_> = requests + .iter() + .map(|request| { + self.ns + .get() + .scoped(|| { + support::command(cfg, &[&request.to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() + }) + .unwrap() + }) + .collect(); + children + .into_iter() + .map(|child| { + let out = child.wait_with_output().unwrap(); + out.status.code().unwrap() + }) + .collect() + } + + fn launch_request(&self, vm: &str, net_id: u16) -> Value { + json!({ + "op": "vm_launch", + "id": self.id(vm), + "uid": 900_001, + "gid": 900_002, + "kernel": self.kernel, + "device": self.rootfs.path(), + "limits": {}, + "net_id": net_id, + "firewall": open(), + }) + } + + fn teardown(&self, cfg: &Path, vm: &str) { + self.ns + .run(cfg, json!({"op": "vm_teardown", "id": self.id(vm)})) + .success(); + } + + fn has_netns(&self, vm: &str) -> bool { + Path::new("/run/netns") + .join(format!("hyper-{}", self.id(vm))) + .exists() + } + + fn ip(&self, args: &[&str]) -> Value { + let out = self + .ns + .get() + .scoped(|| Command::new("ip").args(args).output().unwrap()) + .unwrap(); + assert!(out.status.success(), "ip {args:?}: {out:?}"); + let stdout = String::from_utf8(out.stdout).unwrap(); + if stdout.trim().is_empty() { + return json!([]); + } + serde_json::from_str(&stdout).unwrap() + } + + fn observe(&self) -> HostNet { + let in_vm_range = |addr: Ipv6Addr| u128::from(addr) >> 48 == u128::from(guest(0)) >> 48; + let addrs = |listing: Value, key: &str| -> BTreeSet { + listing + .as_array() + .unwrap() + .iter() + .filter_map(|entry| entry[key].as_str()) + .filter_map(|dst| dst.split('/').next().unwrap().parse().ok()) + .filter(|addr| in_vm_range(*addr)) + .collect() + }; + HostNet { + veths: self + .ip(&["-j", "link", "show"]) + .as_array() + .unwrap() + .iter() + .filter_map(|link| link["ifname"].as_str()) + .filter(|name| name.starts_with("vm-veth-")) + .map(str::to_owned) + .collect(), + guest_routes: addrs(self.ip(&["-j", "-6", "route", "show"]), "dst"), + proxies: addrs(self.ip(&["-j", "-6", "neigh", "show", "proxy"]), "dst"), + } + } + + fn nft_rules(&self, vm: &str, table: &str) -> Option> { + let ns = NetNs::open( + format!("hyper-{}", self.id(vm)) + .as_str() + .try_into() + .unwrap(), + ) + .ok()?; + let out = ns + .scoped(|| { + Command::new("nft") + .args(["-j", "list", "table", "inet", table]) + .output() + .unwrap() + }) + .unwrap(); + if !out.status.success() { + return None; + } + let listing: Value = serde_json::from_slice(&out.stdout).unwrap(); + Some( + listing["nftables"] + .as_array() + .unwrap() + .iter() + .filter_map(|item| item.get("rule")) + .map(|rule| rule["expr"].clone()) + .collect(), + ) + } + + /// Every destination-port match in `table`, as `(protocol, first, last)`. + fn dports(&self, vm: &str, table: &str) -> Vec<(String, u64, u64)> { + self.nft_rules(vm, table) + .expect("the table is listed") + .iter() + .flat_map(|expr| expr.as_array().unwrap().clone()) + .filter_map(|statement| { + let m = statement.get("match")?; + let payload = m["left"].get("payload")?; + (payload["field"] == "dport").then(|| { + let protocol = payload["protocol"].as_str().unwrap().to_owned(); + match &m["right"] { + Value::Number(port) => { + (protocol, port.as_u64().unwrap(), port.as_u64().unwrap()) + } + range => ( + protocol, + range["range"][0].as_u64().unwrap(), + range["range"][1].as_u64().unwrap(), + ), + } + }) + }) + .collect() + } +} + +fn open() -> Value { + json!({"inbound": [], "outbound_blocked": []}) +} + +macro_rules! require { + () => { + match Host::new() { + Some(host) => host, + None => { + eprintln!("SKIP: needs root and losetup/dmsetup"); + return; + } + } + }; +} + +#[rstest] +#[case::first(0)] +#[case::past_one_octet(64)] +#[case::last(16_383)] +fn teardown_leaves_no_trace_of_the_vm_network_and_is_idempotent_as_root(#[case] net_id: u16) { + let host = require!(); + let cfg = host.config(Some("lo")); + + assert_eq!(host.launch(&cfg, "a", net_id, open()), RECORDER_EXIT); + let launched = (host.observe(), host.has_netns("a")); + host.teardown(&cfg, "a"); + let torn_down = (host.observe(), host.has_netns("a")); + host.teardown(&cfg, "a"); + + assert_eq!( + (launched, torn_down, (host.observe(), host.has_netns("a"))), + ( + (HostNet::of(&[net_id]), true), + (HostNet::default(), false), + (HostNet::default(), false), + ) + ); +} + +#[rstest] +fn tearing_one_vm_down_leaves_its_neighbours_network_alone_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + + for (vm, net_id) in [("a", 1), ("b", 2), ("c", 3)] { + assert_eq!(host.launch(&cfg, vm, net_id, open()), RECORDER_EXIT); + } + host.teardown(&cfg, "b"); + + assert_eq!( + (host.observe(), ["a", "b", "c"].map(|vm| host.has_netns(vm))), + (HostNet::of(&[1, 3]), [true, false, true]) + ); +} + +#[rstest] +fn a_relaunch_onto_another_net_id_replaces_the_old_network_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + + assert_eq!(host.launch(&cfg, "a", 1, open()), RECORDER_EXIT); + assert_eq!(host.launch(&cfg, "a", 2, open()), RECORDER_EXIT); + + assert_eq!(host.observe(), HostNet::of(&[2])); +} + +#[rstest] +fn a_launch_onto_a_held_net_id_fails_and_leaves_the_holder_intact_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + + assert_eq!(host.launch(&cfg, "a", 5, open()), RECORDER_EXIT); + let refused = host.launch(&cfg, "b", 5, open()); + + assert_eq!( + ( + refused, + host.observe(), + host.has_netns("a"), + host.has_netns("b") + ), + (REFUSED_EXIT, HostNet::of(&[5]), true, false) + ); +} + +#[rstest] +fn a_launch_whose_uplink_is_missing_leaves_no_network_behind_as_root() { + let host = require!(); + let cfg = host.config(Some("nope0")); + + let refused = host.launch(&cfg, "a", 4, open()); + + assert_eq!( + (refused, host.observe(), host.has_netns("a")), + (REFUSED_EXIT, HostNet::default(), false) + ); +} + +#[rstest] +fn a_teardown_without_net_config_unlinks_and_the_next_one_sweeps_its_proxy_as_root() { + let host = require!(); + let (with_net, without_net) = (host.config(Some("lo")), host.config(None)); + + assert_eq!(host.launch(&with_net, "a", 6, open()), RECORDER_EXIT); + assert_eq!(host.launch(&with_net, "b", 7, open()), RECORDER_EXIT); + host.teardown(&without_net, "a"); + let unswept = host.observe(); + host.teardown(&with_net, "never-launched"); + + assert_eq!( + (unswept, host.observe(), host.has_netns("a")), + ( + HostNet { + proxies: [guest(6), guest(7)].into(), + ..HostNet::of(&[7]) + }, + HostNet::of(&[7]), + false + ) + ); +} + +#[rstest] +fn setting_the_firewall_replaces_the_policy_whole_and_keeps_the_namespace_table_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + let launched_with = json!({ + "inbound": [{"protocol": "tcp", "ports": [22, 22]}], + "outbound_blocked": [{"protocol": "tcp", "ports": [25, 25]}], + }); + assert_eq!(host.launch(&cfg, "a", 8, launched_with), RECORDER_EXIT); + let namespace_table = host.nft_rules("a", "hyper"); + + host.ns + .run( + &cfg, + json!({"op": "vm_firewall", "id": host.id("a"), "firewall": { + "inbound": [{"protocol": "udp", "ports": [6000, 6010]}], + "outbound_blocked": [], + }}), + ) + .success(); + + assert_eq!( + ( + host.dports("a", "hyper-firewall"), + host.nft_rules("a", "hyper") == namespace_table, + ), + (vec![("udp".to_owned(), 6000, 6010)], true) + ); +} + +#[rstest] +fn a_policy_at_the_rule_cap_on_the_edge_ports_is_installed_whole_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + let edges = [[1, 1], [65_535, 65_535], [1, 65_535]]; + let rules = |protocol: &str| -> Vec<(String, u64, u64)> { + edges + .iter() + .map(|[first, last]| (*first, *last)) + .chain((0..125).map(|i| (1000 + 2 * i, 1001 + 2 * i))) + .map(|(first, last)| (protocol.to_owned(), first, last)) + .collect() + }; + let request = |rules: &[(String, u64, u64)]| -> Vec { + rules + .iter() + .map(|(protocol, first, last)| json!({"protocol": protocol, "ports": [first, last]})) + .collect() + }; + let (inbound, outbound) = (rules("tcp"), rules("udp")); + + let launched = host.launch( + &cfg, + "a", + 9, + json!({"inbound": request(&inbound), "outbound_blocked": request(&outbound)}), + ); + + assert_eq!( + (launched, host.dports("a", "hyper-firewall")), + (RECORDER_EXIT, [inbound, outbound].concat()) + ); +} + +#[rstest] +fn concurrent_launches_and_teardowns_end_in_exactly_the_running_vms_network_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + let first: Vec<(String, u16)> = (0..6).map(|i| (format!("a{i}"), 10 + i)).collect(); + let second: Vec<(String, u16)> = (0..6).map(|i| (format!("b{i}"), 20 + i)).collect(); + + let launched = host.concurrently( + &cfg, + &first + .iter() + .map(|(vm, net_id)| host.launch_request(vm, *net_id)) + .collect::>(), + ); + let mixed = host.concurrently( + &cfg, + &first + .iter() + .step_by(2) + .map(|(vm, _)| json!({"op": "vm_teardown", "id": host.id(vm)})) + .chain( + second + .iter() + .map(|(vm, net_id)| host.launch_request(vm, *net_id)), + ) + .collect::>(), + ); + let running: Vec = first + .iter() + .skip(1) + .step_by(2) + .chain(&second) + .map(|(_, net_id)| *net_id) + .collect(); + + assert_eq!( + (launched, mixed, host.observe()), + ( + vec![RECORDER_EXIT; 6], + [vec![0; 3], vec![RECORDER_EXIT; 6]].concat(), + HostNet::of(&running) + ) + ); +} diff --git a/native/suidhelper/tests/e2e/vm_net_traffic.rs b/native/suidhelper/tests/e2e/vm_net_traffic.rs new file mode 100644 index 00000000..e5e89790 --- /dev/null +++ b/native/suidhelper/tests/e2e/vm_net_traffic.rs @@ -0,0 +1,504 @@ +//! Real connections across a whole node: `host_init`'s table on the host and `vm_launch`'s +//! namespaces, routes and tables for two VMs, with an internet (`wan`) beyond the uplink. +//! +//! Firecracker's `tap0` is swapped for a veth of the same name whose peer is the guest's +//! `eth0`, holding what `vm_launch` gave the tap, so every rule that matches on `tap0` +//! sees guest traffic. Laws, per the networking cookbook: +//! - the guest reaches the internet over IPv4 (double NAT) and IPv6 (routed /128); +//! - inbound, exactly the ports in the VM's open ranges reach it, both ends inclusive, for +//! TCP and UDP alike; +//! - outbound, exactly the ports in its blocked ranges never leave it; +//! - VMs reach each other over their public IPv6 addresses only on the ports the +//! receiving VM opened; +//! - DNS to the gateway reaches the resolver, and nothing else on the gateway answers; +//! - `vm_firewall` takes effect on the next connection. +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use hyperrs_linux::net::link::DuplicateAddrDetection; +use hyperrs_linux::net::netns::NetNs; +use hyperrs_linux::net::route::Route; +use hyperrs_linux::net::{IfIndex, IfName, IpNet, Rtnl}; +use pretty_assertions::assert_eq; +use rstest::rstest; +use serde_json::{json, Value}; +use std::fs; +use std::net::{Ipv6Addr, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::thread; +use std::time::Duration; + +mod support; +use socket2::{Domain, Protocol, Socket, Type}; +use support::{is_root, MapperDevice, ScratchNs}; + +const RECORDER_EXIT: i32 = 7; +const DROPPED_AFTER: Duration = Duration::from_secs(2); +const GUEST_TCP: [u16; 4] = [6999, 7000, 7002, 7003]; +const GUEST_UDP: [u16; 2] = [7100, 7101]; +const WAN_TCP: [u16; 5] = [53, 7199, 7200, 7201, 7202]; +/// Bound to one address each, so a reply leaves from the address the request reached. +const WAN_UDP: [&str; 3] = ["1.1.1.1:53", "198.51.100.1:7202", "[2001:db8:ffff::1]:7202"]; + +static LABS: AtomicU32 = AtomicU32::new(0); + +#[derive(Debug, Clone, Copy)] +enum Vm { + A, + B, +} + +impl Vm { + fn net_id(self) -> u16 { + match self { + Self::A => 1, + Self::B => 2, + } + } + + fn guest_v6(self) -> Ipv6Addr { + Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 1, 0, 0, self.net_id()) + } + + fn firewall(self) -> Value { + match self { + Self::A => json!({ + "inbound": [ + {"protocol": "tcp", "ports": [7000, 7002]}, + {"protocol": "udp", "ports": [7100, 7100]}, + ], + "outbound_blocked": [{"protocol": "tcp", "ports": [7200, 7201]}], + }), + Self::B => json!({ + "inbound": [{"protocol": "tcp", "ports": [7000, 7000]}], + "outbound_blocked": [], + }), + } + } +} + +/// How the internet's router reaches the VM range. +#[derive(Debug, Clone, Copy)] +enum Upstream { + /// Routed to the host's link-local address. + Routed, + /// On-link, so the router solicits each guest address and the host must answer by proxy. + OnLink, +} + +#[derive(Debug, Clone, Copy)] +enum Src { + Guest(Vm), + Wan, +} + +#[derive(Debug, Clone, Copy)] +enum Proto { + Tcp, + Udp, + /// An ICMP echo; the port is ignored. + Ping, +} + +struct Lab { + host: ScratchNs, + wan: ScratchNs, + guests: [ScratchNs; 2], + cfg: PathBuf, + tag: u32, + _tmp: TempDir, + _rootfs: MapperDevice, + _listeners: Vec, +} + +impl Drop for Lab { + fn drop(&mut self) { + for vm in [Vm::A, Vm::B] { + let _ = NetNs::delete(self.netns(vm).as_str().try_into().unwrap()); + } + } +} + +impl Lab { + fn new(upstream: Upstream) -> Option { + if !is_root() { + return None; + } + let tag = LABS.fetch_add(1, Ordering::Relaxed); + let tmp = TempDir::new().unwrap(); + let base = fs::canonicalize(tmp.path()).unwrap(); + let work = base.join("srv"); + fs::create_dir_all(work.join("jails")).unwrap(); + let firecracker = base.join("firecracker"); + fs::write(&firecracker, "#!/bin/true\n").unwrap(); + let jailer = base.join("jailer"); + fs::copy(env!("CARGO_BIN_EXE_fake-jailer"), &jailer).unwrap(); + fs::set_permissions(&jailer, fs::Permissions::from_mode(0o755)).unwrap(); + fs::write(work.join("vmlinux"), "kernel image").unwrap(); + let backing = base.join("rootfs.img"); + fs::write(&backing, vec![0u8; 1024 * 1024]).unwrap(); + let rootfs = MapperDevice::linear( + &format!("hyper-e2e-traffic-{}-{tag}", std::process::id()), + &backing, + )?; + let cfg = base.join("config.toml"); + fs::write( + &cfg, + format!( + "work_dir = \"{}\"\n[tools]\njailer = \"{}\"\nfirecracker = \"{}\"\n\ + [net]\nuplink = \"if0\"\nipv6_prefix = \"2001:db8::/64\"\n", + work.display(), + jailer.display(), + firecracker.display() + ), + ) + .unwrap(); + fs::set_permissions(&cfg, fs::Permissions::from_mode(0o644)).unwrap(); + + let mut lab = Self { + host: ScratchNs::new(&format!("traffic{tag}-host")), + wan: ScratchNs::new(&format!("traffic{tag}-wan")), + guests: [ + ScratchNs::new(&format!("traffic{tag}-ga")), + ScratchNs::new(&format!("traffic{tag}-gb")), + ], + cfg, + tag, + _tmp: tmp, + _rootfs: rootfs, + _listeners: Vec::new(), + }; + lab.wire_the_internet(upstream); + lab.host.run(&lab.cfg, json!({"op": "host_init"})).success(); + for vm in [Vm::A, Vm::B] { + let launched = lab.host.run( + &lab.cfg, + json!({ + "op": "vm_launch", + "id": lab.id(vm), + "uid": 900_001, + "gid": 900_002, + "kernel": work.join("vmlinux"), + "device": lab._rootfs.path(), + "limits": {}, + "net_id": vm.net_id(), + "firewall": vm.firewall(), + }), + ); + assert_eq!(launched.get_output().status.code(), Some(RECORDER_EXIT)); + lab.plug_in_the_guest(vm); + } + + let tcp = |ns: &ScratchNs, port| { + ns.get() + .scoped(|| TcpListener::bind((Ipv6Addr::UNSPECIFIED, port))) + .unwrap() + .unwrap() + }; + let echo = |ns: &ScratchNs, at: SocketAddr| { + let socket = ns.get().scoped(|| UdpSocket::bind(at)).unwrap().unwrap(); + thread::spawn(move || { + let mut buf = [0u8; 64]; + while let Ok((n, from)) = socket.recv_from(&mut buf) { + let _ = socket.send_to(&buf[..n], from); + } + }); + }; + let mut listeners: Vec = WAN_TCP.iter().map(|p| tcp(&lab.wan, *p)).collect(); + WAN_UDP + .iter() + .for_each(|at| echo(&lab.wan, at.parse().unwrap())); + for (guest, vm) in lab.guests.iter().zip([Vm::A, Vm::B]) { + listeners.extend(GUEST_TCP.iter().map(|p| tcp(guest, *p))); + GUEST_UDP + .iter() + .for_each(|p| echo(guest, guest_port(vm, *p))); + } + lab._listeners = listeners; + Some(lab) + } + + fn id(&self, vm: Vm) -> String { + format!("vm{}t{}{vm:?}", std::process::id(), self.tag).to_lowercase() + } + + fn netns(&self, vm: Vm) -> String { + format!("hyper-{}", self.id(vm)) + } + + fn guest(&self, vm: Vm) -> &NetNs { + match vm { + Vm::A => self.guests[0].get(), + Vm::B => self.guests[1].get(), + } + } + + fn wire_the_internet(&self, upstream: Upstream) { + let host = self.host.get().rtnl().unwrap(); + let wan = self.wan.get().rtnl().unwrap(); + host.create_veth(&name("if0"), &name("wan"), self.wan.get()) + .unwrap(); + up(&host, "lo", &[]); + let if0 = up(&host, "if0", &["203.0.113.2/24", "fe80::a/64"]); + up( + &wan, + "lo", + &["198.51.100.1/32", "1.1.1.1/32", "2001:db8:ffff::1/128"], + ); + let wan_link = up(&wan, "wan", &["203.0.113.1/24", "fe80::b/64"]); + let vm_range = match upstream { + Upstream::Routed => route("2001:db8:0:0:1::/80", Some("fe80::a"), wan_link), + Upstream::OnLink => route("2001:db8::/64", None, wan_link), + }; + for (rtnl, route) in [ + (&host, route("0.0.0.0/0", Some("203.0.113.1"), if0)), + (&host, route("::/0", Some("fe80::b"), if0)), + (&wan, vm_range), + ] { + rtnl.add_route(&route).unwrap(); + } + self.host.get().enable_forwarding().unwrap(); + } + + fn plug_in_the_guest(&self, vm: Vm) { + let ns = NetNs::open(self.netns(vm).as_str().try_into().unwrap()).unwrap(); + let inner = ns.rtnl().unwrap(); + inner + .link(&name("tap0")) + .unwrap() + .unwrap() + .delete() + .unwrap(); + inner + .create_veth(&name("tap0"), &name("eth0"), self.guest(vm)) + .unwrap(); + let tap = up(&inner, "tap0", &["172.16.0.1/30", "fe80::1/64"]); + inner + .add_route(&route(&format!("{}/128", vm.guest_v6()), None, tap)) + .unwrap(); + + let guest = self.guest(vm).rtnl().unwrap(); + up(&guest, "lo", &[]); + let eth0 = up( + &guest, + "eth0", + &[ + "172.16.0.2/30", + &format!("{}/128", vm.guest_v6()), + "fe80::2/64", + ], + ); + for (dst, via) in [("0.0.0.0/0", "172.16.0.1"), ("::/0", "fe80::1")] { + guest.add_route(&route(dst, Some(via), eth0)).unwrap(); + } + } + + fn set_firewall(&self, vm: Vm, firewall: Value) { + self.host + .run( + &self.cfg, + json!({"op": "vm_firewall", "id": self.id(vm), "firewall": firewall}), + ) + .success(); + } + + fn reaches(&self, from: Src, proto: Proto, to: SocketAddr) -> bool { + let ns = match from { + Src::Guest(vm) => self.guest(vm), + Src::Wan => self.wan.get(), + }; + match proto { + Proto::Tcp => ns + .scoped(|| TcpStream::connect_timeout(&to, DROPPED_AFTER).is_ok()) + .unwrap(), + Proto::Udp => { + let local: SocketAddr = match to { + SocketAddr::V4(_) => "0.0.0.0:0".parse().unwrap(), + SocketAddr::V6(_) => "[::]:0".parse().unwrap(), + }; + let socket = ns.scoped(|| UdpSocket::bind(local)).unwrap().unwrap(); + socket.set_read_timeout(Some(DROPPED_AFTER)).unwrap(); + socket.send_to(b"ping", to).unwrap(); + let mut buf = [0u8; 64]; + socket + .recv_from(&mut buf) + .is_ok_and(|(n, _)| &buf[..n] == b"ping") + } + Proto::Ping => ns.scoped(|| pings(to)).unwrap(), + } + } +} + +/// Whether an echo request to `to` gets its reply back. +fn pings(to: SocketAddr) -> bool { + const ID: u16 = 0x4859; + let (domain, protocol, request, reply) = match to { + SocketAddr::V4(_) => (Domain::IPV4, Protocol::ICMPV4, 8, 0), + SocketAddr::V6(_) => (Domain::IPV6, Protocol::ICMPV6, 128, 129), + }; + let socket = Socket::new(domain, Type::RAW, Some(protocol)).unwrap(); + socket.set_read_timeout(Some(DROPPED_AFTER)).unwrap(); + let mut echo = [request, 0, 0, 0, (ID >> 8) as u8, ID as u8, 0, 1]; + if matches!(to, SocketAddr::V4(_)) { + let sum = echo + .chunks(2) + .map(|w| u32::from(u16::from_be_bytes([w[0], w[1]]))) + .sum::(); + echo[2..4].copy_from_slice(&(!(((sum & 0xffff) + (sum >> 16)) as u16)).to_be_bytes()); + } + let target = SocketAddr::new(to.ip(), 0); + socket.send_to(&echo, &target.into()).unwrap(); + let mut packet = [0u8; 1500]; + while let Ok(n) = std::io::Read::read(&mut &socket, &mut packet) { + let icmp = match to { + SocketAddr::V4(_) => &packet[usize::from(packet[0] & 0x0f) * 4..n], + SocketAddr::V6(_) => &packet[..n], + }; + if icmp.len() >= 8 && icmp[0] == reply && u16::from_be_bytes([icmp[4], icmp[5]]) == ID { + return true; + } + } + false +} + +fn name(s: &str) -> IfName { + s.parse().unwrap() +} + +fn up(rtnl: &Rtnl, link: &str, addrs: &[&str]) -> IfIndex { + let link = rtnl.link(&name(link)).unwrap().unwrap(); + link.disable_addr_gen().unwrap(); + for addr in addrs { + link.add_addr(addr.parse().unwrap(), DuplicateAddrDetection::Disabled) + .unwrap(); + } + link.set_up().unwrap(); + link.index() +} + +fn route(dst: &str, via: Option<&str>, dev: IfIndex) -> Route { + match dst.parse::().unwrap() { + IpNet::V4(dst) => Route::V4 { + dst, + via: via.map(|via| via.parse().unwrap()), + dev, + }, + IpNet::V6(dst) => Route::V6 { + dst, + via: via.map(|via| via.parse().unwrap()), + dev, + }, + } +} + +fn guest_port(vm: Vm, port: u16) -> SocketAddr { + SocketAddr::from((vm.guest_v6(), port)) +} + +macro_rules! require { + ($upstream:expr) => { + match Lab::new($upstream) { + Some(lab) => lab, + None => { + eprintln!("SKIP: needs root and losetup/dmsetup"); + return; + } + } + }; +} + +#[rstest] +#[case::guest_to_the_internet_over_ipv4(Src::Guest(Vm::A), Proto::Tcp, "198.51.100.1:7202", true)] +#[case::guest_to_the_internet_over_ipv6( + Src::Guest(Vm::A), + Proto::Tcp, + "[2001:db8:ffff::1]:7202", + true +)] +#[case::below_the_blocked_range(Src::Guest(Vm::A), Proto::Tcp, "198.51.100.1:7199", true)] +#[case::first_blocked_port(Src::Guest(Vm::A), Proto::Tcp, "198.51.100.1:7200", false)] +#[case::last_blocked_port(Src::Guest(Vm::A), Proto::Tcp, "198.51.100.1:7201", false)] +#[case::blocked_over_ipv6(Src::Guest(Vm::A), Proto::Tcp, "[2001:db8:ffff::1]:7200", false)] +#[case::another_vm_blocks_nothing(Src::Guest(Vm::B), Proto::Tcp, "198.51.100.1:7200", true)] +#[case::below_the_open_range(Src::Wan, Proto::Tcp, "[2001:db8::1:0:0:1]:6999", false)] +#[case::first_open_port(Src::Wan, Proto::Tcp, "[2001:db8::1:0:0:1]:7000", true)] +#[case::last_open_port(Src::Wan, Proto::Tcp, "[2001:db8::1:0:0:1]:7002", true)] +#[case::above_the_open_range(Src::Wan, Proto::Tcp, "[2001:db8::1:0:0:1]:7003", false)] +#[case::open_udp_port(Src::Wan, Proto::Udp, "[2001:db8::1:0:0:1]:7100", true)] +#[case::closed_udp_port(Src::Wan, Proto::Udp, "[2001:db8::1:0:0:1]:7101", false)] +#[case::tcp_rule_opens_no_udp(Src::Wan, Proto::Udp, "[2001:db8::1:0:0:2]:7100", false)] +#[case::vm_to_vm_on_an_open_port(Src::Guest(Vm::A), Proto::Tcp, "[2001:db8::1:0:0:2]:7000", true)] +#[case::vm_to_vm_on_a_closed_port(Src::Guest(Vm::A), Proto::Tcp, "[2001:db8::1:0:0:2]:7003", false)] +#[case::vm_to_vm_over_the_transit_range(Src::Guest(Vm::A), Proto::Tcp, "10.200.0.10:7000", false)] +#[case::udp_to_the_internet_over_ipv4(Src::Guest(Vm::A), Proto::Udp, "198.51.100.1:7202", true)] +#[case::udp_to_the_internet_over_ipv6( + Src::Guest(Vm::A), + Proto::Udp, + "[2001:db8:ffff::1]:7202", + true +)] +#[case::dns_over_udp_reaches_the_resolver(Src::Guest(Vm::A), Proto::Udp, "172.16.0.1:53", true)] +#[case::ping_to_the_internet_over_ipv4(Src::Guest(Vm::A), Proto::Ping, "198.51.100.1:0", true)] +#[case::ping_to_the_internet_over_ipv6( + Src::Guest(Vm::A), + Proto::Ping, + "[2001:db8:ffff::1]:0", + true +)] +#[case::inbound_ping_is_not_an_open_port(Src::Wan, Proto::Ping, "[2001:db8::1:0:0:1]:0", false)] +#[case::the_gateway_does_not_answer_ping(Src::Guest(Vm::A), Proto::Ping, "172.16.0.1:0", false)] +#[case::dns_over_tcp_reaches_the_resolver(Src::Guest(Vm::A), Proto::Tcp, "172.16.0.1:53", true)] +#[case::other_gateway_ports_are_silent(Src::Guest(Vm::A), Proto::Tcp, "172.16.0.1:7202", false)] +#[case::guest_to_the_host_uplink(Src::Guest(Vm::A), Proto::Tcp, "203.0.113.2:7202", false)] +fn a_launched_node_carries_exactly_what_the_cookbook_allows_as_root( + #[case] from: Src, + #[case] proto: Proto, + #[case] to: &str, + #[case] reachable: bool, +) { + let lab = require!(Upstream::Routed); + + assert_eq!(lab.reaches(from, proto, to.parse().unwrap()), reachable); +} + +#[rstest] +fn a_new_firewall_governs_the_next_connection_as_root() { + let lab = require!(Upstream::Routed); + let before = + [7000, 7003].map(|port| lab.reaches(Src::Wan, Proto::Tcp, guest_port(Vm::A, port))); + + lab.set_firewall( + Vm::A, + json!({"inbound": [{"protocol": "tcp", "ports": [7003, 7003]}], "outbound_blocked": []}), + ); + + assert_eq!( + ( + before, + [7000, 7003].map(|port| lab.reaches(Src::Wan, Proto::Tcp, guest_port(Vm::A, port))), + lab.reaches( + Src::Guest(Vm::A), + Proto::Tcp, + "198.51.100.1:7200".parse().unwrap() + ), + ), + ([true, false], [false, true], true) + ); +} + +#[rstest] +#[case::open_port(7000, true)] +#[case::closed_port(7003, false)] +fn an_on_link_upstream_reaches_a_guest_through_proxy_ndp_as_root( + #[case] port: u16, + #[case] reachable: bool, +) { + let lab = require!(Upstream::OnLink); + + assert_eq!( + lab.reaches(Src::Wan, Proto::Tcp, guest_port(Vm::B, port)), + reachable + ); +} diff --git a/native/suidhelper/tests/net_addressing.rs b/native/suidhelper/tests/net_addressing.rs new file mode 100644 index 00000000..4f82837c --- /dev/null +++ b/native/suidhelper/tests/net_addressing.rs @@ -0,0 +1,150 @@ +//! Laws of a node's address plan, over every prefix the config accepts rather than the +//! cookbook's one `/64`: +//! - a node prefix parses exactly when it is global unicast (`2000::/3`) and `/64` or +//! shorter, and parsing clears its host bits; +//! - `:1::/80` and every guest address sit inside the node prefix, each guest +//! address maps back to its NetId, and distinct NetIds get distinct addresses; +//! - `Blocked::overlapping` agrees with an interval-intersection oracle, never pairs an +//! IPv4 range with an IPv6 one, and names the first overlapping range. + +use hyper_suidhelper::net::{Blocked, NetId, NodePrefix}; +use hyperrs_linux::net::IpNet; +use proptest::prelude::*; +use std::net::{Ipv4Addr, Ipv6Addr}; +use test_strategy::proptest; + +fn mask(bits: u32, len: u8) -> u128 { + match u32::from(len) { + 0 => 0, + len => u128::MAX << (bits - len) & (u128::MAX >> (128 - bits)), + } +} + +fn net_id(id: u16) -> NetId { + id.try_into().unwrap() +} + +/// `[first, last]` of the addresses `net` covers, as integers. +fn interval(net: IpNet) -> (u128, u128, bool) { + match net { + IpNet::V4(net) => { + let m = mask(32, net.prefix.get()); + let first = u128::from(u32::from(net.addr)) & m; + (first, first | (!m & u128::from(u32::MAX)), false) + } + IpNet::V6(net) => { + let m = mask(128, net.prefix.get()); + let first = u128::from(net.addr) & m; + (first, first | !m, true) + } + } +} + +fn shares_an_address(a: IpNet, b: IpNet) -> bool { + let ((a0, a1, a6), (b0, b1, b6)) = (interval(a), interval(b)); + a6 == b6 && a0 <= b1 && b0 <= a1 +} + +fn global_unicast_prefix() -> impl Strategy { + (any::(), 3u8..=64).prop_map(|(bits, len)| { + let addr = (bits & !(0b111 << 125)) | (0b001 << 125); + (Ipv6Addr::from(addr), len) + }) +} + +fn ip_net() -> impl Strategy { + prop_oneof![ + (any::(), 0u8..=32).prop_map(|(a, l)| format!("{}/{l}", Ipv4Addr::from(a))), + (any::(), 0u8..=128).prop_map(|(a, l)| format!("{}/{l}", Ipv6Addr::from(a))), + (0u8..=255, 0u8..=255, 8u8..=24).prop_map(|(a, b, l)| format!("10.{a}.{b}.0/{l}")), + (0u16..4, 0u8..=128).prop_map(|(a, l)| format!("2001:db8:{a}::/{l}")), + ] + .prop_map(|s| s.parse().unwrap()) +} + +#[proptest] +fn a_node_prefix_parses_exactly_when_global_unicast_and_at_most_64_bits( + addr: u128, + #[strategy(0u8..=128)] len: u8, +) { + let s = format!("{}/{len}", Ipv6Addr::from(addr)); + let global = len >= 3 && addr >> 125 == 0b001; + let parsed = s.parse::(); + + prop_assert_eq!(parsed.is_ok(), global && len <= 64, "{}", s); + if let Ok(prefix) = parsed { + prop_assert_eq!(u128::from(prefix.get().addr), addr & mask(128, len)); + prop_assert_eq!(prefix.get().prefix.get(), len); + } +} + +#[proptest] +fn every_guest_address_sits_in_the_node_prefix_and_maps_back_to_its_net_id( + #[strategy(global_unicast_prefix())] node: (Ipv6Addr, u8), + #[strategy(0u16..NetId::COUNT)] id: u16, +) { + let (addr, len) = node; + let prefix: NodePrefix = format!("{addr}/{len}").parse().unwrap(); + let network = u128::from(prefix.get().addr); + let range = prefix.vm_range(); + let guest = prefix.guest(net_id(id)); + + prop_assert_eq!(range.prefix.get(), 80); + prop_assert_eq!(u128::from(range.addr) & mask(128, len), network); + prop_assert_eq!(u128::from(guest) & mask(128, 80), u128::from(range.addr)); + prop_assert_eq!(u128::from(guest) & mask(128, len), network); + prop_assert_eq!(prefix.guest_net_id(guest), Some(net_id(id))); +} + +#[proptest] +fn distinct_net_ids_get_distinct_guest_addresses_under_any_prefix( + #[strategy(global_unicast_prefix())] node: (Ipv6Addr, u8), + #[strategy(0u16..NetId::COUNT)] a: u16, + #[strategy(0u16..NetId::COUNT)] b: u16, +) { + prop_assume!(a != b); + let (addr, len) = node; + let prefix: NodePrefix = format!("{addr}/{len}").parse().unwrap(); + + prop_assert_ne!(prefix.guest(net_id(a)), prefix.guest(net_id(b))); +} + +#[proptest] +fn no_address_outside_the_vm_range_maps_to_a_net_id( + #[strategy(global_unicast_prefix())] node: (Ipv6Addr, u8), + addr: u128, +) { + let (prefix_addr, len) = node; + let prefix: NodePrefix = format!("{prefix_addr}/{len}").parse().unwrap(); + + if let Some(id) = prefix.guest_net_id(Ipv6Addr::from(addr)) { + prop_assert_eq!(u128::from(prefix.guest(id)), addr); + } +} + +#[proptest] +fn overlapping_names_the_first_blocked_range_sharing_an_address( + #[strategy(prop::collection::vec(ip_net(), 0..8))] ranges: Vec, + #[strategy(ip_net())] net: IpNet, +) { + let blocked = + Blocked::try_from(ranges.iter().map(ToString::to_string).collect::>()).unwrap(); + + prop_assert_eq!( + blocked.overlapping(net), + ranges + .iter() + .copied() + .find(|range| shares_an_address(net, *range)) + ); +} + +#[proptest] +fn overlap_is_symmetric(#[strategy(ip_net())] a: IpNet, #[strategy(ip_net())] b: IpNet) { + let blocked = |net: IpNet| Blocked::try_from(vec![net.to_string()]).unwrap(); + + prop_assert_eq!( + blocked(b).overlapping(a).is_some(), + blocked(a).overlapping(b).is_some() + ); +} diff --git a/native/suidhelper/tests/request_wire_properties.rs b/native/suidhelper/tests/request_wire_properties.rs new file mode 100644 index 00000000..6ee9c900 --- /dev/null +++ b/native/suidhelper/tests/request_wire_properties.rs @@ -0,0 +1,269 @@ +//! The node's JSON encoder (Jason) and this parser must read every request the same way, so +//! [`Request::from_args`] is held to JSON semantics and to exact numeric limits. +//! +//! Laws: +//! - JSON semantics, not bytes: any key order and any insignificant whitespace, at any depth, +//! parses exactly when the canonical form does, and routes to the same op; +//! - every op name routes to its own variant, never to a sibling with the same fields (the two +//! socket claims share `{id}`); +//! - a key repeated in any object, at any depth, is refused even when both values agree, so no +//! two JSON parsers can disagree on which value wins; +//! - bounded integers are accepted exactly up to their documented maximum and refused past it; +//! zero is refused exactly where a field is non-zero. + +use hyper_suidhelper::cli::{Request, RequestError}; +use proptest::prelude::*; +use rstest::rstest; +use serde_json::{json, Map, Value}; +use std::ffi::OsString; +use test_strategy::proptest; + +const OPS: &[&str] = &[ + "vm_launch", + "vm_claim_api_socket", + "vm_claim_vsock_socket", + "vm_teardown", + "vm_firewall", + "img_activate", + "dm_remove", + "thin_pool_create", + "thin_volume_create", + "thin_destroy", + "thin_fork_delta", + "host_init", + "version", +]; + +fn parse_raw(raw: &str) -> Result { + Request::from_args([OsString::from(raw)].into_iter()) +} + +fn parse(request: &Value) -> Result { + parse_raw(&request.to_string()) +} + +fn fixture() -> Vec<(String, Value)> { + let fixture: Map = + serde_json::from_str(include_str!("fixtures/requests.json")).unwrap(); + fixture.into_iter().collect() +} + +fn request(which: usize) -> Value { + let mut all = fixture(); + let len = all.len(); + all.swap_remove(which % len).1 +} + +fn routes_to(request: &Request, op: &str) -> bool { + op == match request { + Request::VmLaunch(_) => "vm_launch", + Request::VmClaimApiSocket(_) => "vm_claim_api_socket", + Request::VmClaimVsockSocket(_) => "vm_claim_vsock_socket", + Request::VmTeardown(_) => "vm_teardown", + Request::VmFirewall(_) => "vm_firewall", + Request::ImgActivate(_) => "img_activate", + Request::DmRemove(_) => "dm_remove", + Request::ThinPoolCreate(_) => "thin_pool_create", + Request::ThinVolumeCreate(_) => "thin_volume_create", + Request::ThinDestroy(_) => "thin_destroy", + Request::ThinForkDelta(_) => "thin_fork_delta", + Request::HostInit(_) => "host_init", + Request::Version(_) => "version", + } +} + +/// One generated seed drives every shuffle and spacing choice of a rendering. +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + self.0 >> 33 + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + + fn space(&mut self) -> &'static str { + [" ", "\n", "\t", "\r\n", "", " "][self.below(6)] + } +} + +fn render(value: &Value, rng: &mut Lcg) -> String { + match value { + Value::Object(map) => { + let mut entries: Vec<(&String, &Value)> = map.iter().collect(); + for i in (1..entries.len()).rev() { + entries.swap(i, rng.below(i + 1)); + } + let body: Vec = entries + .into_iter() + .map(|(k, v)| { + let key = Value::String(k.clone()); + let (a, b, c) = (rng.space(), rng.space(), rng.space()); + format!("{a}{key}{b}:{c}{}", render(v, rng)) + }) + .collect(); + format!("{{{}{}}}", body.join(","), rng.space()) + } + Value::Array(items) => { + let body: Vec = items + .iter() + .map(|v| format!("{}{}", rng.space(), render(v, rng))) + .collect(); + format!("[{}{}]", body.join(","), rng.space()) + } + scalar => scalar.to_string(), + } +} + +fn object_paths(value: &Value, at: String, out: &mut Vec) { + match value { + Value::Object(map) => { + out.push(at.clone()); + for (k, v) in map { + object_paths(v, format!("{at}/{k}"), out); + } + } + Value::Array(items) => { + for (i, v) in items.iter().enumerate() { + object_paths(v, format!("{at}/{i}"), out); + } + } + _ => {} + } +} + +/// `value` rendered with the first key of the object at `path` written twice, same value both. +fn with_duplicate_key(value: &Value, path: &str) -> Option { + const MARKER: &str = "__duplicate_marker__"; + let (key, repeated) = value.pointer(path)?.as_object()?.iter().next()?; + let duplicate = format!("{}:{}", Value::String(key.clone()), repeated); + let mut marked = value.clone(); + marked + .pointer_mut(path)? + .as_object_mut()? + .insert(MARKER.into(), json!(0)); + Some( + marked + .to_string() + .replacen(&format!("\"{MARKER}\":0"), &duplicate, 1), + ) +} + +#[rstest] +fn every_op_routes_to_its_own_variant() { + for (name, request) in fixture() { + let parsed = parse(&request).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!( + routes_to(&parsed, request["op"].as_str().unwrap()), + "{name} routed to another op" + ); + } +} + +#[rstest] +fn the_fixture_names_every_op() { + let mut named: Vec = fixture() + .into_iter() + .map(|(_, r)| r["op"].as_str().unwrap().to_owned()) + .collect(); + named.sort(); + named.dedup(); + let mut ops = OPS.to_vec(); + ops.sort(); + assert_eq!(named, ops); +} + +#[proptest] +fn key_order_and_whitespace_never_change_the_parse(which: usize, seed: u64) { + let request = request(which); + let raw = render(&request, &mut Lcg(seed)); + let parsed = parse_raw(&raw); + prop_assert!(parsed.is_ok(), "{raw}: {:?}", parsed.err()); + prop_assert!( + routes_to(&parsed.unwrap(), request["op"].as_str().unwrap()), + "{raw}" + ); +} + +#[proptest] +fn a_repeated_key_in_any_object_is_refused(which: usize, pick: usize) { + let request = request(which); + let mut paths = Vec::new(); + object_paths(&request, String::new(), &mut paths); + let path = &paths[pick % paths.len()]; + prop_assume!(!request + .pointer(path) + .unwrap() + .as_object() + .unwrap() + .is_empty()); + let raw = with_duplicate_key(&request, path).unwrap(); + prop_assert!( + matches!(parse_raw(&raw), Err(RequestError::Json(_))), + "accepted a repeated key at {path:?}: {raw}" + ); +} + +fn launch_with(field: &str, value: Value) -> Value { + let mut request = fixture() + .into_iter() + .find(|(name, _)| name == "vm_launch") + .unwrap() + .1; + request[field] = value; + request +} + +fn volume_with_thin_id(id: Value) -> Value { + json!({"op": "thin_volume_create", "pool": "hyper-pool", "thin_id": id, + "name": "hyper-v1", "origin": "/dev/mapper/hyper-img"}) +} + +#[proptest] +fn a_net_id_is_accepted_exactly_below_the_transit_capacity(#[strategy(0u32..=70_000)] id: u32) { + let accepted = parse(&launch_with("net_id", json!(id))).is_ok(); + prop_assert_eq!(accepted, id < 16_384, "net_id {}", id); +} + +#[proptest] +fn a_thin_id_is_accepted_exactly_within_24_bits( + #[strategy(prop_oneof![0u64..1 << 25, (1u64 << 24) - 4..(1 << 24) + 4, any::()])] id: u64, +) { + let accepted = parse(&volume_with_thin_id(json!(id))).is_ok(); + prop_assert_eq!(accepted, id < 1 << 24, "thin_id {}", id); +} + +#[rstest] +#[case::uid_max(launch_with("uid", json!(u32::MAX)), true)] +#[case::uid_past_u32(launch_with("uid", json!(u64::from(u32::MAX) + 1)), false)] +#[case::gid_zero_parses_and_is_left_to_the_range_check(launch_with("gid", json!(0)), true)] +#[case::thin_id_max(volume_with_thin_id(json!((1u64 << 24) - 1)), true)] +#[case::limits_empty(launch_with("limits", json!({})), true)] +#[case::limits_null_entries(launch_with("limits", json!({"memory_max": null, "cpu_max": null})), true)] +#[case::memory_max_u64(launch_with("limits", json!({"memory_max": u64::MAX})), true)] +#[case::memory_max_zero(launch_with("limits", json!({"memory_max": 0})), false)] +#[case::cpu_quota_zero(launch_with("limits", json!({"cpu_max": {"quota_us": 0, "period_us": 1}})), false)] +#[case::cpu_period_zero(launch_with("limits", json!({"cpu_max": {"quota_us": 1, "period_us": 0}})), false)] +#[case::cpu_max_missing_period(launch_with("limits", json!({"cpu_max": {"quota_us": 1}})), false)] +#[case::vm_id_64_chars(launch_with("id", json!("a".repeat(64))), true)] +#[case::vm_id_65_chars(launch_with("id", json!("a".repeat(65))), false)] +#[case::vm_id_inner_dash(launch_with("id", json!("v-1")), true)] +#[case::vm_id_non_ascii_digit(launch_with("id", json!("v\u{0661}")), false)] +#[case::device_traversing_mapper(launch_with("device", json!("/dev/mapper/../sda1")), false)] +fn bounded_fields_are_accepted_exactly_up_to_their_limit( + #[case] request: Value, + #[case] accepted: bool, +) { + assert_eq!( + parse(&request).is_ok(), + accepted, + "{request}: {:?}", + parse(&request).err() + ); +} From 67c07c6dc0c52418eb7de7ad59161aca147312ff Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sat, 14 Nov 2026 11:13:07 +0000 Subject: [PATCH 2/6] test: adversarial relay, scheduler, pool, wire and install suites Property and example tests for the agent relay cap and dialer, scheduler placement, id pools, meter rebase, page tokens, socket pins, kernel trust, the Elixir/helper wire, tarball and temp-dir cleanup, firecracker.install and live guest network isolation. try_run_contract needs the started app, so it is tagged :integration. --- .../accumulator_rebase_properties_test.exs | 51 ++++ test/e2e/hostile_guest_network_test.exs | 86 +++++++ test/e2e/vm_firewall_test.exs | 96 ++++++++ .../cluster/scheduler_properties_test.exs | 196 ++++++++++++++++ .../page_forged_token_properties_test.exs | 70 ++++++ .../agent/relay_budget_scope_test.exs | 97 ++++++++ .../agent/relay_cap_properties_test.exs | 220 ++++++++++++++++++ .../agent/relay_dialer_properties_test.exs | 95 ++++++++ test/hyper/node/fire_vmm/agent_relay_test.exs | 145 ++++++++++++ .../jailer_socket_pin_properties_test.exs | 115 +++++++++ .../provider_trust_properties_test.exs | 138 +++++++++++ test/hyper/node/id_pool_properties_test.exs | 93 ++++++++ test/hyper/node/try_run_contract_test.exs | 98 ++++++++ .../suid_helper/wire_properties_test.exs | 87 +++++++ test/mix/tasks/firecracker.install_test.exs | 72 ++++++ test/redist/targz_replace_test.exs | 95 ++++++++ test/sys/tmp_remove_properties_test.exs | 136 +++++++++++ 17 files changed, 1890 insertions(+) create mode 100644 test/controls/accumulator_rebase_properties_test.exs create mode 100644 test/e2e/hostile_guest_network_test.exs create mode 100644 test/e2e/vm_firewall_test.exs create mode 100644 test/hyper/cluster/scheduler_properties_test.exs create mode 100644 test/hyper/grpc/page_forged_token_properties_test.exs create mode 100644 test/hyper/node/fire_vmm/agent/relay_budget_scope_test.exs create mode 100644 test/hyper/node/fire_vmm/agent/relay_cap_properties_test.exs create mode 100644 test/hyper/node/fire_vmm/agent/relay_dialer_properties_test.exs create mode 100644 test/hyper/node/fire_vmm/agent_relay_test.exs create mode 100644 test/hyper/node/fire_vmm/jailer_socket_pin_properties_test.exs create mode 100644 test/hyper/node/fire_vmm/vm_linux/provider_trust_properties_test.exs create mode 100644 test/hyper/node/id_pool_properties_test.exs create mode 100644 test/hyper/node/try_run_contract_test.exs create mode 100644 test/hyper/suid_helper/wire_properties_test.exs create mode 100644 test/mix/tasks/firecracker.install_test.exs create mode 100644 test/redist/targz_replace_test.exs create mode 100644 test/sys/tmp_remove_properties_test.exs diff --git a/test/controls/accumulator_rebase_properties_test.exs b/test/controls/accumulator_rebase_properties_test.exs new file mode 100644 index 00000000..47d1dcc9 --- /dev/null +++ b/test/controls/accumulator_rebase_properties_test.exs @@ -0,0 +1,51 @@ +defmodule Controls.AccumulatorRebasePropertiesTest do + @moduledoc """ + Laws for `Controls.Accumulator.rebase/1`, the reboot hook (#134): the + counter's source is known to have been recreated at zero. + + * rebasing never changes what has accrued but not yet been flushed — a + reboot between flushes loses nothing; + * rebasing is idempotent; + * after a rebase, the first reading accrues in full, whatever the old + counter last read (above or below it) and whether or not a flush ran. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Controls.Accumulator + alias Unit.Time + + defp readings, do: list_of(integer(0..1_000_000_000), min_length: 1) + + defp fold(acc, values), do: Enum.reduce(values, acc, &Accumulator.observe(&2, Time.us(&1))) + + defp total_us(acc), do: Time.as_us(Accumulator.total(acc)) + + property "a rebase keeps everything accrued since the last flush" do + check all(values <- readings()) do + acc = fold(Accumulator.new(Time.zero()), values) + assert total_us(Accumulator.rebase(acc)) == total_us(acc) + end + end + + property "rebasing twice is rebasing once" do + check all(values <- readings(), next <- integer(0..1_000_000_000)) do + once = values |> then(&fold(Accumulator.new(Time.zero()), &1)) |> Accumulator.rebase() + twice = Accumulator.rebase(once) + + assert total_us(Accumulator.observe(twice, Time.us(next))) == + total_us(Accumulator.observe(once, Time.us(next))) + end + end + + property "the first reading after a rebase accrues in full" do + check all(values <- readings(), flush? <- boolean(), next <- integer(0..1_000_000_000)) do + acc = fold(Accumulator.new(Time.zero()), values) + acc = if flush?, do: Accumulator.flush(acc), else: acc + + rebased = acc |> Accumulator.rebase() |> Accumulator.observe(Time.us(next)) + + assert total_us(rebased) == total_us(acc) + next + end + end +end diff --git a/test/e2e/hostile_guest_network_test.exs b/test/e2e/hostile_guest_network_test.exs new file mode 100644 index 00000000..8b113533 --- /dev/null +++ b/test/e2e/hostile_guest_network_test.exs @@ -0,0 +1,86 @@ +defmodule Hyper.E2e.HostileGuestNetworkTest do + @moduledoc """ + A root shell inside a real guest, turned against the host it runs on. + + Laws (docs/cookbook/networking.md, Security): + + * a guest opens no connection to any address the host holds, over IPv4 or + IPv6, on any interface; + * a guest never reaches the provider's metadata service; + * while it still reaches the internet, so the refusals above are the + firewall and not a guest without a network. + """ + use ExUnit.Case, async: false + + import Hyper.E2e + + @moduletag :integration + @moduletag timeout: :timer.minutes(10) + + @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19") + @connect_timeout_s 3 + + setup_all do + {:ok, img_id} = Hyper.Img.OciLoader.load(@image) + {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) + on_exit(fn -> Hyper.Node.stop_image_vm(vm) end) + {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"]) + %{vm: vm} + end + + defp host_addresses do + {:ok, ifaddrs} = :inet.getifaddrs() + + for {_name, opts} <- ifaddrs, + {:addr, addr} <- opts, + not loopback?(addr), + not link_local?(addr), + uniq: true, + do: addr + end + + defp loopback?({127, _, _, _}), do: true + defp loopback?({0, 0, 0, 0, 0, 0, 0, 1}), do: true + defp loopback?(_), do: false + + defp link_local?({169, 254, _, _}), do: true + defp link_local?({a, _, _, _, _, _, _, _}), do: Bitwise.band(a, 0xFFC0) == 0xFE80 + defp link_local?(_), do: false + + defp guest_tcp(vm, addr, port) do + {:ok, %{exit_code: code}} = + await_exec(vm, [ + "/usr/bin/nc", + "-z", + "-w", + "#{@connect_timeout_s}", + to_string(:inet.ntoa(addr)), + "#{port}" + ]) + + code + end + + test "the guest opens no connection to any host address", %{vm: vm} do + {:ok, listener} = + :gen_tcp.listen(0, [:inet6, {:ipv6_v6only, false}, active: false, reuseaddr: true]) + + on_exit(fn -> :gen_tcp.close(listener) end) + {:ok, port} = :inet.port(listener) + addrs = host_addresses() + assert addrs != [] + + for addr <- addrs, do: guest_tcp(vm, addr, port) + + assert :gen_tcp.accept(listener, 1_000) == {:error, :timeout}, + "a guest connection reached the host on one of #{inspect(addrs)}" + end + + test "the guest never reaches the metadata service", %{vm: vm} do + assert guest_tcp(vm, {169, 254, 169, 254}, 80) != 0 + end + + test "the guest still reaches the internet", %{vm: vm} do + assert guest_tcp(vm, {1, 1, 1, 1}, 53) == 0 + end +end diff --git a/test/e2e/vm_firewall_test.exs b/test/e2e/vm_firewall_test.exs new file mode 100644 index 00000000..d0464982 --- /dev/null +++ b/test/e2e/vm_firewall_test.exs @@ -0,0 +1,96 @@ +defmodule Hyper.E2e.VmFirewallTest do + @moduledoc """ + `Hyper.set_vm_firewall/2` against two real guests on one node. + + Laws (docs/cookbook/networking.md, Security): + + * a VM's inbound ports are closed until the operator opens them, even to + another VM on the same host reaching it over its public IPv6 address; + * opening a port admits new connections, and closing it again refuses them; + * an outbound-blocked port never leaves the VM, and unblocking it restores it. + """ + use ExUnit.Case, async: false + + import Hyper.E2e + + alias Hyper.Vm.Firewall + + @moduletag :integration + @moduletag timeout: :timer.minutes(10) + + @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19") + @port 8080 + + setup_all do + {:ok, img_id} = Hyper.Img.OciLoader.load(@image) + vms = for _ <- 1..2, do: boot(img_id) + on_exit(fn -> Enum.each(vms, &Hyper.Node.stop_image_vm/1) end) + [client, server] = vms + %{client: client, server: server, server_ip6: ip6(server)} + end + + defp boot(img_id) do + {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) + {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"]) + vm + end + + defp ip6(vm) do + {:ok, %{stdout: cmdline}} = await_exec(vm, ["/bin/cat", "/proc/cmdline"]) + [_, ip6] = Regex.run(~r/\bhyper\.ip6=(\S+)/, cmdline) + ip6 + end + + defp listen(vm) do + {:ok, %{exit_code: 0}} = + await_exec(vm, [ + "/bin/sh", + "-c", + "[ -e /run/listening ] || { touch /run/listening; " <> + "(setsid sh -c 'while :; do nc -l -p #{@port}; sleep 0.1; done' " <> + "/dev/null 2>&1 &); }" + ]) + + :ok + end + + defp reaches?(vm, host, port) do + {:ok, %{exit_code: code}} = + await_exec(vm, ["/usr/bin/nc", "-z", "-w", "3", host, "#{port}"]) + + code == 0 + end + + defp set!(vm, inbound, outbound_blocked) do + {:ok, firewall} = Firewall.new(inbound, outbound_blocked) + assert Hyper.set_vm_firewall(Hyper.id(vm), firewall) == :ok + end + + test "an inbound port opens and closes for another VM on the host", ctx do + listen(ctx.server) + + closed = reaches?(ctx.client, ctx.server_ip6, @port) + set!(ctx.server, [{:tcp, @port..@port//1}], []) + opened = reaches?(ctx.client, ctx.server_ip6, @port) + set!(ctx.server, [], []) + reclosed = reaches?(ctx.client, ctx.server_ip6, @port) + + assert {closed, opened, reclosed} == {false, true, false} + end + + test "a guest reaches its own listener, so a refusal is the firewall", ctx do + listen(ctx.server) + + assert reaches?(ctx.server, "127.0.0.1", @port) + end + + test "an outbound-blocked port never leaves the VM until unblocked", ctx do + before = reaches?(ctx.client, "1.1.1.1", 53) + set!(ctx.client, [], [{:tcp, 53..53//1}]) + blocked = reaches?(ctx.client, "1.1.1.1", 53) + set!(ctx.client, [], []) + unblocked = reaches?(ctx.client, "1.1.1.1", 53) + + assert {before, blocked, unblocked} == {true, false, true} + end +end diff --git a/test/hyper/cluster/scheduler_properties_test.exs b/test/hyper/cluster/scheduler_properties_test.exs new file mode 100644 index 00000000..202f429f --- /dev/null +++ b/test/hyper/cluster/scheduler_properties_test.exs @@ -0,0 +1,196 @@ +defmodule Hyper.Cluster.SchedulerPropertiesTest do + @moduledoc """ + Laws for placement (`Hyper.Cluster.Scheduler`). + + * `colocation_score/2` equals an independent oracle — the bytes of the + VM's layers that are mounted on the node — is invariant under + reordering the layers, and is unchanged by adding a layer the node does + not hold; + * `candidates/2` lists every fitting node exactly once, most colocated + bytes first; + * `place/3` walks candidates in rank order, confirming each at most once: + it stops at the first confirmation (never confirming a second node, so a + VM is never duplicated), stops at the first node lost mid-call (which + may have booted the VM) with `:node_unreachable`, skips refusals, and + answers `:no_capacity` only when every candidate refused. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Cluster.Budget + alias Hyper.Cluster.Scheduler + alias Hyper.Node.Budget.NodeState + alias Hyper.Vm.Instance.Spec + alias Unit.{Bandwidth, Information} + + @moduletag :capture_log + + setup do + if Process.whereis(Budget.name()) do + :ok + else + _ = Application.ensure_all_started(:horde) + start_supervised!(Budget) + :ok + end + end + + defp layer_ids, do: list_of(string(:alphanumeric, min_length: 1, max_length: 6), max_length: 8) + + defp sized_layers do + gen all( + ids <- uniq_list_of(string(:alphanumeric, min_length: 1, max_length: 6), max_length: 8), + sizes <- list_of(integer(0..1_000_000), length: length(ids)) + ) do + Enum.zip(ids, Enum.map(sizes, &Information.bytes/1)) + end + end + + property "colocation score is the byte sum of the VM's layers the node has mounted" do + check all(layers <- sized_layers(), mounted <- layer_ids()) do + state = node_state(:oracle@nowhere, mounted) + mounted_set = MapSet.new(mounted) + + oracle = + for {id, size} <- layers, MapSet.member?(mounted_set, id), reduce: 0 do + acc -> acc + Information.as_bytes(size) + end + + assert Scheduler.colocation_score(state, layers) == oracle + assert Scheduler.colocation_score(state, Enum.reverse(layers)) == oracle + end + end + + property "a layer the node does not hold never changes its score" do + check all(layers <- sized_layers(), mounted <- layer_ids(), size <- integer(1..1_000_000)) do + state = node_state(:oracle@nowhere, mounted) + foreign = {"not-mounted-" <> Enum.join(mounted, "-"), Information.bytes(size)} + + assert Scheduler.colocation_score(state, [foreign | layers]) == + Scheduler.colocation_score(state, layers) + end + end + + property "candidates are every fitting node once, most colocated bytes first" do + check all(count <- integer(1..5), bytes <- uniq_list_of(integer(1..1_000), length: count)) do + nodes = Enum.map(1..count, &:"cand_#{&1}@nowhere") + states = Enum.zip(nodes, bytes) + layers = Enum.map(states, fn {node, b} -> {layer_for(node), Information.mib(b)} end) + + ranked = with_advertised(nodes, fn -> Scheduler.candidates(spec(), layers) end) + + expected = states |> Enum.sort_by(&elem(&1, 1), :desc) |> Enum.map(&elem(&1, 0)) + assert ranked == expected + end + end + + property "place confirms at most one node, in rank order, stopping at the first ok or loss" do + check all(outcomes <- list_of(member_of([:ok, :refuse, :lost]), min_length: 1, max_length: 5)) do + nodes = Enum.map(1..length(outcomes), &:"walk_#{&1}@nowhere") + plan = Map.new(Enum.zip(nodes, outcomes)) + layers = nodes |> Enum.with_index() |> Enum.map(&rank_layer/1) + test = self() + + attempt = fn node -> + send(test, {:attempt, node}) + + case Map.fetch!(plan, node) do + :ok -> {:ok, {:booted_on, node}} + :refuse -> {:error, :mem_exhausted} + :lost -> {:error, {:lost_mid_call, :noconnection}} + end + end + + result = with_advertised(nodes, fn -> Scheduler.place(spec(), layers, attempt) end) + attempted = drain_attempts() + + {walked, rest} = Enum.split_while(Enum.zip(nodes, outcomes), &(elem(&1, 1) == :refuse)) + + {expected_attempts, expected_result} = + case rest do + [{node, :ok} | _] -> + {Enum.map(walked, &elem(&1, 0)) ++ [node], {:ok, {node, {:booted_on, node}}}} + + [{node, :lost} | _] -> + {Enum.map(walked, &elem(&1, 0)) ++ [node], {:error, :node_unreachable}} + + [] -> + {nodes, {:error, :no_capacity}} + end + + assert attempted == expected_attempts + assert result == expected_result + end + end + + defp drain_attempts(acc \\ []) do + receive do + {:attempt, node} -> drain_attempts([node | acc]) + after + 0 -> Enum.reverse(acc) + end + end + + defp rank_layer({node, i}), do: {layer_for(node), Information.mib(1_000 - i)} + + defp layer_for(node), do: "layer-" <> Atom.to_string(node) + + defp with_advertised(nodes, fun) do + keys = + Enum.map(nodes, fn node -> + key = {:node, {__MODULE__, node}} + + {:ok, _} = + Horde.Registry.register(Budget.name(), key, node_state(node, [layer_for(node)])) + + key + end) + + await_fitting(length(nodes)) + + try do + fun.() + after + Enum.each(keys, &Horde.Registry.unregister(Budget.name(), &1)) + await_fitting(0) + end + end + + defp await_fitting(n, tries \\ 500) do + fitting = Enum.count(Budget.all_states(), &NodeState.fits?(&1, spec())) + + cond do + fitting == n -> :ok + tries == 0 -> flunk("budget registry never settled at #{n} fitting nodes (saw #{fitting})") + true -> await_fitting(n, tries - 1) + end + end + + # 64 TiB of memory: no real node's own advertisement fits, so only the + # states this suite registers are candidates. + defp spec do + %Spec{ + vcpus: 1, + mem: Information.tib(64), + disk: Information.gib(1), + disk_bw: Bandwidth.mibps(1), + net_bw: Bandwidth.mibps(1) + } + end + + defp node_state(node, layers) do + %NodeState{ + node: node, + mem_free: Information.tib(128), + disk_free: Information.gib(100), + cpu_load: 0.0, + cpu_capacity: 8, + cpu_max_load: 1.0, + disk_bw_load: Bandwidth.zero(), + disk_bw_ceiling: Bandwidth.gibps(1), + net_bw_load: Bandwidth.zero(), + net_bw_ceiling: Bandwidth.gibps(1), + layers: layers + } + end +end diff --git a/test/hyper/grpc/page_forged_token_properties_test.exs b/test/hyper/grpc/page_forged_token_properties_test.exs new file mode 100644 index 00000000..47fd2cc7 --- /dev/null +++ b/test/hyper/grpc/page_forged_token_properties_test.exs @@ -0,0 +1,70 @@ +defmodule Hyper.Grpc.PageForgedTokenPropertiesTest do + @moduledoc """ + `Hyper.Grpc.Page` under a hostile ListVms caller, who controls both + `page_token` (any string) and `page_size` (any int32), over the real + `Hyper.Grpc.Pageable.Vm`. + + Laws: + + * total: no token string and no page size makes `paginate/4` raise — it + answers a page or `:bad_page_token`; + * a forged token cannot read outside keyset order: whatever cursor `c` it + decodes to, the page is exactly the first `min(size, max)` VMs (default + for a non-positive size) whose id sorts strictly after `c`, so a forged + cursor neither skips nor re-lists a VM relative to that position; + * the next token a forged page returns resumes exactly after that page. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Grpc.Page + alias Hyper.Grpc.Pageable.Vm, as: Pageable + + defp vms do + gen all( + ids <- uniq_list_of(string(:printable, min_length: 1, max_length: 12), max_length: 60), + nodes <- list_of(member_of([:a@h, :b@h, :c@h]), length: length(ids)) + ) do + Enum.zip(ids, nodes) + end + end + + defp int32, do: one_of([integer(-5..1_200), member_of([-2_147_483_648, 2_147_483_647])]) + + defp effective(size) when size <= 0, do: Pageable.default_page_size() + defp effective(size), do: min(size, Pageable.max_page_size()) + + property "any token string and int32 size answers a page or :bad_page_token, never a raise" do + check all( + entries <- vms(), + size <- int32(), + token <- one_of([string(:printable), string(:utf8), binary()]) + ) do + case Page.paginate(entries, size, token, Pageable) do + {:ok, {page, next}} -> assert is_list(page) and is_binary(next) + {:error, :bad_page_token} -> :ok + end + end + end + + property "a forged cursor pages exactly the VMs sorting after it, and resumes after the page" do + check all(entries <- vms(), cursor <- binary(min_length: 1, max_length: 14), size <- int32()) do + token = Base.url_encode64(cursor, padding: false) + {:ok, {page, next}} = Page.paginate(entries, size, token, Pageable) + + after_cursor = entries |> Enum.filter(fn {id, _} -> id > cursor end) |> Enum.sort() + assert page == Enum.take(after_cursor, effective(size)) + + case next do + "" -> + assert length(after_cursor) <= effective(size) + + _ -> + {:ok, {following, _}} = Page.paginate(entries, size, next, Pageable) + + assert following == + after_cursor |> Enum.drop(length(page)) |> Enum.take(effective(size)) + end + end + end +end diff --git a/test/hyper/node/fire_vmm/agent/relay_budget_scope_test.exs b/test/hyper/node/fire_vmm/agent/relay_budget_scope_test.exs new file mode 100644 index 00000000..d55c2857 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent/relay_budget_scope_test.exs @@ -0,0 +1,97 @@ +defmodule Hyper.Node.FireVMM.Agent.RelayBudgetScopeTest do + @moduledoc """ + Scope of the relay's 16 MiB guest-to-host reply cap (#132): + + - it is per connection, not per relay: successive connections each get a + fresh budget, so a VM that has answered more than the cap in total over + several execs is still served; + - it caps only the guest-to-host direction: the host can send the guest + more than the cap on one connection. + """ + use ExUnit.Case, async: true + + alias Hyper.Node.FireVMM.Agent.Relay + alias Unit.Information + + @cap Information.as_bytes(Information.mib(16)) + @recv_chunk Information.as_bytes(Information.mib(2)) + + setup do + vsock_path = tmp_path("vsock") + listen_path = tmp_path("listen") + + {:ok, fc} = :socket.open(:local, :stream) + :ok = :socket.bind(fc, %{family: :local, path: vsock_path}) + :ok = :socket.listen(fc, 16) + + start_supervised!( + {Relay, %{vm_id: "vscope", vsock_uds: vsock_path, listen_path: listen_path}} + ) + + on_exit(fn -> + :socket.close(fc) + File.rm(vsock_path) + end) + + {:ok, %{fc: fc, listen_path: listen_path}} + end + + defp tmp_path(label), + do: Path.join(System.tmp_dir!(), "rscope-#{label}-#{System.unique_integer([:positive])}.sock") + + defp guest(fc, fun) do + parent = self() + + spawn_link(fn -> + {:ok, conn} = :socket.accept(fc) + {:ok, "CONNECT " <> _} = :socket.recv(conn) + :ok = :socket.send(conn, "OK 5\n") + fun.(conn, parent) + end) + end + + defp client(listen_path) do + {:ok, sock} = :socket.open(:local, :stream) + :ok = :socket.connect(sock, %{family: :local, path: listen_path}) + sock + end + + defp drain(sock, acc \\ []) do + case :socket.recv(sock, 0, 5_000) do + {:ok, data} -> drain(sock, [acc | data]) + {:error, reason} -> {reason, IO.iodata_to_binary(acc)} + end + end + + defp recv_exactly(sock, n, acc \\ []) + defp recv_exactly(_sock, 0, acc), do: IO.iodata_to_binary(acc) + + defp recv_exactly(sock, n, acc) do + {:ok, data} = :socket.recv(sock, min(n, @recv_chunk), 5_000) + recv_exactly(sock, n - byte_size(data), [acc | data]) + end + + test "each connection gets a fresh budget", %{fc: fc, listen_path: listen_path} do + per_conn = :crypto.strong_rand_bytes(div(@cap * 3, 4)) + + for _ <- 1..3 do + guest(fc, fn conn, _ -> + :ok = :socket.send(conn, per_conn) + :socket.close(conn) + end) + + assert drain(client(listen_path)) == {:closed, per_conn} + end + end + + test "the host can send the guest more than the cap", %{fc: fc, listen_path: listen_path} do + upload = :crypto.strong_rand_bytes(@cap + Information.as_bytes(Information.mib(4))) + + guest(fc, fn conn, parent -> + send(parent, {:guest_got, recv_exactly(conn, byte_size(upload))}) + end) + + :ok = :socket.send(client(listen_path), upload) + assert_receive {:guest_got, ^upload}, 10_000 + end +end diff --git a/test/hyper/node/fire_vmm/agent/relay_cap_properties_test.exs b/test/hyper/node/fire_vmm/agent/relay_cap_properties_test.exs new file mode 100644 index 00000000..1ba3617a --- /dev/null +++ b/test/hyper/node/fire_vmm/agent/relay_cap_properties_test.exs @@ -0,0 +1,220 @@ +defmodule Hyper.Node.FireVMM.Agent.RelayCapPropertiesTest do + @moduledoc """ + The relay's guest-to-host reply cap (#132), against a hostile guest that + chooses how many bytes to stream and how to chunk them: + + - **never over the cap** — the client never receives more than the cap, + however the guest chunks its stream; + - **transparent under the cap** — a guest that stays within the cap is + relayed byte-for-byte, and the connection stays open; + - **cumulative, not per-chunk** — the budget is spent across the whole + connection, so many small chunks overflow it exactly like one big one; + - **prefix integrity** — what the client receives is always a prefix of what + the guest sent: the cap truncates, it never reorders or corrupts; + - **the cap is inclusive** — a guest may send exactly the cap; one byte more + closes the connection after exactly the cap was delivered; + - **overflow tears down both sides** — the guest-side (vsock) connection is + closed too, so an overflowing guest cannot pin a host-side connection; + - **overflow is per connection** — it never disturbs another connection or + the relay's ability to accept new ones. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Node.FireVMM.Agent.Relay + alias Unit.Information + + @cap Information.as_bytes(Information.mib(16)) + @quiet_ms 300 + + defp tmp_path(label) do + Path.join(System.tmp_dir!(), "relaycap-#{label}-#{System.unique_integer([:positive])}.sock") + end + + setup do + vsock_path = tmp_path("vsock") + listen_path = tmp_path("listen") + + {:ok, fc} = :socket.open(:local, :stream) + :ok = :socket.bind(fc, %{family: :local, path: vsock_path}) + :ok = :socket.listen(fc, 16) + + {:ok, relay} = + Relay.start_link(%{vm_id: "vcap", vsock_uds: vsock_path, listen_path: listen_path}) + + true = Process.unlink(relay) + + on_exit(fn -> + if Process.alive?(relay), do: GenServer.stop(relay) + :socket.close(fc) + File.rm(vsock_path) + File.rm(listen_path) + end) + + %{fc: fc, listen_path: listen_path} + end + + defp guest(fc, script) do + test = self() + + spawn(fn -> + {:ok, conn} = :socket.accept(fc) + {:ok, "CONNECT " <> _} = :socket.recv(conn, 0, 2_000) + :ok = :socket.send(conn, "OK 5\n") + script.(conn) + send(test, {:guest_sees, guest_fate(conn)}) + end) + end + + defp guest_fate(conn) do + case :socket.recv(conn, 0, 3_000) do + {:ok, _} -> guest_fate(conn) + {:error, reason} -> reason + end + end + + defp connect(listen_path) do + {:ok, client} = :socket.open(:local, :stream) + :ok = :socket.connect(client, %{family: :local, path: listen_path}) + client + end + + defp drain(sock, acc) do + case :socket.recv(sock, 0, @quiet_ms) do + {:ok, data} -> drain(sock, [acc | data]) + {:error, :closed} -> {:closed, IO.iodata_to_binary(acc)} + {:error, :timeout} -> {:open, IO.iodata_to_binary(acc)} + end + end + + defp payload(total, seed) do + _ = :rand.seed(:exsss, seed) + :rand.bytes(total) + end + + defp chunked(bin, sizes), do: chunked(bin, sizes, sizes) + + defp chunked(<<>>, _sizes, _all), do: [] + defp chunked(bin, [], all), do: chunked(bin, all, all) + + defp chunked(bin, [n | sizes], all) do + take = min(n, byte_size(bin)) + <> = bin + [chunk | chunked(rest, sizes, all)] + end + + defp send_chunks(conn, chunks), do: Enum.reduce_while(chunks, :ok, &send_chunk(conn, &1, &2)) + + defp send_chunk(conn, chunk, :ok) do + case :socket.send(conn, chunk) do + :ok -> {:cont, :ok} + {:error, _} -> {:halt, :refused} + end + end + + property "the client gets a prefix of the guest's stream, never more than the cap", + %{fc: fc, listen_path: listen_path} do + check all( + total <- + one_of([ + integer(0..65_536), + integer((@cap - Information.as_bytes(Information.mib(1)))..@cap), + integer((@cap + 1)..(@cap + Information.as_bytes(Information.mib(2)))) + ]), + sizes <- + list_of(integer(512..Information.as_bytes(Information.mib(4))), min_length: 1), + seed <- {integer(1..1_000_000), integer(1..1_000_000), integer(1..1_000_000)}, + max_runs: 12 + ) do + sent = payload(total, seed) + chunks = chunked(sent, sizes) + guest(fc, fn conn -> send_chunks(conn, chunks) end) + client = connect(listen_path) + + {fate, got} = drain(client, []) + :socket.close(client) + + assert byte_size(got) <= @cap, "relayed #{byte_size(got)}B past the #{@cap}B cap" + assert got == binary_part(sent, 0, byte_size(got)), "received bytes are not a prefix" + + if total <= @cap do + assert {fate, byte_size(got)} == {:open, total} + else + assert fate == :closed, "#{total}B guest stream left the connection open" + end + end + end + + test "many small chunks overflow the cap the same as one big chunk", + %{fc: fc, listen_path: listen_path} do + chunk = :binary.copy(<<7>>, 4_096) + n = div(@cap, 4_096) + 1 + + guest(fc, fn conn -> send_chunks(conn, List.duplicate(chunk, n)) end) + client = connect(listen_path) + + assert {:closed, got} = drain(client, []) + assert byte_size(got) <= @cap + end + + test "exactly the cap is delivered and kept open; the next byte closes after the cap", + %{fc: fc, listen_path: listen_path} do + guest = + guest(fc, fn conn -> + :ok = :socket.send(conn, :binary.copy(<<1>>, @cap)) + + receive do + :one_more -> :ok = :socket.send(conn, <<2>>) + end + end) + + client = connect(listen_path) + assert {:open, got} = drain(client, []) + assert byte_size(got) == @cap + + send(guest, :one_more) + assert {:closed, ""} = drain(client, []) + assert_receive {:guest_sees, :closed}, 5_000 + end + + test "overflow closes the guest-side connection, not just the client's", + %{fc: fc, listen_path: listen_path} do + guest(fc, fn conn -> send_chunks(conn, [:binary.copy(<<3>>, @cap + 1)]) end) + client = connect(listen_path) + + assert {:closed, got} = drain(client, []) + assert byte_size(got) <= @cap + assert_receive {:guest_sees, :closed}, 5_000 + end + + test "one connection overflowing leaves a concurrent connection and the relay serving", + %{fc: fc, listen_path: listen_path} do + guest(fc, &echo/1) + calm = connect(listen_path) + assert ping(calm, "before") + + guest(fc, fn conn -> send_chunks(conn, [:binary.copy(<<4>>, @cap + 1)]) end) + flood = connect(listen_path) + assert {:closed, _} = drain(flood, []) + + assert ping(calm, "after") + guest(fc, &echo/1) + assert ping(connect(listen_path), "new") + end + + defp echo(conn) do + case :socket.recv(conn, 0, 10_000) do + {:ok, data} -> + :ok = :socket.send(conn, data) + echo(conn) + + {:error, _} -> + :ok + end + end + + defp ping(sock, msg) do + :ok = :socket.send(sock, msg) + :socket.recv(sock, byte_size(msg), 2_000) == {:ok, msg} + end +end diff --git a/test/hyper/node/fire_vmm/agent/relay_dialer_properties_test.exs b/test/hyper/node/fire_vmm/agent/relay_dialer_properties_test.exs new file mode 100644 index 00000000..be851083 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent/relay_dialer_properties_test.exs @@ -0,0 +1,95 @@ +defmodule Hyper.Node.FireVMM.Agent.RelayDialerPropertiesTest do + @moduledoc """ + `RelayDialer.dial/3`'s handshake refusals (#132), over arbitrary reply lines + the vsock UDS can answer `CONNECT` with: + + - **length bound** — a line whose content (before the LF) is 64 bytes or more + is always `{:error, :vsock_line_too_long}`, whatever its bytes; a shorter + one is never refused for length; + - **exact acceptance** — a short line is accepted iff it starts with the + exact bytes `"OK "`; any other short line is `{:error, {:vsock_no_ok, + line}}` carrying the line verbatim (case, leading space, missing space and + NUL/non-UTF-8 bytes do not pass); + - **no over-read** — on acceptance every byte after the LF is left on the + socket for the caller. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Node.FireVMM.Agent.RelayDialer + + @max_line_bytes 64 + + setup do + path = Path.join(System.tmp_dir!(), "dialprop-#{System.unique_integer([:positive])}.sock") + {:ok, srv} = :socket.open(:local, :stream) + :ok = :socket.bind(srv, %{family: :local, path: path}) + :ok = :socket.listen(srv, 4) + + on_exit(fn -> + :socket.close(srv) + File.rm(path) + end) + + %{path: path, srv: srv} + end + + defp answer(srv, bytes) do + spawn_link(fn -> + {:ok, conn} = :socket.accept(srv) + {:ok, "CONNECT 1024\n"} = :socket.recv(conn, 0, 2_000) + _ = :socket.send(conn, bytes) + Process.sleep(:infinity) + end) + end + + defp no_lf(gen), do: map(gen, &:binary.replace(&1, "\n", "x", [:global])) + + defp line do + one_of([ + no_lf(binary(max_length: @max_line_bytes + 8)), + map(no_lf(binary(max_length: @max_line_bytes)), &("OK " <> &1)), + member_of(["ok 5", " OK 5", "OK5", "OK", "OK\t5", "OK\0 5", <<"OK", 0xC3, " 5">>]), + map( + integer((@max_line_bytes - 2)..(@max_line_bytes + 2)), + &("OK " <> :binary.copy("5", &1 - 3)) + ) + ]) + end + + property "a line is refused for length iff it is 64 bytes or more, else decided by its OK prefix", + %{path: path, srv: srv} do + check all(line <- line(), tail <- binary(max_length: 32), max_runs: 200) do + answer(srv, line <> "\n" <> tail) + result = RelayDialer.dial(path, 1024, 1_000) + + cond do + byte_size(line) >= @max_line_bytes -> + assert result == {:error, :vsock_line_too_long} + + String.starts_with?(line, "OK ") -> + assert {:ok, sock} = result + + if tail != "" do + assert :socket.recv(sock, byte_size(tail), 1_000) == {:ok, tail} + end + + :socket.close(sock) + + true -> + assert result == {:error, {:vsock_no_ok, line}} + end + end + end + + property "an unterminated line of 64 bytes or more is refused without waiting for the deadline", + %{path: path, srv: srv} do + check all(junk <- no_lf(binary(min_length: @max_line_bytes, max_length: 4_096)), max_runs: 30) do + answer(srv, junk) + {us, result} = :timer.tc(fn -> RelayDialer.dial(path, 1024, 5_000) end) + + assert result == {:error, :vsock_line_too_long} + assert us < 1_000_000, "waited #{div(us, 1_000)}ms on an already-overlong line" + end + end +end diff --git a/test/hyper/node/fire_vmm/agent_relay_test.exs b/test/hyper/node/fire_vmm/agent_relay_test.exs new file mode 100644 index 00000000..60b56884 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent_relay_test.exs @@ -0,0 +1,145 @@ +defmodule Hyper.Node.FireVMM.AgentRelayTest do + @moduledoc """ + `Agent.exec/3` through a real `Relay` and a fake Firecracker vsock that + splices each `CONNECT` to an in-process gRPC guest: + + - a reply under the relay's 16 MiB cap round-trips intact: the cap bounds + a hostile guest, it must not truncate or reject an honest large reply; + - a reply over the cap is an error that is not `:timeout`, and the relay + keeps serving: the next exec on the same VM succeeds; + - concurrent execs on one VM each get their own reply, never another + call's. + + `async: false`: `Hyper.Cfg.Toml.put_cache/1` is process-global. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Agent.V1.{ExecResponse, HealthResponse} + alias Hyper.Node.FireVMM.Agent + alias Hyper.Node.FireVMM.Agent.Relay + alias Unit.Information + + @vm_id "vrelayexec" + + defmodule Guest do + @moduledoc false + use GRPC.Server, service: Hyper.Agent.V1.GuestAgent.Service + + def exec(%{argv: ["/bin/head", bytes]}, _stream), + do: %ExecResponse{exit_code: 0, stdout: :binary.copy("y", String.to_integer(bytes))} + + def exec(%{argv: ["/bin/echo" | rest]}, _stream) do + Process.sleep(:rand.uniform(50)) + %ExecResponse{exit_code: 0, stdout: Enum.join(rest, " ")} + end + + def health(_req, _stream), do: %HealthResponse{ok: true} + end + + setup_all do + {:ok, _} = Application.ensure_all_started(:grpc) + {:ok, _} = Application.ensure_all_started(:cowboy) + original = Hyper.Cfg.Toml.reload() + tmp = Path.join(System.tmp_dir!(), "agent-relay-#{System.unique_integer([:positive])}") + :ok = Hyper.Cfg.Toml.put_cache(Map.put(original, "work_dir", tmp)) + dir = Hyper.Cfg.Dirs.socket_dir() + File.mkdir_p!(dir) + + guest_path = Path.join(dir, "guest-#{@vm_id}.sock") + vsock_path = Path.join(dir, "vsock-#{@vm_id}.sock") + {:ok, _pid, _port} = GRPC.Server.start(Guest, 0, adapter_opts: [ip: {:local, guest_path}]) + + {:ok, fc} = :socket.open(:local, :stream) + :ok = :socket.bind(fc, %{family: :local, path: vsock_path}) + :ok = :socket.listen(fc, 64) + firecracker = spawn(fn -> fake_firecracker(fc, guest_path) end) + + {:ok, relay} = + Relay.start_link(%{ + vm_id: @vm_id, + vsock_uds: vsock_path, + listen_path: Agent.relay_socket_path(@vm_id) + }) + + true = Process.unlink(relay) + + on_exit(fn -> + GenServer.stop(relay) + Process.exit(firecracker, :kill) + GRPC.Server.stop(Guest) + Hyper.Cfg.Toml.put_cache(original) + File.rm_rf(tmp) + end) + + :ok + end + + defp fake_firecracker(listen, guest_path) do + with {:ok, vsock} <- :socket.accept(listen) do + splice(vsock, guest_path) + fake_firecracker(listen, guest_path) + end + end + + defp splice(vsock, guest_path) do + spawn(fn -> + {:ok, "CONNECT " <> _} = :socket.recv(vsock) + :ok = :socket.send(vsock, "OK 5\n") + {:ok, guest} = :socket.open(:local, :stream) + :ok = :socket.connect(guest, %{family: :local, path: guest_path}) + _ = spawn(fn -> pipe(guest, vsock) end) + pipe(vsock, guest) + end) + end + + defp pipe(from, to) do + with {:ok, data} <- :socket.recv(from), + :ok <- :socket.send(to, data) do + pipe(from, to) + else + _ -> + :socket.close(from) + :socket.close(to) + end + end + + test "an honest reply under the cap round-trips intact" do + for mib <- [1, 5, 15] do + bytes = Information.as_bytes(Information.mib(mib)) + + assert {:ok, %{exit_code: 0, stdout: stdout}} = + Agent.exec(@vm_id, ["/bin/head", "#{bytes}"], timeout: 20_000) + + assert {mib, byte_size(stdout), stdout == :binary.copy("y", bytes)} == {mib, bytes, true} + end + end + + test "a reply over the cap is a non-timeout error and the relay keeps serving" do + over = Information.as_bytes(Information.mib(17)) + + assert {:error, reason} = Agent.exec(@vm_id, ["/bin/head", "#{over}"], timeout: 20_000) + assert reason != :timeout + + assert {:ok, %{stdout: "still here"}} = + Agent.exec(@vm_id, ["/bin/echo", "still", "here"], timeout: 5_000) + end + + property "concurrent execs on one VM each get their own reply" do + check all( + words <- + uniq_list_of(string(:alphanumeric, min_length: 1), min_length: 2, max_length: 16), + max_runs: 10 + ) do + results = + words + |> Task.async_stream(&Agent.exec(@vm_id, ["/bin/echo", &1], timeout: 10_000), + max_concurrency: length(words), + timeout: 15_000 + ) + |> Enum.map(fn {:ok, {:ok, %{stdout: out}}} -> out end) + + assert results == words + end + end +end diff --git a/test/hyper/node/fire_vmm/jailer_socket_pin_properties_test.exs b/test/hyper/node/fire_vmm/jailer_socket_pin_properties_test.exs new file mode 100644 index 00000000..50b65b17 --- /dev/null +++ b/test/hyper/node/fire_vmm/jailer_socket_pin_properties_test.exs @@ -0,0 +1,115 @@ +defmodule Hyper.Node.FireVMM.JailerSocketPinPropertiesTest do + @moduledoc """ + The BEAM-side half of the socket-pin contract from #135. + + Laws: + + * wire contract with the helper: for every work dir and VM id, `host_socket/1` + and `host_vsock/1` are exactly the pins the helper's claim ops create, + `/socks/fc--api.socket` and `/socks/fc--vsock.sock` + (Rust: `Config::socket_dir` joined with `SocketPin::name`); + * the BEAM never dials a jail path: neither pin lies under `chroot_base/0`; + * naming is injective: no two `(id, socket)` pairs, nor a pin and a relay + socket, share a path, so one VM's teardown or claim can never hit another's; + * reclaim sweeps exactly what Hyper names: after a reboot every Firecracker pin + and relay socket for any set of VMs is removed, and every look-alike survives. + + `async: false` because the work dir is set through `Hyper.Cfg.Toml.put_cache/1`, + which is process-global. + """ + + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Node.FireVMM.{Agent, Jailer} + alias Hyper.Node.Reclaim + + setup do + original = Hyper.Cfg.Toml.reload() + on_exit(fn -> Hyper.Cfg.Toml.put_cache(original) end) + {:ok, original: original} + end + + defp vm_id do + StreamData.string(Enum.concat(?a..?z, ?2..?7), min_length: 1, max_length: 16) + |> StreamData.map(&("v" <> &1)) + end + + defp work_dir do + StreamData.string(:alphanumeric, min_length: 1, max_length: 12) + |> StreamData.list_of(min_length: 1, max_length: 4) + |> StreamData.map(&Path.join(["/", "srv-prop" | &1])) + end + + defp with_work_dir(original, dir), + do: Hyper.Cfg.Toml.put_cache(Map.put(original, "work_dir", dir)) + + property "the BEAM dials exactly the pins the helper creates, never a jail path", + %{original: original} do + check all(dir <- work_dir(), id <- vm_id()) do + with_work_dir(original, dir) + + assert {Jailer.host_socket(id), Jailer.host_vsock(id)} == + {"#{dir}/socks/fc-#{id}-api.socket", "#{dir}/socks/fc-#{id}-vsock.sock"} + + chroot = Hyper.Cfg.Dirs.chroot_base() <> "/" + refute String.starts_with?(Jailer.host_socket(id), chroot) + refute String.starts_with?(Jailer.host_vsock(id), chroot) + end + end + + property "no two VMs or socket kinds share a socket path", %{original: original} do + with_work_dir(original, "/srv/hyper") + + check all(ids <- StreamData.uniq_list_of(vm_id(), min_length: 2, max_length: 12)) do + paths = + Enum.flat_map(ids, fn id -> + [Jailer.host_socket(id), Jailer.host_vsock(id), Agent.relay_socket_path(id)] + end) + + assert length(Enum.uniq(paths)) == 3 * length(ids) + end + end + + property "reclaim removes every pin and relay socket Hyper names and spares look-alikes", + %{original: original} do + check all( + ids <- StreamData.uniq_list_of(vm_id(), min_length: 1, max_length: 6), + max_runs: 40 + ) do + tmp = Path.join(System.tmp_dir!(), "hyper-pin-#{System.unique_integer([:positive])}") + with_work_dir(original, tmp) + dir = Hyper.Cfg.Dirs.socket_dir() + File.mkdir_p!(dir) + + named = + Enum.flat_map(ids, fn id -> + [Jailer.host_socket(id), Jailer.host_vsock(id), Agent.relay_socket_path(id)] + end) + + look_alikes = + Enum.flat_map(ids, fn id -> + [ + "fc-#{id}-api.socket.bak", + "fc-#{id}-vsock.sock~", + "fc-#{id}", + "#{id}-api.socket", + "fc-#{id}-api.sock", + "grpc-#{id}.socket" + ] + end) + |> Enum.map(&Path.join(dir, &1)) + + for path <- named ++ look_alikes, do: File.write!(path, "") + + try do + Reclaim.reclaim_sockets() + + assert dir |> File.ls!() |> Enum.map(&Path.join(dir, &1)) |> Enum.sort() == + Enum.sort(look_alikes) + after + File.rm_rf!(tmp) + end + end + end +end diff --git a/test/hyper/node/fire_vmm/vm_linux/provider_trust_properties_test.exs b/test/hyper/node/fire_vmm/vm_linux/provider_trust_properties_test.exs new file mode 100644 index 00000000..a91b08f6 --- /dev/null +++ b/test/hyper/node/fire_vmm/vm_linux/provider_trust_properties_test.exs @@ -0,0 +1,138 @@ +defmodule Hyper.Node.FireVMM.VmLinux.ProviderTrustPropertiesTest do + @moduledoc """ + Kernel trust in `Hyper.Node.FireVMM.VmLinux.Provider`: a cached kernel is + only booted when no jail could have rewritten it. + + Laws: + + * `usable?/2` oracle: for every permission mode (setuid/setgid/sticky + included) and every expected owner, a regular file is usable exactly when + it carries no group- or other-write bit and is owned by root or that + owner; with no known owner (`nil`) only root qualifies; + * a non-regular path (directory, FIFO, dangling or live symlink) is never + usable, and judging it never blocks; + * `install_state/3` oracle: with each build independently faithful, stale, + missing, loosened, or replaced by a symlink to a faithful copy, the state + is `:ok` iff every build is faithful, `:not_installed` iff none is, and + `:bad_install` otherwise. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Node.FireVMM.VmLinux.{Manifest, Provider} + + setup do + dir = Path.join(System.tmp_dir!(), "vmlinux-trust-#{System.unique_integer([:positive])}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf!(dir) end) + {:ok, dir: dir, me: File.stat!(dir).uid} + end + + property "usable?/2 holds exactly for root- or owner-owned files no group or other can write", + %{dir: dir, me: me} do + path = Path.join(dir, "vmlinux") + File.write!(path, "kernel") + + check all( + mode <- integer(0..0o7777), + owner <- member_of([me, me + 1, 0, nil]), + max_runs: 400 + ) do + File.chmod!(path, mode) + %File.Stat{mode: actual, uid: uid} = File.stat!(path) + + expected = + Bitwise.band(actual, 0o022) == 0 and (uid == 0 or uid == owner) + + assert Provider.usable?(path, owner) == expected, + "mode #{Integer.to_string(actual, 8)}, owner #{inspect(owner)}" + end + end + + test "a non-regular path is never usable and judging it never blocks", %{dir: dir, me: me} do + trusted = Path.join(dir, "trusted") + File.write!(trusted, "kernel") + File.chmod!(trusted, 0o644) + + fifo = Path.join(dir, "fifo") + {_, 0} = System.cmd("mkfifo", ["-m", "0644", fifo]) + + subdir = Path.join(dir, "subdir") + File.mkdir_p!(subdir) + File.chmod!(subdir, 0o755) + + live = Path.join(dir, "live-link") + File.ln_s!(trusted, live) + dangling = Path.join(dir, "dangling-link") + File.ln_s!(Path.join(dir, "nowhere"), dangling) + + verdicts = + for {name, path} <- [fifo: fifo, directory: subdir, live_link: live, dangling: dangling] do + task = Task.async(fn -> Provider.usable?(path, me) end) + {name, Task.yield(task, 2_000) || Task.shutdown(task, :brutal_kill)} + end + + assert verdicts == [ + fifo: {:ok, false}, + directory: {:ok, false}, + live_link: {:ok, false}, + dangling: {:ok, false} + ] + end + + defp build_condition, do: member_of([:faithful, :stale, :missing, :loose, :symlinked]) + + property "install_state/3 is :ok iff every build is faithful and :not_installed iff none is", + %{dir: dir, me: me} do + builds = Manifest.builds_for(:x86_64) + assert length(builds) > 1 + + check all(conditions <- list_of(build_condition(), length: length(builds)), max_runs: 100) do + case_dir = Path.join(dir, "case-#{System.unique_integer([:positive])}") + File.mkdir_p!(case_dir) + staged = Enum.zip_with(builds, conditions, &stage(case_dir, &1, &2)) + faithful = Enum.count(conditions, &(&1 == :faithful)) + + expected = + cond do + faithful == length(builds) -> :ok + faithful == 0 -> {:error, :not_installed} + true -> {:error, :bad_install} + end + + assert Provider.install_state(case_dir, staged, me) == expected, inspect(conditions) + end + end + + defp stage(dir, build, condition) do + content = "kernel #{build.asset}" + sha = Base.encode16(:crypto.hash(:sha256, content), case: :lower) + path = Path.join(dir, build.asset) + + case condition do + :faithful -> + write(path, content, 0o644) + + :stale -> + write(path, "previous release #{build.asset}", 0o644) + + :missing -> + :ok + + :loose -> + write(path, content, 0o664) + + :symlinked -> + real = Path.join(dir, "real-#{build.asset}") + write(real, content, 0o644) + File.ln_s!(real, path) + end + + %{build | sha256: sha} + end + + defp write(path, content, mode) do + File.write!(path, content) + File.chmod!(path, mode) + end +end diff --git a/test/hyper/node/id_pool_properties_test.exs b/test/hyper/node/id_pool_properties_test.exs new file mode 100644 index 00000000..3ad7ad7e --- /dev/null +++ b/test/hyper/node/id_pool_properties_test.exs @@ -0,0 +1,93 @@ +defmodule Hyper.Node.IdPoolPropertiesTest do + @moduledoc """ + Laws for the two per-VM id pools, `Hyper.Node.Users` (uid/gid) and + `Hyper.Node.Net` (network slot), which share one bump-pointer + FIFO + free-queue design (#134). Each id is a VM identity: handing a held id to a + second VM is a uid / netns collision between two guests. + + Over any interleaving of claims, explicit releases, owner binds and owner + deaths by well-behaved callers (each held id returned exactly once): + + * no id is ever handed out while it is still held; + * every id handed out lies in the configured range; + * a claim is refused (`:exhausted`) exactly when every id in the range is + held — a returned id is always reusable, so nothing leaks; + * a returned id is not reissued while a never-issued id remains. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Node.{Net, Users} + + @pools [users: {Users, {:new}, :free}, net: {Net, :claim, :release}] + + defp op do + one_of([ + constant(:claim), + tuple({constant(:release), integer(0..1_000)}), + tuple({constant(:bind_and_die), integer(0..1_000)}) + ]) + end + + for {name, {mod, claim_msg, release_msg}} <- @pools do + property "#{name}: never reissues a held id, never leaks one, stays in range" do + mod = unquote(mod) + claim_msg = unquote(Macro.escape(claim_msg)) + release_tag = unquote(release_msg) + + check all( + min <- integer(0..60_000), + span <- integer(0..12), + ops <- list_of(op(), max_length: 80) + ) do + max = min + span + pool = new_pool(mod, min, max) + + Enum.reduce(ops, {pool, [], MapSet.new()}, fn + :claim, {pool, held, ever} -> + {:reply, reply, pool} = mod.handle_call(claim_msg, {self(), make_ref()}, pool) + + case reply do + {:ok, id} -> + refute id in held, "#{inspect(mod)} reissued held id #{id}" + assert id in min..max + fresh_left? = MapSet.size(ever) < span + 1 + + if fresh_left?, + do: refute(MapSet.member?(ever, id), "reused #{id} while fresh ids remained") + + {pool, held ++ [id], MapSet.put(ever, id)} + + {:error, :exhausted} -> + assert length(held) == span + 1, "refused with only #{length(held)} held" + {pool, held, ever} + end + + {:release, _}, {pool, [], ever} -> + {pool, [], ever} + + {:release, pick}, {pool, held, ever} -> + id = Enum.at(held, rem(pick, length(held))) + {:noreply, pool} = mod.handle_cast({release_tag, id}, pool) + {pool, List.delete(held, id), ever} + + {:bind_and_die, _}, {pool, [], ever} -> + {pool, [], ever} + + {:bind_and_die, pick}, {pool, held, ever} -> + id = Enum.at(held, rem(pick, length(held))) + owner = spawn(fn -> :ok end) + {:reply, :ok, pool} = mod.handle_call({:bind, id, owner}, {self(), make_ref()}, pool) + ref = pool.owners |> Enum.find(fn {_ref, bound} -> bound == id end) |> elem(0) + {:noreply, pool} = mod.handle_info({:DOWN, ref, :process, owner, :normal}, pool) + {pool, List.delete(held, id), ever} + end) + end + end + end + + defp new_pool(Users, min, max), do: %Users.State{max: max, next: min} + + defp new_pool(Net, min, max), + do: %Net.State{host: :unused_by_the_id_pool, max: max, next: min} +end diff --git a/test/hyper/node/try_run_contract_test.exs b/test/hyper/node/try_run_contract_test.exs new file mode 100644 index 00000000..bd07bf81 --- /dev/null +++ b/test/hyper/node/try_run_contract_test.exs @@ -0,0 +1,98 @@ +defmodule Hyper.Node.TryRunContractTest do + @moduledoc """ + `Hyper.Node.try_run/3`'s no-leak contract (#134), against this node's real + `Hyper.Node.Budget.Hard` ledger: every VM `start_fun` boots is either + admitted (and held against its pid) or torn down through `stop_fun` — never + both, never neither; concurrent placements never admit past the node's hard + headroom; and once the admitted VMs die the headroom returns exactly to + where it started, so a refusal or an admission leaves no reservation behind. + """ + use ExUnit.Case, async: false + + alias Hyper.Node.Budget.Hard + alias Hyper.Vm.Instance.Spec + alias Unit.{Bandwidth, Information} + + @moduletag :capture_log + @moduletag :integration + @racers 6 + + test "concurrent placements never over-admit, and every refused VM is stopped" do + before = Hard.headroom() + spec = spec(Information.bytes(div(Information.as_bytes(before.mem), 3) + 1)) + test = self() + + stop = fn pid -> + send(test, {:stopped, pid}) + Process.exit(pid, :kill) + :ok + end + + results = + 1..@racers + |> Enum.map(fn _ -> + Task.async(fn -> + vm = spawn(fn -> Process.sleep(:infinity) end) + {vm, Hyper.Node.try_run(spec, fn -> {:ok, vm} end, stop)} + end) + end) + |> Task.await_many() + + stopped = drain_stopped() + {admitted, refused} = Enum.split_with(results, &match?({_, {:ok, _}}, &1)) + + assert length(admitted) in 1..2, inspect(results) + assert Enum.map(refused, &elem(&1, 0)) |> Enum.sort() == Enum.sort(stopped) + assert Enum.all?(admitted, fn {vm, {:ok, pid}} -> vm == pid and Process.alive?(vm) end) + refute Enum.any?(refused, fn {vm, _} -> Process.alive?(vm) end) + + Enum.each(admitted, fn {vm, _} -> Process.exit(vm, :kill) end) + assert eventually(fn -> Hard.headroom() == before end), "a reservation outlived its VM" + end + + test "a VM that failed to boot is never stopped, and reserves nothing" do + before = Hard.headroom() + test = self() + + assert Hyper.Node.try_run( + spec(Information.mib(1)), + fn -> {:error, :kernel_missing} end, + fn pid -> send(test, {:stopped, pid}) end + ) == {:error, :kernel_missing} + + refute_received {:stopped, _} + assert Hard.headroom() == before + end + + defp drain_stopped(acc \\ []) do + receive do + {:stopped, pid} -> drain_stopped([pid | acc]) + after + 0 -> acc + end + end + + defp eventually(fun, tries \\ 100) do + cond do + fun.() -> + true + + tries == 0 -> + false + + true -> + Process.sleep(20) + eventually(fun, tries - 1) + end + end + + defp spec(mem) do + %Spec{ + vcpus: 0.01, + mem: mem, + disk: Information.bytes(1), + disk_bw: Bandwidth.zero(), + net_bw: Bandwidth.zero() + } + end +end diff --git a/test/hyper/suid_helper/wire_properties_test.exs b/test/hyper/suid_helper/wire_properties_test.exs new file mode 100644 index 00000000..e07ab156 --- /dev/null +++ b/test/hyper/suid_helper/wire_properties_test.exs @@ -0,0 +1,87 @@ +defmodule Hyper.SuidHelper.WirePropertiesTest do + @moduledoc """ + The node and the helper agree on the wire, checked against the real stamped helper binary + (the one `Hyper.SuidHelper.Expected` was captured from), not a fixture. + + Laws: + + * every VM id `Hyper.Vm.Id.generate/0` mints is a VM id the helper parses, for both socket + claims; + * every firewall `Hyper.Vm.Firewall.new/2` accepts, up to `max_rules/0` in each direction + and over the whole `1..65535` port space, becomes a `vm_firewall` request the helper + parses; + * a request the helper refuses to parse surfaces as `{:error, {2, "invalid request: ..."}}`, + never as success or as another error. + + The helper runs unprivileged and not setuid here: a request it parses gets past the caller + gate (the binary's group is the runner's) and then fails raising euid, before touching + anything. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.SuidHelper + alias Hyper.Vm.Firewall + + @helper "native/target/*-unknown-linux-musl/release/hyper-suidhelper" + |> Path.wildcard() + |> List.first() + + @moduletag skip: if(@helper, do: false, else: "no stamped helper; run mix compile") + + setup do + previous = Application.get_env(:hyper, Hyper.Cfg.Tools, []) + + Application.put_env( + :hyper, + Hyper.Cfg.Tools, + Keyword.put(previous, :suidhelper, Path.expand(@helper)) + ) + + on_exit(fn -> Application.put_env(:hyper, Hyper.Cfg.Tools, previous) end) + end + + defp parsed?({:error, {2, "invalid request" <> _}}), do: false + defp parsed?({:error, {2, _message}}), do: true + + defp rule do + gen all( + protocol <- member_of([:tcp, :udp]), + first <- one_of([constant(1), constant(65_535), integer(1..65_535)]), + last <- one_of([constant(first), constant(65_535), integer(first..65_535)]) + ) do + {protocol, first..last//1} + end + end + + defp rules do + one_of([ + list_of(rule(), max_length: 8), + list_of(rule(), length: Firewall.max_rules()) + ]) + end + + property "every minted VM id is one the helper parses" do + check all(id <- repeatedly(&Hyper.Vm.Id.generate/0), max_runs: 25) do + for claim <- [&SuidHelper.Vm.claim_api_socket/1, &SuidHelper.Vm.claim_vsock_socket/1] do + result = claim.(id) + assert parsed?(result), "#{id}: #{inspect(result)}" + end + end + end + + property "every firewall the node accepts is one the helper parses" do + check all(inbound <- rules(), outbound <- rules(), max_runs: 25) do + {:ok, firewall} = Firewall.new(inbound, outbound) + result = SuidHelper.Vm.firewall(Hyper.Vm.Id.generate(), firewall) + assert parsed?(result), inspect(result) + end + end + + test "a request the helper cannot parse is an invalid-request error, not success" do + for id <- ["v_1", "-v1", "", String.duplicate("a", 65), "v1/../x", "vé"] do + assert {:error, {2, "invalid request" <> _}} = SuidHelper.Vm.claim_api_socket(id), + inspect(id) + end + end +end diff --git a/test/mix/tasks/firecracker.install_test.exs b/test/mix/tasks/firecracker.install_test.exs new file mode 100644 index 00000000..8bd9ea3f --- /dev/null +++ b/test/mix/tasks/firecracker.install_test.exs @@ -0,0 +1,72 @@ +defmodule Mix.Tasks.Firecracker.InstallTest do + @moduledoc """ + `mix firecracker.install` against the real pinned release. + + Laws: + + * whatever already sits at `/firecracker` or `/jailer` - a + symlink or a hard link to someone else's file - is replaced, never + written through: the file it named keeps its bytes; + * each installed binary is a fresh regular file, mode `0755`, that runs as + the pinned version; + * reinstalling over a finished install succeeds and yields the same bytes. + """ + use ExUnit.Case, async: false + + @moduletag :external + @moduletag timeout: 300_000 + + setup do + previous = Mix.shell() + Mix.shell(Mix.Shell.Process) + prefix = Path.join(System.tmp_dir!(), "fc-install-#{System.unique_integer([:positive])}") + File.mkdir_p!(prefix) + + on_exit(fn -> + Mix.shell(previous) + File.rm_rf!(prefix) + end) + + {:ok, prefix: prefix} + end + + defp install(prefix), do: Mix.Tasks.Firecracker.Install.run(["--prefix", prefix]) + + defp assert_fresh_binary(path, victims) do + stat = File.lstat!(path) + assert stat.type == :regular + assert Bitwise.band(stat.mode, 0o7777) == 0o755 + refute stat.inode in Enum.map(victims, &File.stat!(&1).inode) + {out, 0} = System.cmd(path, ["--version"], stderr_to_stdout: true) + assert out =~ "v1.16.0" + end + + test "planted links at the install paths are replaced, never written through", %{ + prefix: prefix + } do + sym_victim = Path.join(prefix, "sym-victim") + hard_victim = Path.join(prefix, "hard-victim") + File.write!(sym_victim, "operator data") + File.write!(hard_victim, "other data") + File.ln_s!(sym_victim, Path.join(prefix, "firecracker")) + File.ln!(hard_victim, Path.join(prefix, "jailer")) + + install(prefix) + + assert File.read!(sym_victim) == "operator data" + assert File.read!(hard_victim) == "other data" + assert_fresh_binary(Path.join(prefix, "firecracker"), [sym_victim, hard_victim]) + assert_fresh_binary(Path.join(prefix, "jailer"), [sym_victim, hard_victim]) + end + + test "reinstalling over a finished install succeeds with the same bytes", %{prefix: prefix} do + install(prefix) + first = for b <- ~w(firecracker jailer), do: File.read!(Path.join(prefix, b)) + + install(prefix) + second = for b <- ~w(firecracker jailer), do: File.read!(Path.join(prefix, b)) + + assert :crypto.hash(:sha256, first) == :crypto.hash(:sha256, second) + for b <- ~w(firecracker jailer), do: assert_fresh_binary(Path.join(prefix, b), []) + end +end diff --git a/test/redist/targz_replace_test.exs b/test/redist/targz_replace_test.exs new file mode 100644 index 00000000..fda49d16 --- /dev/null +++ b/test/redist/targz_replace_test.exs @@ -0,0 +1,95 @@ +defmodule Redist.TargzReplaceTest do + @moduledoc """ + `Redist.Targz.install/3` into a `dest_dir` that already holds files - the + reinstall path `mix firecracker.install` relies on to never keep an old or + planted binary. + + Laws: + + * the archive wins: every entry lands with the archive's bytes, whatever + regular file previously sat at that path (stale release or plant); + * a planted symlink at an entry path is never written through - the + file it points at is unchanged, whether the install succeeds or refuses; + * an archive symlink cannot be used to write outside `dest_dir`. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Redist.Support.HttpServer + alias Redist.Targz + + setup do + {:ok, _} = Application.ensure_all_started(:req) + tmp = Path.join(System.tmp_dir!(), "targz-replace-#{System.unique_integer([:positive])}") + File.mkdir_p!(tmp) + on_exit(fn -> File.rm_rf!(tmp) end) + {:ok, server: HttpServer.start(), tmp: tmp} + end + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) + + defp targz(tmp, entries) do + path = Path.join(tmp, "build-#{System.unique_integer([:positive])}.tar.gz") + :ok = :erl_tar.create(String.to_charlist(path), entries, [:compressed]) + bytes = File.read!(path) + File.rm!(path) + bytes + end + + defp install(server, bytes, dest) do + url = HttpServer.put_file(server, "a-#{System.unique_integer([:positive])}.tar.gz", bytes) + Targz.install(url, sha256(bytes), dest) + end + + property "every entry lands with the archive's bytes over whatever file was there", + %{server: server, tmp: tmp} do + check all( + fresh <- binary(min_length: 1, max_length: 512), + stale <- binary(max_length: 512), + max_runs: 25 + ) do + dest = Path.join(tmp, "dest-#{System.unique_integer([:positive])}") + File.mkdir_p!(Path.join(dest, "release")) + File.write!(Path.join(dest, "release/firecracker"), stale) + + bytes = targz(tmp, [{~c"release/firecracker", fresh}]) + + assert install(server, bytes, dest) == :ok + assert File.read!(Path.join(dest, "release/firecracker")) == fresh + end + end + + test "a planted symlink at an entry path is never written through", %{ + server: server, + tmp: tmp + } do + victim = Path.join(tmp, "victim") + File.write!(victim, "operator data") + dest = Path.join(tmp, "dest") + File.mkdir_p!(dest) + File.ln_s!(victim, Path.join(dest, "firecracker")) + + _ = install(server, targz(tmp, [{~c"firecracker", "archive bytes"}]), dest) + + assert File.read!(victim) == "operator data" + end + + test "an archive symlink cannot be followed out of dest_dir", %{server: server, tmp: tmp} do + outside = Path.join(tmp, "outside") + File.mkdir_p!(outside) + src = Path.join(tmp, "src") + File.mkdir_p!(src) + File.ln_s!(outside, Path.join(src, "link")) + + archive = Path.join(tmp, "evil.tar.gz") + {:ok, tar} = :erl_tar.open(String.to_charlist(archive), [:write, :compressed]) + :ok = :erl_tar.add(tar, String.to_charlist(Path.join(src, "link")), ~c"link", []) + :ok = :erl_tar.add(tar, "pwned", ~c"link/escape.txt", []) + :ok = :erl_tar.close(tar) + + got = install(server, File.read!(archive), Path.join(tmp, "dest")) + + assert match?({:error, _}, got) + assert File.ls!(outside) == [] + end +end diff --git a/test/sys/tmp_remove_properties_test.exs b/test/sys/tmp_remove_properties_test.exs new file mode 100644 index 00000000..91ddb7cf --- /dev/null +++ b/test/sys/tmp_remove_properties_test.exs @@ -0,0 +1,136 @@ +defmodule Sys.TmpRemovePropertiesTest do + @moduledoc """ + Cleanup laws of `Sys.Tmp.with_tempdir/2` against trees an unpacked image can + leave behind (nested dirs of any mode down to 0000, files of any mode, and + symlinks pointing out of the tree at dirs and files with restrictive modes): + + * the temp dir is always gone afterwards, on return and on raise; + * nothing reachable only through a symlink is removed, and no symlink + target outside the tree has its mode changed (cleanup makes the tree + traversable without ever chmodding through a link); + * if `fun` swaps the temp dir itself for a symlink to an outside dir, + cleanup removes the link and leaves the outside dir and its contents + as they were. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + import Bitwise + + alias Sys.Tmp + + @moduletag :tmp_dir + + @dir_modes [0o000, 0o100, 0o200, 0o300, 0o400, 0o500, 0o555, 0o700] + @file_modes [0o000, 0o400, 0o644, 0o4755] + + defp tree do + leaf = + StreamData.one_of([ + StreamData.tuple({StreamData.constant(:file), StreamData.member_of(@file_modes)}), + StreamData.member_of([{:link, :outside_dir}, {:link, :outside_file}, {:link, :dangling}]) + ]) + + StreamData.tree(leaf, fn child -> + StreamData.tuple( + {StreamData.constant(:dir), StreamData.member_of(@dir_modes), + StreamData.list_of(child, max_length: 3)} + ) + end) + end + + defp outside(tmp) do + root = Path.join(tmp, "o#{System.unique_integer([:positive])}") + dir = Path.join(root, "dir") + File.mkdir_p!(dir) + File.write!(Path.join(dir, "keep"), "keep") + file = Path.join(root, "file") + File.write!(file, "outside") + File.chmod!(file, 0o400) + File.chmod!(dir, 0o500) + %{root: root, dir: dir, file: file} + end + + defp build!({:file, mode}, path, _out) do + File.write!(path, "x") + File.chmod!(path, mode) + end + + defp build!({:link, :outside_dir}, path, out), do: File.ln_s!(out.dir, path) + defp build!({:link, :outside_file}, path, out), do: File.ln_s!(out.file, path) + defp build!({:link, :dangling}, path, out), do: File.ln_s!(Path.join(out.root, "nope"), path) + + defp build!({:dir, mode, children}, path, out) do + File.mkdir!(path) + + children + |> Enum.with_index() + |> Enum.each(fn {child, i} -> build!(child, Path.join(path, "n#{i}"), out) end) + + File.chmod!(path, mode) + end + + defp assert_outside_intact(out) do + assert band(File.lstat!(out.dir).mode, 0o7777) == 0o500 + assert band(File.lstat!(out.file).mode, 0o7777) == 0o400 + assert File.read!(Path.join(out.dir, "keep")) == "keep" + assert File.read!(out.file) == "outside" + refute File.exists?(Path.join(out.root, "nope")) + end + + defp release(out) do + File.chmod!(out.dir, 0o700) + File.chmod!(out.file, 0o600) + end + + property "any image-shaped tree is removed and nothing behind a symlink is touched", %{ + tmp_dir: tmp + } do + check all(shape <- tree(), raise? <- StreamData.boolean(), max_runs: 150) do + out = outside(tmp) + + work = fn dir -> + build!(shape, Path.join(dir, "rootfs"), out) + if raise?, do: raise("boom"), else: dir + end + + dir = + if raise? do + holder = self() + + assert_raise RuntimeError, fn -> + Tmp.with_tempdir("tmp-prop", fn dir -> + send(holder, {:dir, dir}) + work.(dir) + end) + end + + assert_receive {:dir, dir} + dir + else + Tmp.with_tempdir("tmp-prop", work) + end + + assert {:error, :enoent} = File.lstat(dir), inspect(shape) + assert_outside_intact(out) + release(out) + end + end + + test "a temp dir swapped for a symlink to an outside dir is unlinked, not followed", %{ + tmp_dir: tmp + } do + out = outside(tmp) + + dir = + Tmp.with_tempdir("tmp-prop", fn dir -> + File.rmdir!(dir) + File.ln_s!(out.dir, dir) + dir + end) + + assert {:error, :enoent} = File.lstat(dir) + assert_outside_intact(out) + release(out) + end +end From ce51ae8de73d150fb8826328dfc5621bde8ab919 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 16 Nov 2026 02:00:58 +0000 Subject: [PATCH 3/6] ci(suidhelper): run the root suite on the runner's toolchain Under sudo, root's HOME is /root, so rustup installed a second nightly in /root/.rustup and CARGO pointed there; tests that rebuild the helper as the invoking user then failed to exec it with EACCES. --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faf6a518..9baa9cb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -313,7 +313,9 @@ jobs: # Root-gated cases are tagged by name: every test that needs root ends in # `_as_root` and self-skips when run unprivileged. We select them by that - # suffix (via sudo -E so the runner's cargo/rustup home and PATH are kept) + # suffix (via sudo -E so the runner's PATH is kept; RUSTUP_HOME explicitly, + # since root's HOME is /root and its own toolchain there is 0700, which the + # tests that rebuild the helper as the invoking user cannot exec) # so the root-only paths -- sys-test ok, fake-bin argv capture, mknod/chown # jail build -- actually run. A new root test is picked up automatically by # following the naming convention; no test-binary enumeration to maintain. @@ -325,7 +327,7 @@ jobs: # spawned helper's profile reach the collector. - name: Gated tests as root + coverage run: | - sudo -E env "PATH=$PATH" \ + sudo -E env "PATH=$PATH" "RUSTUP_HOME=$HOME/.rustup" \ cargo llvm-cov nextest \ -p hyper-suidhelper -p hyperrs-fs -p hyperrs-firecracker -p hyperrs-linux -p hyperrs-process \ --features hyper-suidhelper/insecure_mode \ From 80e6e676a9c7ad7e96cf29b2cdcc61bd354c5373 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 16 Nov 2026 02:01:44 +0000 Subject: [PATCH 4/6] ci(kvm): let VM traffic past the runner Docker's forward drop policy --- .github/scripts/provision-kvm-host.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh index c1916f7c..326e3caa 100755 --- a/.github/scripts/provision-kvm-host.sh +++ b/.github/scripts/provision-kvm-host.sh @@ -94,6 +94,12 @@ ipv6_prefix="${ipv6_prefix:-2001:db8::/64}" # carries IPv6 link-local addresses, so it must be on. sudo sysctl -qw net.ipv6.conf.all.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 sudo sysctl -qw net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 +# The runner's Docker sets the iptables forward policy to drop, which drops VM +# traffic too (install.md: allow forwarding to and from vm-veth-*). +for ipt in iptables ip6tables; do + sudo "$ipt" -I FORWARD -i 'vm-veth+' -j ACCEPT + sudo "$ipt" -I FORWARD -o 'vm-veth+' -j ACCEPT +done sudo mkdir -p /etc/hyper sudo tee /etc/hyper/config.toml >/dev/null < Date: Mon, 16 Nov 2026 02:15:15 +0000 Subject: [PATCH 5/6] test(suidhelper): build the helper in the invoker's cargo home, not root's --- native/suidhelper/tests/e2e/setuid_support.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/native/suidhelper/tests/e2e/setuid_support.rs b/native/suidhelper/tests/e2e/setuid_support.rs index 49efa488..1c44eab9 100644 --- a/native/suidhelper/tests/e2e/setuid_support.rs +++ b/native/suidhelper/tests/e2e/setuid_support.rs @@ -92,6 +92,9 @@ pub fn static_insecure_helper() -> PathBuf { if let Some(home) = home_of(uid) { cargo.env("HOME", home); } + // Root's cargo exports its own CARGO_HOME, which the invoker cannot write; + // without it cargo falls back to the invoker's $HOME/.cargo. + cargo.env_remove("CARGO_HOME"); cargo.uid(uid).gid(gid); } assert!( From 78950f5eb98e25208c1193b5364f5b41c476f2fd Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Mon, 16 Nov 2026 02:25:26 +0000 Subject: [PATCH 6/6] test(suidhelper): give the invoker's helper build its own target dir --- native/suidhelper/tests/e2e/setuid_support.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/native/suidhelper/tests/e2e/setuid_support.rs b/native/suidhelper/tests/e2e/setuid_support.rs index 1c44eab9..16c9e76c 100644 --- a/native/suidhelper/tests/e2e/setuid_support.rs +++ b/native/suidhelper/tests/e2e/setuid_support.rs @@ -95,6 +95,15 @@ pub fn static_insecure_helper() -> PathBuf { // Root's cargo exports its own CARGO_HOME, which the invoker cannot write; // without it cargo falls back to the invoker's $HOME/.cargo. cargo.env_remove("CARGO_HOME"); + // Root may have created target/ first (the coverage run does), so hand the + // build its own directories rather than asking the invoker to create them. + fs::create_dir_all(&target_dir).expect("creating the helper's target dir"); + for dir in [ + target_dir.parent().expect("target dir has a parent"), + &target_dir, + ] { + std::os::unix::fs::chown(dir, Some(uid), Some(gid)).expect("chowning target dir"); + } cargo.uid(uid).gid(gid); } assert!(