diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh index c1916f7c..326e3caa 100755 --- a/.github/scripts/provision-kvm-host.sh +++ b/.github/scripts/provision-kvm-host.sh @@ -94,6 +94,12 @@ ipv6_prefix="${ipv6_prefix:-2001:db8::/64}" # carries IPv6 link-local addresses, so it must be on. sudo sysctl -qw net.ipv6.conf.all.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 sudo sysctl -qw net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 +# The runner's Docker sets the iptables forward policy to drop, which drops VM +# traffic too (install.md: allow forwarding to and from vm-veth-*). +for ipt in iptables ip6tables; do + sudo "$ipt" -I FORWARD -i 'vm-veth+' -j ACCEPT + sudo "$ipt" -I FORWARD -o 'vm-veth+' -j ACCEPT +done sudo mkdir -p /etc/hyper sudo tee /etc/hyper/config.toml >/dev/null < Vec { + let mut argv = vec!["/bin/sh".into(), "-c".into(), script.into(), "sh".into()]; + argv.extend_from_slice(args); + argv +} + +fn arg() -> impl Strategy { + any::().prop_map(|s| s.replace('\0', "")) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(64))] + + #[test] + fn argv_reaches_the_program_unchanged(args in vec(arg(), 0..8)) { + let (code, out, _) = run(&sh(r#"for a; do printf '%s\0' "$a"; done"#, &args), &BTreeMap::new(), None); + prop_assert_eq!(code, 0); + let expected: Vec = args.iter().flat_map(|a| a.bytes().chain([0])).collect(); + prop_assert_eq!(out, expected); + } + + #[test] + fn exit_status_is_the_programs_own(n in 0u8..=255) { + let (code, _, _) = run(&sh(&format!("exit {n}"), &[]), &BTreeMap::new(), None); + prop_assert_eq!(code, i32::from(n)); + } + + #[test] + fn non_empty_env_is_exactly_what_the_program_sees( + env in btree_map("[A-Z_][A-Z0-9_]{0,12}", "[ -~]{0,24}", 1..8) + ) { + let (code, out, _) = run(&["/usr/bin/env".into()], &env, None); + prop_assert_eq!(code, 0); + let seen: BTreeMap = std::str::from_utf8(&out) + .unwrap() + .lines() + .map(|l| { + let (k, v) = l.split_once('=').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + prop_assert_eq!(seen, env); + } +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(16))] + + #[test] + fn both_streams_are_captured_byte_for_byte( + stdout in vec(any::(), 0..300_000), + stderr in vec(any::(), 0..300_000), + ) { + let dir = std::env::temp_dir(); + let tag = format!("{}-{}", std::process::id(), rand_tag()); + let out_path = dir.join(format!("ga-out-{tag}")); + let err_path = dir.join(format!("ga-err-{tag}")); + std::fs::File::create(&out_path).unwrap().write_all(&stdout).unwrap(); + std::fs::File::create(&err_path).unwrap().write_all(&stderr).unwrap(); + + let args = [ + out_path.to_string_lossy().into_owned(), + err_path.to_string_lossy().into_owned(), + ]; + let (code, out, err) = + run(&sh(r#"cat "$1" & cat "$2" >&2; wait"#, &args), &BTreeMap::new(), None); + let _ = std::fs::remove_file(&out_path); + let _ = std::fs::remove_file(&err_path); + + prop_assert_eq!(code, 0); + prop_assert!(out == stdout, "stdout differs: {} vs {} bytes", out.len(), stdout.len()); + prop_assert!(err == stderr, "stderr differs: {} vs {} bytes", err.len(), stderr.len()); + } +} + +fn rand_tag() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + NEXT.fetch_add(1, Ordering::Relaxed) +} diff --git a/native/hyperrs-elfsum/tests/stamped_slot_runtime.rs b/native/hyperrs-elfsum/tests/stamped_slot_runtime.rs new file mode 100644 index 00000000..14fe362a --- /dev/null +++ b/native/hyperrs-elfsum/tests/stamped_slot_runtime.rs @@ -0,0 +1,79 @@ +//! The end-to-end contract the helper's install-time verification rests on: +//! the digest `Elf::stamp` writes into a built executable is exactly what +//! `checksum!()` reports when that executable runs, and a verifier that +//! re-zeroes `.note.sum` and BLAKE3-hashes the file recomputes it. Laws: +//! * runtime report == stamped digest (the stamper patched the slot the +//! loader maps, and the read is not constant-folded to the link-time zero); +//! * re-zero + rehash of the stamped file == stamped digest; +//! * an unstamped executable reports all zeroes. +#![cfg(all(target_os = "linux", feature = "stamp"))] + +use hyperrs_elfsum::Elf; +use pretty_assertions::assert_eq; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::Command; + +const CHILD: &str = "HYPER_ELFSUM_REPORT_SLOT"; +const TEST: &str = "a_stamped_executable_reports_its_own_stamp_at_runtime"; + +fn hex(d: &[u8]) -> String { + d.iter().map(|b| format!("{b:02x}")).collect() +} + +fn report(exe: &Path) -> String { + let out = Command::new(exe) + .args(["--exact", TEST, "--nocapture", "--test-threads=1"]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + let stdout = String::from_utf8(out.stdout).unwrap(); + stdout + .split_once("slot=") + .and_then(|(_, rest)| rest.get(..64)) + .unwrap_or_else(|| panic!("child printed no slot: {stdout}")) + .to_string() +} + +#[test] +fn a_stamped_executable_reports_its_own_stamp_at_runtime() { + if std::env::var_os(CHILD).is_some() { + println!("slot={}", hyperrs_elfsum::checksum!().hex()); + return; + } + let dir = std::env::temp_dir().join(format!("elfsum-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let original = fs::read("/proc/self/exe").unwrap(); + + let unstamped = dir.join("unstamped"); + fs::write(&unstamped, &original).unwrap(); + fs::set_permissions(&unstamped, fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(report(&unstamped), "0".repeat(64)); + + let mut bytes = original.clone(); + let digest = Elf::read(&mut bytes[..]).unwrap().stamp(); + let stamped = dir.join("stamped"); + fs::write(&stamped, &bytes).unwrap(); + fs::set_permissions(&stamped, fs::Permissions::from_mode(0o755)).unwrap(); + + assert_eq!(report(&stamped), hex(&digest)); + + let changed: Vec = (0..bytes.len()) + .filter(|&i| bytes[i] != original[i]) + .collect(); + let (first, last) = (changed[0], *changed.last().unwrap()); + assert!( + last - first < digest.len(), + "stamp touched bytes outside one slot" + ); + let at = (0..=bytes.len() - digest.len()) + .find(|&i| bytes[i..i + digest.len()] == digest) + .unwrap(); + let mut rezeroed = bytes.clone(); + rezeroed[at..at + digest.len()].fill(0); + assert_eq!(*blake3::hash(&rezeroed).as_bytes(), digest); + + fs::remove_dir_all(&dir).unwrap(); +} diff --git a/native/hyperrs-firecracker/tests/jail_root_planted.rs b/native/hyperrs-firecracker/tests/jail_root_planted.rs new file mode 100644 index 00000000..06fbdbf0 --- /dev/null +++ b/native/hyperrs-firecracker/tests/jail_root_planted.rs @@ -0,0 +1,158 @@ +//! The jail `root/` is owned by the jail uid, so a Firecracker a hostile guest has taken over can +//! plant any entry there before the helper stages into it. Law: a name the jail uid planted where +//! the helper stages `vmlinux` or `rootfs` is never written through, retyped or re-owned. + +use assert_fs::fixture::ChildPath; +use assert_fs::prelude::*; +use assert_fs::TempDir; +use hyperrs_firecracker::jailer::chroot::{JailLayout, JailRoot, KernelImage}; +use hyperrs_firecracker::VmId; +use hyperrs_fs::dir::DirError; +use hyperrs_fs::file::{self, Any, IsBlockDevice, OpenFlags, Owner, SafeFile}; +use hyperrs_fs::path::{IsAbsolute, SafePathBuf, StrictComponents}; +use nix::errno::Errno; +use nix::unistd::{getgid, getuid}; +use pretty_assertions::assert_eq; +use rstest::{fixture, rstest}; +use std::assert_matches; +use std::fs; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; + +type Strict = SafePathBuf; + +fn strict(p: impl AsRef) -> Strict { + p.as_ref().to_path_buf().try_into().unwrap() +} + +fn vm() -> VmId { + "hostile".parse().unwrap() +} + +struct Jail { + base: TempDir, + root: ChildPath, +} + +impl Jail { + fn open_root(&self) -> JailRoot { + JailLayout::new(&strict(self.base.path()), &vm()) + .open_root() + .unwrap() + } +} + +#[fixture] +fn jail() -> Jail { + let base = TempDir::new().unwrap(); + let root = base.child("firecracker/hostile/root"); + root.create_dir_all().unwrap(); + Jail { base, root } +} + +fn any_block_device() -> Option { + fs::read_dir("/dev") + .ok()? + .flatten() + .map(|e| e.path()) + .find(|p| { + fs::symlink_metadata(p) + .map(|m| m.file_type().is_block_device()) + .unwrap_or(false) + }) +} + +fn me() -> Owner { + Owner { + uid: getuid(), + gid: getgid(), + } +} + +fn is_root() -> bool { + nix::unistd::geteuid().is_root() +} + +#[rstest] +fn add_rootfs_over_a_planted_regular_file_refuses_and_leaves_it(jail: Jail) { + let Some(dev) = any_block_device() else { + eprintln!( + "SKIP add_rootfs_over_a_planted_regular_file_refuses_and_leaves_it: no block device" + ); + return; + }; + let planted = jail.root.child("rootfs"); + planted.write_str("planted").unwrap(); + fs::set_permissions(&planted, fs::Permissions::from_mode(0o644)).unwrap(); + let device = SafeFile::::open(&strict(&dev), OpenFlags::PATH).unwrap(); + + let err = jail.open_root().add_rootfs(&device, me()).unwrap_err(); + + assert_matches!(err, DirError::Open(file::ValidationError::WrongFileType)); + planted.assert("planted"); + assert_eq!(fs::metadata(&planted).unwrap().mode() & 0o7777, 0o644); +} + +#[rstest] +fn add_rootfs_over_a_planted_symlink_refuses_and_never_touches_its_target(jail: Jail) { + let Some(dev) = any_block_device() else { + eprintln!("SKIP add_rootfs_over_a_planted_symlink: no block device"); + return; + }; + let victim = jail.base.child("victim"); + victim.write_str("victim").unwrap(); + fs::set_permissions(&victim, fs::Permissions::from_mode(0o640)).unwrap(); + jail.root.child("rootfs").symlink_to_file(&victim).unwrap(); + let device = SafeFile::::open(&strict(&dev), OpenFlags::PATH).unwrap(); + + let err = jail.open_root().add_rootfs(&device, me()).unwrap_err(); + + assert_matches!( + err, + DirError::Open(file::ValidationError::Open(Errno::ELOOP)) + ); + victim.assert("victim"); + assert_eq!(fs::metadata(&victim).unwrap().mode() & 0o7777, 0o640); +} + +/// Root-only: unprivileged `linkat(AT_EMPTY_PATH)` is `ENOENT` before 6.10, so neither the +/// hard-link nor the `EXDEV` copy path would be reached. +#[rstest] +fn stage_kernel_never_writes_through_a_planted_vmlinux_symlink_as_root( + jail: Jail, + #[values(false, true)] cross_device: bool, +) { + if !is_root() { + eprintln!("SKIP stage_kernel_never_writes_through_a_planted_vmlinux_symlink_as_root"); + return; + } + let src_dir = if cross_device { + match TempDir::new_in("/dev/shm") { + Ok(d) => d, + Err(_) => { + eprintln!("SKIP cross-device staging: no /dev/shm"); + return; + } + } + } else { + TempDir::new_in(jail.base.path()).unwrap() + }; + let src = src_dir.child("vmlinux"); + src.write_str("kernel").unwrap(); + fs::set_permissions(&src, fs::Permissions::from_mode(0o644)).unwrap(); + let victim = jail.base.child("victim"); + victim.write_str("victim").unwrap(); + jail.root.child("vmlinux").symlink_to_file(&victim).unwrap(); + + let err = jail + .open_root() + .stage_kernel(&KernelImage::open(&strict(&src)).unwrap()) + .unwrap_err(); + + assert_eq!(err.errno(), Some(Errno::EEXIST), "{err:?}"); + victim.assert("victim"); + assert!(fs::symlink_metadata(jail.root.join("vmlinux")) + .unwrap() + .file_type() + .is_symlink()); +} diff --git a/native/hyperrs-fs/tests/path_properties.rs b/native/hyperrs-fs/tests/path_properties.rs new file mode 100644 index 00000000..9f4a7863 --- /dev/null +++ b/native/hyperrs-fs/tests/path_properties.rs @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: MIT +//! Contract of the strict path types every confinement check in the helper is built on. +//! +//! Laws under test, each against an independent byte-level oracle: +//! - **Component oracle.** A byte string is a [`StrictComponent`] iff it is non-empty, not `.`/`..`, +//! at most `NAME_MAX` bytes, and holds no `/`, NUL or ASCII whitespace. Non-UTF-8 bytes are fine. +//! - **Path oracle.** A raw byte path is a strict absolute [`SafePathBuf`] iff it starts with `/` +//! and either is exactly `/` or every `/`-separated segment passes the component oracle. Parsing +//! through `&str`, `PathBuf` and `&Path` all agree. +//! - **Already normal.** An accepted path is a fixed point of std's component normalization, so no +//! later `Path::components`/`join` can reinterpret it. +//! - **`strip_base` is component-prefix.** It returns exactly the trailing segments when the base's +//! segments are a proper prefix, `EqualsBase` on equality, `NotUnderBase` otherwise (including a +//! byte-prefix that is not a component prefix, e.g. `/ab` under `/a`). +//! - **`split_last` inverts `join`.** For any non-root path, `parent.join(name)` rebuilds it, the +//! parent is itself strict, and `name` is `file_name()`; `/` has no split. + +use hyperrs_fs::path::{ + ComponentError, IsAbsolute, RelativeError, SafePath, SafePathBuf, StrictComponent, + StrictComponents, ValidationError, NAME_MAX, +}; +use proptest::prelude::*; +use std::assert_matches; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::{Component, Path, PathBuf}; + +type Strict = SafePathBuf; + +fn oracle_component(bytes: &[u8]) -> bool { + !bytes.is_empty() + && bytes != b"." + && bytes != b".." + && bytes.len() <= NAME_MAX + && !bytes + .iter() + .any(|b| matches!(b, b'/' | 0 | b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r')) +} + +fn oracle_path(raw: &[u8]) -> bool { + match raw { + [b'/'] => true, + [b'/', rest @ ..] => rest.split(|&b| b == b'/').all(oracle_component), + _ => false, + } +} + +fn byte() -> impl Strategy { + prop_oneof![ + 6 => b'a'..=b'z', + 2 => Just(b'.'), + 1 => Just(b'/'), + 1 => Just(0u8), + 1 => prop::sample::select(vec![b' ', b'\t', b'\n', 0x0b, 0x0c, b'\r']), + 1 => 0x80u8..=0xff, + 1 => Just(b'-'), + ] +} + +fn component_bytes() -> impl Strategy> { + prop_oneof![ + 8 => prop::collection::vec(byte(), 0..6), + 1 => prop::collection::vec(b'a'..=b'z', NAME_MAX - 2..NAME_MAX + 3), + 1 => prop::sample::select(vec![b".".to_vec(), b"..".to_vec(), b"...".to_vec(), b".a".to_vec()]), + ] +} + +fn raw_path() -> impl Strategy> { + ( + any::(), + prop::collection::vec(component_bytes(), 0..5), + any::(), + ) + .prop_map(|(absolute, segments, trailing)| { + let mut raw = if absolute { b"/".to_vec() } else { Vec::new() }; + raw.extend(segments.join(&b'/')); + if trailing { + raw.push(b'/'); + } + raw + }) +} + +fn strict_raw(raw: &[u8]) -> Result { + PathBuf::from(OsStr::from_bytes(raw)).try_into() +} + +fn seg() -> impl Strategy { + prop::sample::select(vec!["a", "b", "ab", "a.b", "-"]) +} + +fn abs(segments: &[&str]) -> Strict { + format!("/{}", segments.join("/")).parse().unwrap() +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(2048))] + + #[test] + fn component_accepts_exactly_the_oracle(bytes in component_bytes()) { + let got = <&StrictComponent>::try_from(OsStr::from_bytes(&bytes)); + prop_assert_eq!(got.is_ok(), oracle_component(&bytes), "{:?}", bytes); + if let Ok(name) = got { + prop_assert_eq!(name.as_os_str().as_bytes(), &bytes[..]); + let owned = name.to_owned(); + prop_assert_eq!(owned.as_os_str().as_bytes(), &bytes[..]); + } + } + + #[test] + fn path_accepts_exactly_the_oracle(raw in raw_path()) { + let got = strict_raw(&raw); + prop_assert_eq!(got.is_ok(), oracle_path(&raw), "{:?}", String::from_utf8_lossy(&raw)); + if raw.first() != Some(&b'/') { + prop_assert!(matches!(got, Err(ValidationError::NotAbsolute))); + } + let borrowed: Result<&SafePath, _> = + Path::new(OsStr::from_bytes(&raw)).try_into(); + prop_assert_eq!(borrowed.is_ok(), got.is_ok()); + if let Ok(utf8) = std::str::from_utf8(&raw) { + prop_assert_eq!(utf8.parse::().is_ok(), got.is_ok()); + } + } + + #[test] + fn accepted_paths_are_already_normal(raw in raw_path()) { + if let Ok(path) = strict_raw(&raw) { + let normal: PathBuf = path.as_path().components().collect(); + prop_assert_eq!(normal.as_os_str().as_bytes(), &raw[..]); + prop_assert!(path + .as_path() + .components() + .all(|c| matches!(c, Component::RootDir | Component::Normal(_)))); + } + } + + #[test] + fn strip_base_is_a_component_prefix( + base in prop::collection::vec(seg(), 0..3), + path in prop::collection::vec(seg(), 0..5), + ) { + let got = abs(&path).strip_base(&abs(&base)).map(|rel| { + rel.segments() + .map(|s| s.as_os_str().to_string_lossy().into_owned()) + .collect::>() + }); + if base == path { + prop_assert!(matches!(got, Err(RelativeError::EqualsBase)), "{got:?}"); + } else if path.starts_with(&base) { + let want: Vec = path[base.len()..].iter().map(|s| s.to_string()).collect(); + prop_assert_eq!(got.unwrap(), want); + } else { + prop_assert!(matches!(got, Err(RelativeError::NotUnderBase)), "{got:?}"); + } + } + + #[test] + fn split_last_inverts_join(path in prop::collection::vec(seg(), 0..5)) { + let full = abs(&path); + match full.split_last() { + None => prop_assert!(path.is_empty()), + Some((parent, name)) => { + let rebuilt = parent.join(name); + let want_parent = abs(&path[..path.len() - 1]); + prop_assert_eq!(rebuilt.as_path(), full.as_path()); + prop_assert_eq!(parent.as_path(), want_parent.as_path()); + prop_assert!(strict_raw(parent.as_path().as_os_str().as_bytes()).is_ok()); + prop_assert_eq!(Some(name), full.file_name()); + prop_assert_eq!(name.as_os_str(), OsStr::new(path[path.len() - 1])); + } + } + } + + #[test] + fn split_last_inverts_join_on_relative_paths( + base in prop::collection::vec(seg(), 0..3), + rest in prop::collection::vec(seg(), 2..5), + ) { + let full_segments: Vec<&str> = base.iter().chain(rest.iter()).copied().collect(); + let full = abs(&full_segments); + let base = abs(&base); + let rel = full.strip_base(&base).unwrap(); + let (parent, name) = rel.split_last().unwrap(); + let rebuilt = parent.join(name); + prop_assert_eq!(rebuilt.as_path(), rel.as_path()); + prop_assert_eq!(rel.file_name(), Some(name)); + } +} + +#[test] +fn name_max_is_the_exact_component_length_boundary() { + let at = "a".repeat(NAME_MAX); + let over = "a".repeat(NAME_MAX + 1); + assert!(<&StrictComponent>::try_from(at.as_str()).is_ok()); + assert_matches!( + <&StrictComponent>::try_from(over.as_str()), + Err(ComponentError::TooLong(n)) if n == NAME_MAX + 1 + ); + assert!(format!("/{at}/{at}").parse::().is_ok()); + assert_matches!( + format!("/{at}/{over}").parse::(), + Err(ValidationError::Component(ComponentError::TooLong(_))) + ); +} + +#[test] +fn byte_prefix_that_is_not_a_component_prefix_is_not_under_base() { + assert_matches!( + abs(&["ab", "c"]).strip_base(&abs(&["a"])), + Err(RelativeError::NotUnderBase) + ); +} diff --git a/native/hyperrs-linux/tests/dm_serde_properties.rs b/native/hyperrs-linux/tests/dm_serde_properties.rs new file mode 100644 index 00000000..f4fb06bd --- /dev/null +++ b/native/hyperrs-linux/tests/dm_serde_properties.rs @@ -0,0 +1,83 @@ +//! The helper's JSON requests deserialize straight into these types, so serde +//! must be exactly as strict as the checked constructors. Laws: +//! * a JSON integer is a `ThinId` iff it is in `0..=ThinId::MAX`, and it +//! keeps its value; any non-integer JSON is refused; +//! * a JSON string deserializes to a `DmName`, `LoopDevNum` or `Origin` +//! exactly when `FromStr` accepts it, and to the same value. +#![cfg(feature = "serde")] + +use hyperrs_linux::dm::{DmName, Origin, ThinId}; +use hyperrs_linux::loopdev::LoopDevNum; +use proptest::prelude::*; +use serde::de::DeserializeOwned; +use serde_json::{json, Value}; +use std::fmt::Debug; +use std::str::FromStr; +use test_strategy::proptest; + +#[proptest] +fn a_json_integer_is_a_thin_id_exactly_when_it_fits_24_bits( + #[strategy(prop_oneof![ + 0i64..=i64::from(ThinId::MAX), + (i64::from(ThinId::MAX) + 1)..=i64::MAX, + i64::MIN..0, + ])] + n: i64, +) { + let got = serde_json::from_value::(json!(n)); + if (0..=i64::from(ThinId::MAX)).contains(&n) { + prop_assert_eq!(u64::from(got.unwrap()), n as u64); + } else { + prop_assert!(got.is_err(), "accepted {}", n); + } +} + +#[proptest] +fn non_integer_json_is_never_a_thin_id( + #[strategy(prop_oneof![ + (0u32..1000).prop_map(|n| json!(f64::from(n) + 0.5)), + (0u32..1000).prop_map(|n| json!(n.to_string())), + Just(json!(null)), + Just(json!(true)), + Just(json!([1])), + Just(json!({"id": 1})), + ])] + v: Value, +) { + prop_assert!( + serde_json::from_value::(v.clone()).is_err(), + "accepted {}", + v + ); +} + +fn agrees_with_from_str(s: &str) -> Result<(), TestCaseError> +where + T: DeserializeOwned + FromStr + PartialEq + Debug, + T::Err: Debug, +{ + let wire = serde_json::from_value::(json!(s)); + match s.parse::() { + Ok(parsed) => prop_assert_eq!(wire.unwrap(), parsed), + Err(_) => prop_assert!(wire.is_err(), "wire accepted {:?} that FromStr refuses", s), + } + Ok(()) +} + +fn text() -> impl Strategy { + prop_oneof![ + "[a-z0-9#+.:=@_-]{0,12}", + "hyper-[a-z0-9]{0,8}", + "/dev/loop[0-9+-]{0,11}", + "/dev/mapper/[a-z0-9./-]{0,10}", + "/[a-z./]{0,12}", + "\\PC{0,8}", + ] +} + +#[proptest(ProptestConfig::with_cases(1024))] +fn string_wire_types_deserialize_exactly_as_from_str(#[strategy(text())] s: String) { + agrees_with_from_str::(&s)?; + agrees_with_from_str::(&s)?; + agrees_with_from_str::(&s)?; +} diff --git a/native/suidhelper/Cargo.toml b/native/suidhelper/Cargo.toml index 189ccf56..e3d80f2c 100644 --- a/native/suidhelper/Cargo.toml +++ b/native/suidhelper/Cargo.toml @@ -27,6 +27,18 @@ required-features = ["insecure_mode"] name = "config_seam" path = "tests/config_seam.rs" +[[test]] +name = "caller_authorize" +path = "tests/caller_authorize.rs" + +[[test]] +name = "cli_request" +path = "tests/cli_request.rs" + +[[test]] +name = "request_wire_properties" +path = "tests/request_wire_properties.rs" + [[test]] name = "e2e_config" path = "tests/e2e/config.rs" @@ -35,6 +47,18 @@ path = "tests/e2e/config.rs" name = "e2e_firewall" path = "tests/e2e/firewall.rs" +[[test]] +name = "e2e_vm_net" +path = "tests/e2e/vm_net.rs" + +[[test]] +name = "e2e_vm_net_traffic" +path = "tests/e2e/vm_net_traffic.rs" + +[[test]] +name = "e2e_caller_gate" +path = "tests/e2e/caller_gate.rs" + [[test]] name = "e2e_argv" path = "tests/e2e/argv.rs" @@ -47,6 +71,18 @@ path = "tests/e2e/jailer.rs" name = "e2e_chroot_jail" path = "tests/e2e/chroot_jail.rs" +[[test]] +name = "e2e_socket_pin" +path = "tests/e2e/socket_pin.rs" + +[[test]] +name = "e2e_lock_dir_hostile" +path = "tests/e2e/lock_dir_hostile.rs" + +[[test]] +name = "e2e_claim_privacy" +path = "tests/e2e/claim_privacy.rs" + [[test]] name = "e2e_version" path = "tests/e2e/version.rs" @@ -59,6 +95,10 @@ path = "tests/e2e/thin.rs" name = "e2e_fork_delta" path = "tests/e2e/fork_delta.rs" +[[test]] +name = "e2e_fork_delta_live" +path = "tests/e2e/fork_delta_live.rs" + [[test]] name = "e2e_img" path = "tests/e2e/img.rs" @@ -71,6 +111,10 @@ path = "tests/config_uid_gid_range.rs" name = "config_paths" path = "tests/config_paths.rs" +[[test]] +name = "net_addressing" +path = "tests/net_addressing.rs" + [[test]] name = "dm_backing" path = "tests/dm/backing.rs" diff --git a/native/suidhelper/src/net/firewall.rs b/native/suidhelper/src/net/firewall.rs index e0265422..d5d88571 100644 --- a/native/suidhelper/src/net/firewall.rs +++ b/native/suidhelper/src/net/firewall.rs @@ -194,3 +194,128 @@ mod tests { assert!(serde_json::from_value::(policy).is_err()); } } + +#[cfg(test)] +mod refusal_properties { + //! Laws over arbitrary policies as the helper's JSON delivers them: + //! - a policy is accepted exactly when every rule names `tcp` or `udp` over a + //! non-empty range without port 0 and each direction holds at most `Rules::MAX`; + //! - an accepted policy renders one destination-port match per rule, in order, over + //! exactly the ports it named. + + use super::{Firewall, Rules}; + use hyperrs_linux::net::nft::{L4Proto, Match}; + use proptest::prelude::*; + use serde_json::{json, Value}; + use test_strategy::proptest; + + #[derive(Debug, Clone)] + struct Raw { + protocol: &'static str, + first: u16, + last: u16, + } + + impl Raw { + fn valid(&self) -> bool { + matches!(self.protocol, "tcp" | "udp") && self.first != 0 && self.first <= self.last + } + + fn json(&self) -> Value { + json!({"protocol": self.protocol, "ports": [self.first, self.last]}) + } + + fn rendered(&self) -> Match { + let proto = if self.protocol == "tcp" { + L4Proto::Tcp + } else { + L4Proto::Udp + }; + Match::Dport(proto, self.first..=self.last) + } + } + + fn port() -> impl Strategy { + prop_oneof![Just(0), Just(1), Just(u16::MAX), any::()] + } + + fn raw() -> impl Strategy { + ( + prop_oneof![ + 8 => Just("tcp"), + 8 => Just("udp"), + 1 => Just("icmp"), + 1 => Just("TCP"), + 1 => Just(""), + ], + port(), + port(), + ) + .prop_map(|(protocol, first, last)| Raw { + protocol, + first, + last, + }) + } + + fn direction() -> impl Strategy> { + prop_oneof![ + 4 => prop::collection::vec(raw(), 0..8), + 1 => (Rules::MAX - 2..=Rules::MAX + 2).prop_flat_map(|len| { + prop::collection::vec( + (1u16..=u16::MAX).prop_map(|p| Raw { + protocol: "tcp", + first: p, + last: p, + }), + len, + ) + }), + ] + } + + #[proptest] + fn a_policy_is_accepted_exactly_when_every_rule_and_direction_is_valid( + #[strategy(direction())] inbound: Vec, + #[strategy(direction())] outbound_blocked: Vec, + ) { + let ok = |rules: &[Raw]| rules.len() <= Rules::MAX && rules.iter().all(Raw::valid); + let parsed = serde_json::from_value::(json!({ + "inbound": inbound.iter().map(Raw::json).collect::>(), + "outbound_blocked": outbound_blocked.iter().map(Raw::json).collect::>(), + })); + + prop_assert_eq!(parsed.is_ok(), ok(&inbound) && ok(&outbound_blocked)); + if let Ok(firewall) = parsed { + prop_assert_eq!( + firewall.inbound().collect::>(), + inbound.iter().map(Raw::rendered).collect::>() + ); + prop_assert_eq!( + firewall.outbound_blocked().collect::>(), + outbound_blocked + .iter() + .map(Raw::rendered) + .collect::>() + ); + } + } + + #[proptest] + fn ports_outside_u16_are_refused( + #[strategy(prop_oneof![-70_000i64..0, 65_536i64..200_000])] bad: i64, + first: bool, + ) { + let ports = if first { + json!([bad, 80]) + } else { + json!([80, bad]) + }; + let policy = json!({ + "inbound": [{"protocol": "tcp", "ports": ports}], + "outbound_blocked": [], + }); + + prop_assert!(serde_json::from_value::(policy).is_err()); + } +} diff --git a/native/suidhelper/src/net/vm.rs b/native/suidhelper/src/net/vm.rs index 83ed58dd..0ecd6c0a 100644 --- a/native/suidhelper/src/net/vm.rs +++ b/native/suidhelper/src/net/vm.rs @@ -564,6 +564,10 @@ mod tests { impl Lab { fn new(test: &str) -> Self { + Self::with_blocked(test, &Blocked::default()) + } + + fn with_blocked(test: &str, blocked: &Blocked) -> Self { let guest = ScratchNs::new(&format!("{test}-guest")); let vm = ScratchNs::new(&format!("{test}-vm")); let host = ScratchNs::new(&format!("{test}-host")); @@ -630,7 +634,7 @@ mod tests { GUEST_V6.parse().unwrap(), Uid::from_raw(FIRECRACKER), "198.51.100.1".parse().unwrap(), - &Blocked::default(), + blocked, &HOST_ADDRS.map(|addr| addr.parse().unwrap()), ), VmNet::firewall_table(&firewall), @@ -870,4 +874,253 @@ mod tests { assert_eq!(uplink.proxy_neighs().unwrap(), vec![live]); } + + mod hostile_guest { + //! A guest that forges, impersonates and probes, against the namespace's tables. + //! + //! Laws: + //! - a TCP segment whose source port's high byte reads as a neighbour-discovery + //! type is not neighbour discovery: it never reaches a namespace service; + //! - the namespace's link-local and uplink addresses serve the guest nothing; + //! - the DNS redirect never carries a forged source to the resolver; + //! - the jailed Firecracker reaches not even its own guest; + //! - with `[net] blocked` empty, the host's addresses and the transit range stay + //! unreachable from the guest; + //! - a router advertisement from the guest never reaches the namespace, even on a + //! `tap0` that would otherwise act on it. + + use super::{Lab, Probe, DROPPED_AFTER, FIRECRACKER, NAMESPACE}; + use crate::net::Blocked; + use pretty_assertions::assert_eq; + use rstest::rstest; + use socket2::{Domain, Protocol, Socket, Type}; + use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; + use std::process::Command; + use std::time::Duration; + + const SOLICIT_SPORT: u16 = 135 << 8; + const ADVERT_SPORT: u16 = 136 << 8; + const ROUTER_ADVERT: u8 = 134; + const RA_PREFIX: &str = "2001:db8:bad::"; + + fn skip_unless_root(test: &str) -> bool { + if nix::unistd::geteuid().is_root() { + return false; + } + eprintln!("SKIP {test}: needs root"); + true + } + + fn lab_name(parts: &str) -> String { + let mut name: String = parts + .to_lowercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect(); + name.truncate(40); + name + } + + fn eth0(lab: &Lab) -> u32 { + lab.guest + .get() + .rtnl() + .unwrap() + .link(&Lab::name("eth0")) + .unwrap() + .unwrap() + .index() + .get() + } + + /// Link-local destinations are scoped to the guest's `eth0`. + fn target(lab: &Lab, to: &str) -> SocketAddr { + let to: SocketAddr = to.parse().unwrap(); + match to { + SocketAddr::V6(v6) if v6.ip().is_unicast_link_local() => { + SocketAddrV6::new(*v6.ip(), v6.port(), 0, eth0(lab)).into() + } + to => to, + } + } + + fn guest_connects(lab: &Lab, src: Option<&str>, sport: u16, to: &str) -> bool { + let to = target(lab, to); + lab.guest + .get() + .scoped(|| { + let socket = Socket::new(Domain::for_address(to), Type::STREAM, None).unwrap(); + socket.set_reuse_address(true).unwrap(); + let src: IpAddr = match (src, to) { + (Some(src), _) => src.parse().unwrap(), + (None, SocketAddr::V4(_)) => IpAddr::from([0, 0, 0, 0]), + (None, SocketAddr::V6(_)) => Ipv6Addr::UNSPECIFIED.into(), + }; + socket.bind(&SocketAddr::new(src, sport).into()).unwrap(); + socket.connect_timeout(&to.into(), DROPPED_AFTER).is_ok() + }) + .unwrap() + } + + #[rstest] + #[case::solicit_lookalike_to_a_namespace_service( + None, + SOLICIT_SPORT, + "172.16.0.1:7074", + false + )] + #[case::advert_lookalike_to_a_namespace_service( + None, + ADVERT_SPORT, + "172.16.0.1:7074", + false + )] + #[case::solicit_lookalike_over_link_local(None, SOLICIT_SPORT, "[fe80::1]:7074", false)] + #[case::advert_lookalike_over_link_local(None, ADVERT_SPORT, "[fe80::1]:7074", false)] + #[case::link_local_to_a_namespace_service(None, 0, "[fe80::1]:7074", false)] + #[case::uplink_address_to_a_namespace_service(None, 0, "10.200.0.2:7074", false)] + #[case::forged_source_through_the_dns_redirect( + Some("192.0.2.9"), + 0, + "172.16.0.1:53", + false + )] + #[case::forged_ipv6_source_to_the_host( + Some("2001:db8::1:0:0:6"), + SOLICIT_SPORT, + "[2001:db8:ffff::1]:7071", + false + )] + #[case::lookalike_port_still_reaches_the_internet( + None, + SOLICIT_SPORT, + "198.51.100.1:7071", + true + )] + fn a_hostile_guest_reaches_only_what_policy_allows_as_root( + #[case] src: Option<&str>, + #[case] sport: u16, + #[case] to: &str, + #[case] reachable: bool, + ) { + if skip_unless_root("hostile guest") { + return; + } + let lab = Lab::new(&lab_name(&format!("hg-{src:?}-{sport}-{to}"))); + + assert_eq!(guest_connects(&lab, src, sport, to), reachable); + } + + #[rstest] + #[case::as_firecracker(FIRECRACKER, false)] + #[case::as_root(0, true)] + fn the_jailed_firecracker_cannot_reach_its_own_guest_as_root( + #[case] uid: u32, + #[case] reachable: bool, + ) { + if skip_unless_root("namespace to guest") { + return; + } + let lab = Lab::new(&format!("hg-fc-{uid}")); + + assert_eq!( + lab.connects(Probe::Namespace(uid), "172.16.0.2:7071".parse().unwrap()), + reachable + ); + } + + #[rstest] + #[case::the_host_veth("10.200.0.1:7071", false)] + #[case::a_host_address("198.51.100.2:7071", false)] + #[case::a_host_address_over_ipv6("[2001:db8:ffff::2]:7071", false)] + #[case::the_internet("198.51.100.1:7071", true)] + fn with_nothing_blocked_the_guest_still_never_reaches_the_host_as_root( + #[case] to: &str, + #[case] reachable: bool, + ) { + if skip_unless_root("empty blocked") { + return; + } + let lab = Lab::with_blocked( + &lab_name(&format!("hg-nb-{to}")), + &Blocked::try_from(Vec::new()).unwrap(), + ); + + assert_eq!(guest_connects(&lab, None, 0, to), reachable); + } + + /// An RA from `fe80::2` on `eth0` announcing `RA_PREFIX/64` and itself as a default + /// router. + fn send_router_advert(lab: &Lab) { + let index = eth0(lab); + lab.guest + .get() + .scoped(|| { + let socket = + Socket::new(Domain::IPV6, Type::RAW, Some(Protocol::ICMPV6)).unwrap(); + socket.set_multicast_hops_v6(255).unwrap(); + socket.set_unicast_hops_v6(255).unwrap(); + socket.set_multicast_if_v6(index).unwrap(); + let src = SocketAddrV6::new("fe80::2".parse().unwrap(), 0, 0, index); + socket.bind(&src.into()).unwrap(); + let mut ra = vec![ROUTER_ADVERT, 0, 0, 0, 64, 0]; + ra.extend_from_slice(&1800u16.to_be_bytes()); + ra.extend_from_slice(&[0; 8]); + ra.extend_from_slice(&[3, 4, 64, 0xc0]); + ra.extend_from_slice(&86_400u32.to_be_bytes()); + ra.extend_from_slice(&14_400u32.to_be_bytes()); + ra.extend_from_slice(&[0; 4]); + ra.extend_from_slice(&RA_PREFIX.parse::().unwrap().octets()); + let all_nodes = SocketAddrV6::new("ff02::1".parse().unwrap(), 0, 0, index); + socket.send_to(&ra, &all_nodes.into()).unwrap(); + }) + .unwrap(); + } + + fn namespace_routes(lab: &Lab) -> String { + let out = lab + .vm + .get() + .scoped(|| Command::new("ip").args(["-6", "route", "show"]).output()) + .unwrap() + .unwrap(); + String::from_utf8(out.stdout).unwrap() + } + + #[rstest] + fn a_router_advert_from_the_guest_never_reaches_the_namespace_as_root() { + if skip_unless_root("guest router advert") { + return; + } + let lab = Lab::new("hg-ra"); + lab.vm + .get() + .scoped(|| { + std::fs::write("/proc/sys/net/ipv6/conf/tap0/accept_ra", "2").unwrap(); + }) + .unwrap(); + + send_router_advert(&lab); + std::thread::sleep(Duration::from_secs(1)); + + let routes = namespace_routes(&lab); + assert!( + !routes.contains("proto ra") && !routes.contains(RA_PREFIX.trim_end_matches(':')), + "{routes}" + ); + } + + #[rstest] + fn a_namespace_service_is_reachable_from_the_namespace_itself_as_root() { + if skip_unless_root("namespace service control") { + return; + } + let lab = Lab::new("hg-ctl"); + + assert!(lab.connects( + Probe::Namespace(0), + format!("127.0.0.1:{NAMESPACE}").parse().unwrap() + )); + } + } } diff --git a/native/suidhelper/tests/caller_authorize.rs b/native/suidhelper/tests/caller_authorize.rs new file mode 100644 index 00000000..ada45274 --- /dev/null +++ b/native/suidhelper/tests/caller_authorize.rs @@ -0,0 +1,143 @@ +//! The caller gate ([`Caller::authorize`]) against an independent model of the #156 contract: +//! "privileged ops run only for root and members of the group the helper is installed with". +//! +//! Laws: +//! - oracle: a caller is authorized iff its uid is 0, or the install group is readable, is not +//! gid 0, and is the caller's primary or a supplementary gid; +//! - the refusal names the caller's uid and the install group, and a root install group or an +//! unreadable install refuses every non-root caller with exactly that cause; +//! - membership is a set property: reordering or repeating supplementary gids never changes +//! the verdict; +//! - root bypass is about the uid alone: gid 0 or membership of gid 0 grants nothing. + +use hyper_suidhelper::caller::{Caller, CallerError, NodeGroup}; +use nix::errno::Errno; +use nix::unistd::{Gid, Uid}; +use proptest::prelude::*; +use test_strategy::proptest; + +fn gid() -> impl Strategy { + prop_oneof![0u32..4, 990u32..994, any::()] +} + +fn installed() -> impl Strategy> { + prop_oneof![ + 8 => gid().prop_map(Ok), + 1 => prop_oneof![Just(Errno::EACCES), Just(Errno::ENOENT), Just(Errno::ELOOP)].prop_map(Err), + ] +} + +fn caller(uid: u32, gid: u32, groups: &[u32]) -> Caller { + Caller { + uid: Uid::from_raw(uid), + gid: Gid::from_raw(gid), + groups: groups.iter().copied().map(Gid::from_raw).collect(), + } +} + +fn group(installed: Result) -> Result { + installed + .map_err(CallerError::Stat) + .and_then(|g| Gid::from_raw(g).try_into()) +} + +fn model( + uid: u32, + gid: u32, + groups: &[u32], + installed: Result, +) -> Result<(), CallerError> { + if uid == 0 { + return Ok(()); + } + match installed { + Err(errno) => Err(CallerError::Stat(errno)), + Ok(0) => Err(CallerError::RootGroup), + Ok(g) if gid == g || groups.contains(&g) => Ok(()), + Ok(g) => Err(CallerError::NotInGroup { + uid: Uid::from_raw(uid), + group: Gid::from_raw(g), + }), + } +} + +#[proptest] +fn authorize_agrees_with_the_model( + #[strategy(prop_oneof![Just(0u32), 1u32..3, 1000u32..1003, any::()])] uid: u32, + #[strategy(gid())] gid: u32, + #[strategy(proptest::collection::vec(gid(), 0..40))] groups: Vec, + #[strategy(installed())] installed: Result, +) { + prop_assert_eq!( + caller(uid, gid, &groups).authorize(group(installed)), + model(uid, gid, &groups, installed) + ); +} + +#[proptest] +fn a_member_anywhere_in_a_long_group_list_is_authorized( + #[strategy(1u32..)] uid: u32, + #[strategy(1u32..)] node: u32, + #[strategy(proptest::collection::vec(any::(), 0..2_000))] others: Vec, + at: usize, +) { + let mut groups: Vec = others.into_iter().filter(|&g| g != node).collect(); + let primary = groups.pop().unwrap_or(node.wrapping_add(1).max(1)); + prop_assume!(primary != node); + prop_assert!(caller(uid, primary, &groups) + .authorize(group(Ok(node))) + .is_err()); + groups.insert(at % (groups.len() + 1), node); + prop_assert_eq!( + caller(uid, primary, &groups).authorize(group(Ok(node))), + Ok(()) + ); +} + +#[proptest] +fn reordering_or_repeating_supplementary_groups_never_changes_the_verdict( + #[strategy(1u32..)] uid: u32, + #[strategy(gid())] gid: u32, + #[strategy(proptest::collection::vec(gid(), 1..20))] groups: Vec, + #[strategy(gid())] node: u32, + rotate: usize, + repeat: usize, +) { + let verdict = caller(uid, gid, &groups).authorize(group(Ok(node))); + let mut shuffled = groups.clone(); + shuffled.rotate_left(rotate % groups.len()); + shuffled.reverse(); + shuffled.push(groups[repeat % groups.len()]); + prop_assert_eq!( + caller(uid, gid, &shuffled).authorize(group(Ok(node))), + verdict + ); +} + +#[proptest] +fn the_root_group_grants_a_non_root_uid_nothing( + #[strategy(1u32..)] uid: u32, + #[strategy(gid())] node: u32, +) { + let verdict = caller(uid, 0, &[0]).authorize(group(Ok(node))); + if node == 0 { + prop_assert_eq!(verdict, Err(CallerError::RootGroup)); + } else { + prop_assert_eq!( + verdict, + Err(CallerError::NotInGroup { + uid: Uid::from_raw(uid), + group: Gid::from_raw(node) + }) + ); + } +} + +#[proptest] +fn only_gid_zero_is_refused_as_an_install_group(raw: u32) { + let parsed = NodeGroup::try_from(Gid::from_raw(raw)); + prop_assert_eq!(parsed.is_err(), raw == 0); + if raw == 0 { + prop_assert_eq!(parsed, Err(CallerError::RootGroup)); + } +} diff --git a/native/suidhelper/tests/cli_request.rs b/native/suidhelper/tests/cli_request.rs new file mode 100644 index 00000000..ea1e086d --- /dev/null +++ b/native/suidhelper/tests/cli_request.rs @@ -0,0 +1,387 @@ +//! Hostile-request refusal laws for the helper's single JSON argument, the one input every +//! local caller controls before the privilege gate runs. Every valid request in +//! `fixtures/requests.json` is mutated at every leaf and every nested object; each law says the +//! mutated request is always refused as `RequestError::Json`, never accepted and never a crash: +//! +//! - nesting deeper than the parser's recursion limit, at the top level or in any field; +//! - any top-level value that is not an object; +//! - an `op` that is not byte-for-byte a known op (case, whitespace, separators, homoglyphs); +//! - a string field carrying NUL, ASCII whitespace, a traversal or a leading `-`, which would +//! split a dmsetup table or argv, escape a confined path, or read as a flag; +//! - an integer field given as a negative, a fraction, a string, a bool, a list or an exponent; +//! - a bool field given as anything but a JSON bool; +//! - an unknown key in any nested object, not just the top level, so a misspelled limit or +//! firewall key can never be silently dropped. + +use hyper_suidhelper::cli::{Request, RequestError}; +use proptest::prelude::*; +use rstest::rstest; +use serde_json::{json, Value}; +use std::ffi::OsString; +use test_strategy::proptest; + +const PARSER_DEPTH_LIMIT: usize = 128; +const MAX_ARG_STRLEN: usize = 128 * 1024; +const PLACEHOLDER: &str = "__hostile_placeholder__"; + +fn parse_raw(raw: &str) -> Result { + Request::from_args([OsString::from(raw)].into_iter()) +} + +fn parse(request: &Value) -> Result { + parse_raw(&request.to_string()) +} + +fn refused(raw: &str) -> bool { + matches!(parse_raw(raw), Err(RequestError::Json(_))) +} + +fn valid_requests() -> Vec { + let fixture: serde_json::Map = + serde_json::from_str(include_str!("fixtures/requests.json")).unwrap(); + fixture.into_iter().map(|(_, request)| request).collect() +} + +fn pointers(value: &Value, at: String, out: &mut Vec<(String, Value)>) { + out.push((at.clone(), value.clone())); + match value { + Value::Object(map) => { + for (key, child) in map { + pointers(child, format!("{at}/{key}"), out); + } + } + Value::Array(items) => { + for (idx, child) in items.iter().enumerate() { + pointers(child, format!("{at}/{idx}"), out); + } + } + _ => {} + } +} + +fn every_node() -> Vec<(Value, String, Value)> { + valid_requests() + .into_iter() + .flat_map(|request| { + let mut nodes = Vec::new(); + pointers(&request, String::new(), &mut nodes); + nodes + .into_iter() + .filter(|(at, _)| !at.is_empty()) + .map(move |(at, node)| (request.clone(), at, node)) + }) + .collect() +} + +fn replaced(request: &Value, at: &str, with: Value) -> Value { + let mut request = request.clone(); + *request.pointer_mut(at).unwrap() = with; + request +} + +fn op_node(at: &str) -> bool { + at == "/op" +} + +#[test] +fn every_fixture_request_is_accepted_unmutated() { + for request in valid_requests() { + assert!(parse(&request).is_ok(), "{request}"); + } +} + +#[rstest] +#[case::arrays("[", "]")] +#[case::objects("{\"k\":", "}")] +#[case::unclosed_arrays("[", "")] +fn nesting_past_the_parser_limit_is_refused_in_every_field( + #[case] open: &str, + #[case] close: &str, + #[values(PARSER_DEPTH_LIMIT + 1, 4_096, MAX_ARG_STRLEN / 8)] depth: usize, +) { + let deep = format!("{}0{}", open.repeat(depth), close.repeat(depth)); + let top_level = deep.clone(); + assert!(refused(&top_level), "top-level nesting depth {depth}"); + for (request, at, _) in every_node() { + if op_node(&at) { + continue; + } + let marked = replaced(&request, &at, json!(PLACEHOLDER)).to_string(); + let raw = marked.replace(&format!("\"{PLACEHOLDER}\""), &deep); + assert!(refused(&raw), "depth {depth} at {at} of {request}"); + } +} + +fn non_object() -> impl Strategy { + let leaf = prop_oneof![ + Just(Value::Null), + any::().prop_map(Value::from), + any::().prop_map(Value::from), + any::() + .prop_filter("finite", |f| f.is_finite()) + .prop_map(Value::from), + ".{0,16}".prop_map(Value::from), + ]; + let nested = leaf.clone().prop_recursive(3, 16, 4, |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..4).prop_map(Value::Array), + prop::collection::btree_map("op|id|[a-z]{1,4}", inner, 0..3) + .prop_map(|map| Value::Array(vec![Value::Object(map.into_iter().collect())])), + ] + }); + prop_oneof![leaf, nested] +} + +#[proptest] +fn a_top_level_value_that_is_not_an_object_is_refused(#[strategy(non_object())] request: Value) { + prop_assert!(refused(&request.to_string()), "{request}"); +} + +const OPS: &[&str] = &[ + "vm_launch", + "vm_claim_api_socket", + "vm_claim_vsock_socket", + "vm_teardown", + "vm_firewall", + "img_activate", + "dm_remove", + "thin_pool_create", + "thin_volume_create", + "thin_destroy", + "thin_fork_delta", + "host_init", + "version", +]; + +fn near_miss(op: &str, how: usize) -> String { + match how { + 0 => op.to_uppercase(), + 1 => format!(" {op}"), + 2 => format!("{op}\n"), + 3 => op.replace('_', "-"), + 4 => op.replacen('o', "\u{43e}", 1), + 5 => format!("{op}\0"), + 6 => op.replace('_', ""), + _ => { + let mut chars: Vec = op.chars().collect(); + chars[0] = chars[0].to_ascii_uppercase(); + chars.into_iter().collect() + } + } +} + +#[proptest] +fn an_op_that_is_not_exactly_a_known_op_is_refused( + #[strategy(0..OPS.len())] which: usize, + #[strategy(0usize..8)] how: usize, +) { + let canonical = valid_requests() + .into_iter() + .find(|request| request["op"] == OPS[which]) + .unwrap(); + let op = near_miss(OPS[which], how); + prop_assume!(op != OPS[which]); + let request = replaced(&canonical, "/op", json!(op)); + prop_assert!(refused(&request.to_string()), "{request}"); +} + +#[proptest] +fn an_arbitrary_op_name_is_refused(#[strategy("[ -~]{0,24}")] op: String) { + prop_assume!(!OPS.contains(&op.as_str())); + let request = json!({ "op": op }); + prop_assert!(refused(&request.to_string()), "{request}"); +} + +const SPLITTING_BYTES: &[&str] = &["\0", " ", "\t", "\n", "\r", "\u{b}", "\u{c}"]; + +fn spliced(original: &str, fragment: &str, cut: usize) -> String { + format!("{}{fragment}{}", &original[..cut], &original[cut..]) +} + +fn char_boundaries(s: &str) -> impl Iterator + '_ { + s.char_indices().map(|(idx, _)| idx).chain([s.len()]) +} + +fn traversals(original: &str) -> Vec { + let after_separators = original + .match_indices('/') + .map(|(idx, _)| spliced(original, "../", idx + 1)); + after_separators + .chain([ + format!("{original}/.."), + format!("../{original}"), + "..".to_owned(), + ]) + .collect() +} + +#[test] +fn a_splitting_byte_anywhere_in_any_string_field_is_refused() { + for (request, at, node) in every_node() { + let Value::String(original) = node else { + continue; + }; + for fragment in SPLITTING_BYTES { + for cut in char_boundaries(&original) { + let hostile = spliced(&original, fragment, cut); + let mutated = replaced(&request, &at, json!(hostile)); + assert!( + refused(&mutated.to_string()), + "accepted {hostile:?} at {at}: {mutated}" + ); + } + } + } +} + +#[test] +fn a_traversal_component_in_any_string_field_is_refused() { + for (request, at, node) in every_node() { + let Value::String(original) = node else { + continue; + }; + for hostile in traversals(&original) { + let mutated = replaced(&request, &at, json!(hostile)); + assert!( + refused(&mutated.to_string()), + "accepted {hostile:?} at {at}: {mutated}" + ); + } + } +} + +#[test] +fn a_leading_dash_on_any_string_field_is_refused() { + for (request, at, node) in every_node() { + let Value::String(original) = node else { + continue; + }; + for hostile in [ + format!("-{original}"), + format!("--{original}"), + "-".to_owned(), + ] { + let mutated = replaced(&request, &at, json!(hostile)); + assert!( + refused(&mutated.to_string()), + "accepted {hostile:?} at {at}" + ); + } + } +} + +#[test] +fn an_integer_field_in_any_other_json_shape_is_refused() { + let shapes = ["-1", "1.5", "\"1\"", "true", "1e3", "[1]", "{}"]; + for (request, at, node) in every_node() { + if !node.is_u64() { + continue; + } + for shape in shapes { + let marked = replaced(&request, &at, json!(PLACEHOLDER)).to_string(); + let raw = marked.replace(&format!("\"{PLACEHOLDER}\""), shape); + assert!(refused(&raw), "accepted {shape} at {at}: {raw}"); + } + } +} + +#[test] +fn a_bool_field_in_any_other_json_shape_is_refused() { + for (request, at, node) in every_node() { + if !node.is_boolean() { + continue; + } + for shape in [ + json!("true"), + json!(1), + json!(0), + Value::Null, + json!([true]), + ] { + let mutated = replaced(&request, &at, shape.clone()); + assert!(refused(&mutated.to_string()), "accepted {shape} at {at}"); + } + } +} + +#[test] +fn a_nested_object_or_list_in_the_wrong_shape_is_refused() { + for (request, at, node) in every_node() { + let wrong: &[Value] = match node { + Value::Object(_) => &[json!([]), json!("x"), json!(1)], + Value::Array(_) => &[json!({}), json!("x"), json!(1), Value::Null], + _ => continue, + }; + for shape in wrong { + let mutated = replaced(&request, &at, shape.clone()); + assert!( + refused(&mutated.to_string()), + "accepted {shape} at {at}: {mutated}" + ); + } + } +} + +#[proptest] +fn an_unknown_key_in_any_nested_object_is_refused( + #[strategy("[a-z_]{1,12}")] key: String, + #[strategy(any::())] which: prop::sample::Index, +) { + let objects: Vec<(Value, String, Value)> = every_node() + .into_iter() + .filter(|(_, _, node)| node.is_object()) + .collect(); + prop_assume!(!objects.is_empty()); + let (request, at, node) = which.get(&objects).clone(); + prop_assume!(!node.as_object().unwrap().contains_key(&key)); + let mut mutated = request.clone(); + mutated.pointer_mut(&at).unwrap()[&key] = json!(1); + prop_assert!( + refused(&mutated.to_string()), + "accepted {key} at {at}: {mutated}" + ); +} + +#[test] +fn every_nested_object_is_mutated_by_the_unknown_key_law() { + let nested: Vec = every_node() + .into_iter() + .filter(|(_, _, node)| node.is_object()) + .map(|(_, at, _)| at) + .collect(); + for expected in [ + "/limits", + "/limits/cpu_max", + "/firewall", + "/firewall/inbound/0", + ] { + assert!( + nested.iter().any(|at| at == expected), + "{expected} not covered: {nested:?}" + ); + } +} + +#[test] +fn a_misspelled_nested_key_never_silently_drops_a_limit() { + let launch = valid_requests() + .into_iter() + .find(|request| request["op"] == "vm_launch") + .unwrap(); + for (at, typo) in [ + ("/limits", json!({"memroy_max": 1})), + ( + "/limits", + json!({"memory_max": 1, "cpu": {"quota_us": 1, "period_us": 1}}), + ), + ("/limits/cpu_max", json!({"quota_us": 1, "period": 1})), + ("/firewall", json!({"inbound": [], "outbound_block": []})), + ( + "/firewall/inbound/0", + json!({"protocol": "tcp", "port": [22, 22]}), + ), + ] { + let mutated = replaced(&launch, at, typo.clone()); + assert!(refused(&mutated.to_string()), "accepted {typo} at {at}"); + } +} diff --git a/native/suidhelper/tests/e2e/caller_gate.rs b/native/suidhelper/tests/e2e/caller_gate.rs new file mode 100644 index 00000000..a03ea7f3 --- /dev/null +++ b/native/suidhelper/tests/e2e/caller_gate.rs @@ -0,0 +1,265 @@ +//! The #156 caller gate, end to end through the built binary: privileged ops run only for root +//! and members of the group the helper file belongs to. +//! +//! Each case installs a copy of the helper `root:` mode 0711, so a non-root caller cannot +//! read it and the kernel execs it non-dumpable, as it would a setuid binary, and runs it under an +//! exact uid, primary gid and supplementary list. The probe op is `vm_claim_api_socket`: once +//! past the gate its first act is raising euid, which a non-setuid copy cannot do, so a caller +//! the gate let through fails with `raising euid` and touches nothing. +//! +//! Laws: +//! - a non-root caller passes iff the install group is its primary gid or anywhere among its +//! supplementary gids; +//! - the refusal comes before the config is opened: a refused caller learns nothing about it, +//! while an admitted caller is the one who meets the config checks; +//! - `version` answers anyone and never reads the config; +//! - the gate keys on the inode that runs, not the path it was invoked by; +//! - root is admitted whatever its gids, even beside a root install group. +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use nix::unistd::{chown, setgroups, setresgid, setresuid, Gid, Uid}; +use pretty_assertions::assert_eq; +use proptest::prelude::*; +use rstest::rstest; +use serde_json::{json, Value}; +use std::fs; +use std::os::unix::fs::{symlink, PermissionsExt}; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use test_strategy::proptest; + +mod support; +use support::{is_root, BIN}; + +const NOBODY: u32 = 65_534; +const NODE: u32 = 54_321; +const PROBE: &str = r#"{"op":"vm_claim_api_socket","id":"vgate"}"#; +const PAST_THE_GATE: &str = "raising euid to root"; +const NOT_IN_GROUP: &str = "is not in the helper's group"; + +struct Install { + dir: TempDir, + bin: PathBuf, +} + +impl Install { + fn new(group: u32) -> Self { + let dir = TempDir::new().unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let bin = dir.path().join("hyper-suidhelper"); + fs::copy(BIN, &bin).unwrap(); + chown(&bin, Some(Uid::from_raw(0)), Some(Gid::from_raw(group))).unwrap(); + fs::set_permissions(&bin, fs::Permissions::from_mode(0o711)).unwrap(); + Self { dir, bin } + } + + fn config(&self, body: &str, owner: u32) -> PathBuf { + let path = self.dir.path().join("config.toml"); + fs::write(&path, body).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + chown( + &path, + Some(Uid::from_raw(owner)), + Some(Gid::from_raw(owner)), + ) + .unwrap(); + path + } + + fn absent_config(&self) -> PathBuf { + self.dir.path().join("absent.toml") + } +} + +fn run_as(bin: &Path, config: &Path, uid: u32, gid: u32, groups: &[u32], request: &str) -> Output { + let groups: Vec = groups.iter().copied().map(Gid::from_raw).collect(); + let mut cmd = Command::new(bin); + cmd.arg(request) + .env_clear() + .env("HYPER_SETUIDHELPER_IS_INSECURE_MODE", "1") + .env("HYPER_SETUIDHELPER_CONFIG_PATH", config) + .current_dir("/"); + // SAFETY: the hook runs in the forked child before exec and only makes id syscalls. + unsafe { + cmd.pre_exec(move || { + let (gid, uid) = (Gid::from_raw(gid), Uid::from_raw(uid)); + setgroups(&groups)?; + setresgid(gid, gid, gid)?; + setresuid(uid, uid, uid)?; + Ok(()) + }); + } + cmd.output().unwrap() +} + +fn stderr(out: &Output) -> String { + String::from_utf8_lossy(&out.stderr).into_owned() +} + +fn gids() -> impl Strategy { + prop_oneof![ + Just(0u32), + Just(NOBODY), + 1u32..60_000, + 60_000u32..4_000_000_000 + ] + .prop_filter("not the node group", |&g| g != NODE) +} + +#[proptest(cases = 24)] +fn a_caller_passes_iff_it_holds_the_install_group_as_root( + #[strategy(gids())] primary: u32, + #[strategy(proptest::collection::vec(gids(), 0..64))] mut groups: Vec, + member: bool, + at: usize, +) { + if !is_root() { + eprintln!("SKIP caller gate property: needs root to install the helper and switch ids"); + return Ok(()); + } + if member { + groups.insert(at % (groups.len() + 1), NODE); + } + let install = Install::new(NODE); + + let out = run_as( + &install.bin, + &install.absent_config(), + NOBODY, + primary, + &groups, + PROBE, + ); + + let stderr = stderr(&out); + prop_assert_eq!(out.status.code(), Some(2), "{}", stderr); + if member { + prop_assert!(stderr.contains(PAST_THE_GATE), "member refused: {}", stderr); + } else { + prop_assert!( + stderr.contains(NOT_IN_GROUP), + "non-member admitted: {}", + stderr + ); + prop_assert!( + stderr.contains(&format!("uid {NOBODY} ")) && stderr.contains(&NODE.to_string()), + "{}", + stderr + ); + } +} + +const UNTRUSTED_CONFIG: &str = "not owned by root:root"; + +#[rstest] +#[case::a_non_member_is_refused_without_seeing_the_config(&[], NOT_IN_GROUP, UNTRUSTED_CONFIG)] +#[case::a_member_is_the_one_who_meets_the_config_check(&[NODE], UNTRUSTED_CONFIG, PAST_THE_GATE)] +fn the_gate_refuses_before_the_config_is_opened_as_root( + #[case] groups: &[u32], + #[case] refusal: &str, + #[case] unreached: &str, +) { + if !is_root() { + eprintln!("SKIP gate-before-config: needs root"); + return; + } + let install = Install::new(NODE); + let untrusted = install.config("work_dir = \"/srv/hyper\"\n", NOBODY); + + let out = run_as(&install.bin, &untrusted, NOBODY, NOBODY, groups, PROBE); + + let stderr = stderr(&out); + assert_eq!(out.status.code(), Some(2), "{stderr}"); + assert!(stderr.contains(refusal), "{stderr}"); + assert!(!stderr.contains(unreached), "{stderr}"); +} + +#[rstest] +#[case::untrusted_config(true)] +#[case::malformed_config(false)] +fn version_answers_a_non_member_without_reading_the_config_as_root(#[case] untrusted: bool) { + if !is_root() { + eprintln!("SKIP version for non-members: needs root"); + return; + } + let install = Install::new(NODE); + let config = if untrusted { + install.config("work_dir = \"/srv/hyper\"\n", NOBODY) + } else { + install.config("this is = not = toml", 0) + }; + + let out = run_as( + &install.bin, + &config, + NOBODY, + NOBODY, + &[], + r#"{"op":"version"}"#, + ); + + assert_eq!(out.status.code(), Some(0), "{}", stderr(&out)); + let report: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(report["version"], env!("CARGO_PKG_VERSION")); +} + +#[rstest] +#[case::member(&[NODE], PAST_THE_GATE)] +#[case::non_member(&[], NOT_IN_GROUP)] +fn the_gate_keys_on_the_running_inode_not_the_invoked_path_as_root( + #[case] groups: &[u32], + #[case] expected: &str, +) { + if !is_root() { + eprintln!("SKIP gate via symlink: needs root"); + return; + } + let install = Install::new(NODE); + let elsewhere = TempDir::new().unwrap(); + fs::set_permissions(elsewhere.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let link = elsewhere.path().join("hyper-suidhelper"); + symlink(&install.bin, &link).unwrap(); + chown( + elsewhere.path(), + Some(Uid::from_raw(NOBODY)), + Some(Gid::from_raw(NOBODY)), + ) + .unwrap(); + + let out = run_as( + &link, + &install.absent_config(), + NOBODY, + NOBODY, + groups, + PROBE, + ); + + assert!(stderr(&out).contains(expected), "{}", stderr(&out)); +} + +#[rstest] +#[case::root_install_group(0)] +#[case::node_install_group(NODE)] +fn root_is_admitted_whatever_its_gids_as_root(#[case] group: u32) { + if !is_root() { + eprintln!("SKIP root bypass: needs root"); + return; + } + let install = Install::new(group); + + let out = run_as( + &install.bin, + &install.absent_config(), + 0, + NOBODY, + &[NOBODY], + PROBE, + ); + + assert_eq!(out.status.code(), Some(0), "{}", stderr(&out)); + let reply: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(reply, json!({"result": "pending"})); +} diff --git a/native/suidhelper/tests/e2e/claim_privacy.rs b/native/suidhelper/tests/e2e/claim_privacy.rs new file mode 100644 index 00000000..49bf09c4 --- /dev/null +++ b/native/suidhelper/tests/e2e/claim_privacy.rs @@ -0,0 +1,298 @@ +//! `vm_claim_*_socket` through a real setuid install, called by a non-root member of the helper's +//! group the way the node calls it (#135: "keep VM sockets private"). +//! +//! Laws: +//! - the pin is the jail socket's own inode, handed to the *calling* user and group (never to the +//! effective root the setuid bit grants), mode `0600`; +//! - once claimed, only that caller and root can `connect()` to the VM's API or vsock socket, by +//! the pin or by the jail path: another member of the helper's group, another VM's jail uid and +//! `nobody` are all refused `EACCES` by the socket itself, even through a traversable dir; +//! - a socket dir that is a symlink is refused (`ELOOP`) by both claim and teardown, and nothing +//! is linked into or unlinked from the symlink's target; +//! - a socket dir that is not a directory is a refusal, never a crash. +#![cfg(feature = "insecure_mode")] + +use nix::unistd::{Gid, Uid}; +use pretty_assertions::assert_eq; +use rstest::rstest; +use serde_json::json; +use std::fs; +use std::os::unix::fs::{symlink, FileTypeExt, MetadataExt, PermissionsExt}; +use std::os::unix::net::UnixListener; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::Command; + +mod setuid_support; +mod support; +use setuid_support::{run_as, Caller, Install, MEMBER, OTHER_MEMBER, ROOT}; +use support::is_root; + +const ID: &str = "v1"; + +const SOCKETS: [(&str, &str); 2] = [ + ("vm_claim_api_socket", "api.socket"), + ("vm_claim_vsock_socket", "vsock.sock"), +]; + +const OTHER_VMS_JAIL: Caller = Caller { + uid: 900_003, + gid: 900_003, + groups: &[], +}; + +const NOBODY: Caller = Caller { + uid: 65_534, + gid: 65_534, + groups: &[], +}; + +struct Vm { + install: Install, + config: PathBuf, + srv: PathBuf, + _listeners: Vec, +} + +impl Vm { + fn new() -> Self { + let install = Install::new(); + let srv = install.path().join("srv"); + let root = srv.join("jails/firecracker").join(ID).join("root"); + fs::create_dir_all(&root).unwrap(); + let socks = srv.join("socks"); + fs::create_dir_all(&socks).unwrap(); + chown(&socks, MEMBER); + fs::set_permissions(&socks, fs::Permissions::from_mode(0o755)).unwrap(); + let listeners = SOCKETS + .iter() + .map(|(_, socket)| { + let path = root.join(socket); + let listener = UnixListener::bind(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o777)).unwrap(); + listener + }) + .collect(); + let config = install.config(&srv); + Self { + install, + config, + srv, + _listeners: listeners, + } + } + + fn jail_socket(&self, socket: &str) -> PathBuf { + self.srv + .join("jails/firecracker") + .join(ID) + .join("root") + .join(socket) + } + + fn pin(&self, socket: &str) -> PathBuf { + self.srv.join("socks").join(format!("fc-{ID}-{socket}")) + } +} + +fn chown(path: &Path, caller: Caller) { + nix::unistd::chown( + path, + Some(Uid::from_raw(caller.uid)), + Some(Gid::from_raw(caller.gid)), + ) + .unwrap(); +} + +/// `connect()` to `path` as `caller`: `0` on success, otherwise the errno. +fn connect_as(caller: Caller, path: &Path) -> i32 { + let Caller { uid, gid, groups } = caller; + let mut cmd = Command::new("python3"); + cmd.args([ + "-I", + "-c", + "import socket,sys\ns=socket.socket(socket.AF_UNIX)\ntry:\n s.connect(sys.argv[1]); print(0)\nexcept OSError as e:\n print(e.errno)", + ]) + .arg(path) + .env_clear(); + // SAFETY: the hook runs in the forked child before exec and makes only raw id syscalls. + unsafe { + cmd.pre_exec(move || { + let groups: Vec = groups.iter().copied().map(Gid::from_raw).collect(); + nix::unistd::setgroups(&groups)?; + nix::unistd::setresgid(Gid::from_raw(gid), Gid::from_raw(gid), Gid::from_raw(gid))?; + nix::unistd::setresuid(Uid::from_raw(uid), Uid::from_raw(uid), Uid::from_raw(uid))?; + Ok(()) + }) + }; + let out = cmd.output().unwrap(); + String::from_utf8_lossy(&out.stdout) + .trim() + .parse() + .unwrap_or_else(|_| panic!("connect probe failed: {out:?}")) +} + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn a_member_claim_hands_the_socket_to_the_caller_alone_as_root(#[case] (op, socket): (&str, &str)) { + if !is_root() { + eprintln!("SKIP setuid claim: needs root to install setuid"); + return; + } + let vm = Vm::new(); + + let out = run_as( + &vm.install, + &vm.config, + MEMBER, + &json!({"op": op, "id": ID}), + ); + + let pin = fs::symlink_metadata(vm.pin(socket)).unwrap(); + let jail = fs::symlink_metadata(vm.jail_socket(socket)).unwrap(); + assert_eq!( + ( + out.code, + out.stdout.trim(), + pin.file_type().is_socket(), + (pin.dev(), pin.ino()), + (pin.uid(), pin.gid()), + pin.mode() & 0o7777, + ), + ( + Some(0), + r#"{"result":"claimed"}"#, + true, + (jail.dev(), jail.ino()), + (MEMBER.uid, MEMBER.gid), + 0o600, + ), + "{}", + out.stderr + ); +} + +const EACCES: i32 = 13; + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn once_claimed_only_the_caller_and_root_can_connect_as_root(#[case] (op, socket): (&str, &str)) { + if !is_root() { + eprintln!("SKIP setuid claim privacy: needs root to install setuid"); + return; + } + let vm = Vm::new(); + run_as( + &vm.install, + &vm.config, + MEMBER, + &json!({"op": op, "id": ID}), + ); + + let callers = [ + ("caller", MEMBER), + ("root", ROOT), + ("other member", OTHER_MEMBER), + ("other VM's jail uid", OTHER_VMS_JAIL), + ("nobody", NOBODY), + ]; + let got: Vec<(&str, i32, i32)> = callers + .iter() + .map(|&(who, caller)| { + ( + who, + connect_as(caller, &vm.pin(socket)), + connect_as(caller, &vm.jail_socket(socket)), + ) + }) + .collect(); + + assert_eq!( + got, + vec![ + ("caller", 0, 0), + ("root", 0, 0), + ("other member", EACCES, EACCES), + ("other VM's jail uid", EACCES, EACCES), + ("nobody", EACCES, EACCES), + ] + ); +} + +#[rstest] +#[case::claim_api(json!({"op": "vm_claim_api_socket", "id": ID}))] +#[case::claim_vsock(json!({"op": "vm_claim_vsock_socket", "id": ID}))] +#[case::teardown(json!({"op": "vm_teardown", "id": ID}))] +fn a_symlinked_socket_dir_is_refused_and_its_target_untouched_as_root( + #[case] request: serde_json::Value, +) { + if !is_root() { + eprintln!("SKIP symlinked socket dir: needs root to install setuid"); + return; + } + let vm = Vm::new(); + let elsewhere = vm.install.path().join("elsewhere"); + fs::create_dir(&elsewhere).unwrap(); + chown(&elsewhere, MEMBER); + let decoy = elsewhere.join(format!("fc-{ID}-vsock.sock")); + fs::write(&decoy, b"decoy").unwrap(); + fs::remove_dir(vm.srv.join("socks")).unwrap(); + symlink(&elsewhere, vm.srv.join("socks")).unwrap(); + + let out = run_as(&vm.install, &vm.config, MEMBER, &request); + + let mut left: Vec = fs::read_dir(&elsewhere) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + assert_eq!( + ( + out.code, + out.stderr.contains("ELOOP"), + left, + fs::read(&decoy).unwrap() + ), + ( + Some(2), + true, + vec![format!("fc-{ID}-vsock.sock")], + b"decoy".to_vec() + ), + "{request}: {}", + out.stderr + ); +} + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn a_socket_dir_that_is_a_file_is_refused_not_crashed_as_root(#[case] (op, _): (&str, &str)) { + if !is_root() { + eprintln!("SKIP socket dir file: needs root to install setuid"); + return; + } + let vm = Vm::new(); + fs::remove_dir(vm.srv.join("socks")).unwrap(); + fs::write(vm.srv.join("socks"), b"not a dir").unwrap(); + + let out = run_as( + &vm.install, + &vm.config, + MEMBER, + &json!({"op": op, "id": ID}), + ); + + assert_eq!( + ( + out.code, + out.stdout.as_str(), + out.stderr.contains("socket dir") + ), + (Some(2), "", true), + "{}", + out.stderr + ); +} diff --git a/native/suidhelper/tests/e2e/fork_delta_live.rs b/native/suidhelper/tests/e2e/fork_delta_live.rs new file mode 100644 index 00000000..0d9436c9 --- /dev/null +++ b/native/suidhelper/tests/e2e/fork_delta_live.rs @@ -0,0 +1,314 @@ +//! `thin_fork_delta` against a real dm-thin pool, driven through the helper +//! binary. The oracle is the parent itself: replaying the produced COW store +//! as a dm-snapshot over the origin must read back byte-for-byte as the parent. +//! Laws: +//! * for ANY set of whole- and partial-block writes over a non-zero origin +//! (including none at all), replay == parent; +//! * nothing outlives a call: no `hyper-scratch-*`/`hyper-copy-*` device, +//! the scratch thin id is free for reuse, the metadata snapshot is +//! released, and the parent is still live and unchanged; +//! * `written_chunks` is zero exactly when nothing diverged and never more +//! than the number of diverged blocks (each block fits one copy chunk); +//! * two forks of different parents of one pool racing each other both +//! produce their own parent's exact divergence. +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use pretty_assertions::assert_eq; +use proptest::prelude::*; +use proptest::strategy::ValueTree; +use proptest::test_runner::{Config as RunnerConfig, TestRunner}; +use serde_json::json; +use std::collections::BTreeSet; +use std::fs; +use std::io::{Seek, SeekFrom, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; + +mod support; +use support::{is_root, run, stdout_json, DmDevice, LoopDevice, MapperDevice}; + +const BLOCK_SECTORS: u64 = 128; +const BLOCK_BYTES: u64 = BLOCK_SECTORS * 512; +const BLOCKS: u64 = 32; +const SIZE: u64 = BLOCKS * BLOCK_BYTES; +const SCRATCH_ID: u32 = 900; + +fn system_bin(name: &str) -> Option { + let out = Command::new("sh") + .args(["-c", &format!("command -v {name}")]) + .output() + .ok()?; + fs::canonicalize(String::from_utf8(out.stdout).ok()?.trim()).ok() +} + +fn sparse(path: &Path, bytes: u64) { + fs::File::create(path).unwrap().set_len(bytes).unwrap(); +} + +struct Live { + origin: MapperDevice, + pool: DmDevice, + _data: LoopDevice, + meta: LoopDevice, + config: PathBuf, + base: PathBuf, + _dir: TempDir, + tag: String, +} + +impl Live { + fn new(tag: &str) -> Option { + let (dmsetup, thin_dump) = (system_bin("dmsetup")?, system_bin("thin_dump")?); + let dir = TempDir::new().unwrap(); + let base = fs::canonicalize(dir.path()).unwrap(); + let thin_dump_copy = base.join("thin_dump"); + fs::copy(&thin_dump, &thin_dump_copy).unwrap(); + fs::set_permissions(&thin_dump_copy, fs::Permissions::from_mode(0o755)).unwrap(); + let config = base.join("config.toml"); + fs::write( + &config, + format!( + "work_dir = \"{}\"\n[tools]\ndmsetup = \"{}\"\nthin_dump = \"{}\"\n", + base.display(), + dmsetup.display(), + thin_dump_copy.display() + ), + ) + .unwrap(); + fs::set_permissions(&config, fs::Permissions::from_mode(0o644)).unwrap(); + + sparse(&base.join("meta.img"), 8 << 20); + sparse(&base.join("data.img"), 64 << 20); + let origin_bytes: Vec = (0..SIZE).map(|i| (i % 241) as u8 | 0x80).collect(); + fs::write(base.join("origin.img"), origin_bytes).unwrap(); + let meta = LoopDevice::attach(&base.join("meta.img"))?; + let data = LoopDevice::attach(&base.join("data.img"))?; + let tag = format!("hyper-e2e-fdl-{tag}-{}", std::process::id()); + let pool = DmDevice::create( + &format!("{tag}-pool"), + &format!( + "0 {} thin-pool {} {} {BLOCK_SECTORS} 0", + (64u64 << 20) / 512, + meta.path(), + data.path() + ), + )?; + let origin = MapperDevice::linear(&format!("{tag}-origin"), &base.join("origin.img"))?; + Some(Self { + origin, + pool, + _data: data, + meta, + config, + base, + _dir: dir, + tag, + }) + } + + fn pool_name(&self) -> String { + format!("{}-pool", self.tag) + } + + fn parent(&self, id: u32) -> DmDevice { + assert!(self.pool.message(&format!("create_thin {id}"))); + DmDevice::create( + &format!("{}-parent-{id}", self.tag), + &format!( + "0 {} thin {} {id} {}", + SIZE / 512, + self.pool.path(), + self.origin.path() + ), + ) + .expect("parent thin") + } + + fn fork(&self, parent: &DmDevice, scratch_id: u32, cow: &Path) -> assert_cmd::assert::Assert { + sparse(cow, 8 << 20); + run( + &self.config, + json!({ + "op": "thin_fork_delta", + "pool": self.pool_name(), + "meta": self.meta.path(), + "parent": parent.path().trim_start_matches("/dev/mapper/"), + "scratch_id": scratch_id, + "origin": self.origin.path(), + "cow": cow, + "chunk_sectors": 8, + }), + ) + } + + fn replay(&self, cow: &Path) -> Vec { + let lo = LoopDevice::attach(cow).expect("cow loop"); + let replay = DmDevice::create( + &format!("{}-replay", self.tag), + &format!( + "0 {} snapshot {} {} P 8", + SIZE / 512, + self.origin.path(), + lo.path() + ), + ) + .expect("the cow store replays as a snapshot"); + fs::read(replay.path()).unwrap() + } + + fn assert_nothing_outlived(&self, scratch_id: u32) { + let listed = Command::new("dmsetup").arg("ls").output().unwrap(); + let listed = String::from_utf8_lossy(&listed.stdout); + for prefix in ["hyper-scratch-", "hyper-copy-"] { + let leftover = format!("{prefix}{scratch_id}"); + assert!( + !listed + .lines() + .any(|l| l.split_whitespace().next() == Some(&leftover)), + "{leftover} outlived the call:\n{listed}" + ); + } + assert!( + self.pool.message("reserve_metadata_snap"), + "the metadata snapshot was left reserved" + ); + assert!(self.pool.message("release_metadata_snap")); + assert!( + self.pool.message(&format!("create_thin {scratch_id}")), + "scratch id {scratch_id} was not freed" + ); + assert!(self.pool.message(&format!("delete {scratch_id}"))); + } +} + +#[derive(Debug, Clone)] +enum Divergence { + Block { + block: u64, + fill: u8, + }, + Partial { + block: u64, + offset: u64, + len: u64, + fill: u8, + }, +} + +impl Divergence { + fn block(&self) -> u64 { + match *self { + Divergence::Block { block, .. } | Divergence::Partial { block, .. } => block, + } + } + + fn apply(&self, dev: &str) { + let (at, len, fill) = match *self { + Divergence::Block { block, fill } => (block * BLOCK_BYTES, BLOCK_BYTES, fill), + Divergence::Partial { + block, + offset, + len, + fill, + } => (block * BLOCK_BYTES + offset * 512, len * 512, fill), + }; + let mut f = fs::OpenOptions::new().write(true).open(dev).unwrap(); + f.seek(SeekFrom::Start(at)).unwrap(); + f.write_all(&vec![fill; len as usize]).unwrap(); + f.sync_all().unwrap(); + } +} + +fn writes() -> impl Strategy> { + let one = prop_oneof![ + (0..BLOCKS, any::()).prop_map(|(block, fill)| Divergence::Block { block, fill }), + (0..BLOCKS, 0u64..BLOCK_SECTORS, any::()).prop_flat_map(|(block, offset, fill)| { + (1..=BLOCK_SECTORS - offset).prop_map(move |len| Divergence::Partial { + block, + offset, + len, + fill, + }) + }), + ]; + prop::collection::vec(one, 0..10) +} + +macro_rules! require { + ($live:expr) => { + match (is_root(), $live) { + (true, Some(live)) => live, + _ => { + eprintln!("SKIP: needs root, dmsetup, thin_dump and dm-thin"); + return; + } + } + }; +} + +#[test] +fn fork_delta_replays_any_divergence_exactly_and_leaves_nothing_behind_as_root() { + let live = require!(Live::new("prop")); + let mut runner = TestRunner::new(RunnerConfig { + cases: 8, + failure_persistence: None, + ..RunnerConfig::default() + }); + let mut cases: Vec> = vec![Vec::new()]; + for _ in 0..7 { + cases.push(writes().new_tree(&mut runner).unwrap().current()); + } + + for (n, case) in cases.iter().enumerate() { + let parent = live.parent(10 + n as u32); + for w in case { + w.apply(&parent.path()); + } + let before = fs::read(parent.path()).unwrap(); + let cow = live.base.join(format!("cow-{n}.img")); + + let out = live.fork(&parent, SCRATCH_ID, &cow).success(); + + let diverged: BTreeSet = case.iter().map(Divergence::block).collect(); + let written = stdout_json(&out)["written_chunks"].as_u64().unwrap(); + assert_eq!(written == 0, diverged.is_empty(), "{case:?}: {written}"); + assert!(written <= diverged.len() as u64, "{case:?}: {written}"); + assert!( + live.replay(&cow) == before, + "replay differs from the parent: {case:?}" + ); + assert!( + fs::read(parent.path()).unwrap() == before, + "the fork changed the parent: {case:?}" + ); + live.assert_nothing_outlived(SCRATCH_ID); + } +} + +#[test] +fn racing_forks_of_two_parents_each_get_their_own_divergence_as_root() { + let live = require!(Live::new("race")); + let (a, b) = (live.parent(20), live.parent(21)); + for block in [0, 7, BLOCKS - 1] { + Divergence::Block { block, fill: 0x0a }.apply(&a.path()); + } + for block in [3, 7, 19] { + Divergence::Block { block, fill: 0x0b }.apply(&b.path()); + } + let (want_a, want_b) = (fs::read(a.path()).unwrap(), fs::read(b.path()).unwrap()); + let (cow_a, cow_b) = (live.base.join("cow-a.img"), live.base.join("cow-b.img")); + + std::thread::scope(|s| { + let ta = s.spawn(|| live.fork(&a, 901, &cow_a).try_success().is_ok()); + let tb = s.spawn(|| live.fork(&b, 902, &cow_b).try_success().is_ok()); + assert!(ta.join().unwrap(), "fork of a failed under contention"); + assert!(tb.join().unwrap(), "fork of b failed under contention"); + }); + + assert!(live.replay(&cow_a) == want_a, "a's replay is not a"); + assert!(live.replay(&cow_b) == want_b, "b's replay is not b"); + live.assert_nothing_outlived(901); + live.assert_nothing_outlived(902); +} diff --git a/native/suidhelper/tests/e2e/jailer.rs b/native/suidhelper/tests/e2e/jailer.rs index 2bf96344..f70ab22d 100644 --- a/native/suidhelper/tests/e2e/jailer.rs +++ b/native/suidhelper/tests/e2e/jailer.rs @@ -417,3 +417,96 @@ fn teardown_waits_for_the_vm_lock_an_in_flight_launch_holds_as_root(launch: Opti String::from_utf8_lossy(&out.stderr) ); } + +fn vm_lock_dir() -> SafeDir { + let locks = PathBuf::from("/run/hyper-suidhelper/locks"); + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&locks) + .unwrap(); + fs::set_permissions("/run/hyper-suidhelper", fs::Permissions::from_mode(0o755)).unwrap(); + let locks: SafePathBuf = locks.try_into().unwrap(); + SafeFile::open(&locks, OpenFlags::RDONLY).unwrap() +} + +#[rstest] +fn launch_waits_for_a_teardown_holding_the_vm_lock_before_touching_the_jail_as_root( + launch: Option, +) { + let launch = require!(launch); + let cfg = launch.config(""); + let stale = launch.plant_stale_jail(); + let dir = vm_lock_dir(); + let name = format!("vm-{}", launch.id); + let held = FileLock::acquire( + &dir, + name.as_str().try_into().unwrap(), + Duration::from_secs(5), + ) + .unwrap(); + + let request = launch.request(json!({})).to_string(); + let mut child = launch + .ns + .get() + .scoped(|| { + support::command(&cfg, &[&request]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() + }) + .unwrap(); + thread::sleep(Duration::from_millis(1500)); + let while_held = ( + child.try_wait().unwrap().is_none(), + stale.exists(), + launch.recorded_argv().is_none(), + ); + drop(held); + let out = child.wait_with_output().unwrap(); + + assert_eq!( + ( + while_held, + out.status.code(), + stale.exists(), + launch.recorded_argv().is_some() + ), + ((true, true, true), Some(RECORDER_EXIT), false, true), + "{}", + String::from_utf8_lossy(&out.stderr) + ); +} + +#[rstest] +fn a_launch_that_never_gets_the_vm_lock_stages_nothing_and_never_execs_as_root( + launch: Option, +) { + let launch = require!(launch); + let cfg = launch.config(""); + let stale = launch.plant_stale_jail(); + let dir = vm_lock_dir(); + let name = format!("vm-{}", launch.id); + let _held = FileLock::acquire( + &dir, + name.as_str().try_into().unwrap(), + Duration::from_secs(5), + ) + .unwrap(); + + let out = launch.run(&cfg, launch.request(json!({}))).failure(); + + assert_eq!( + ( + out.get_output().status.code() == Some(RECORDER_EXIT), + stale.exists(), + launch.recorded_argv().is_none(), + Path::new("/run/netns").join(launch.netns()).exists(), + ), + (false, true, true, false), + "{}", + String::from_utf8_lossy(&out.get_output().stderr) + ); +} diff --git a/native/suidhelper/tests/e2e/lock_dir_hostile.rs b/native/suidhelper/tests/e2e/lock_dir_hostile.rs new file mode 100644 index 00000000..6fe66bbc --- /dev/null +++ b/native/suidhelper/tests/e2e/lock_dir_hostile.rs @@ -0,0 +1,172 @@ +//! The per-VM lock that serializes `vm_launch` and `vm_teardown` (#135) lives in the fixed +//! `/run/hyper-suidhelper/locks`. Whoever controls that directory controls the serialization, so +//! the helper must only ever lock inside a directory chain only root can write. Driven through a +//! real setuid install by a non-root member of its group, each case in a private mount namespace +//! whose `/run` is a fresh tmpfs, so the host's own lock dir is never touched. +//! +//! Laws: +//! - on an empty `/run`, the first member caller (even with umask 000) creates +//! `/run/hyper-suidhelper` as `root:root 0755` and `locks` as `root:root 0700`, and a teardown +//! leaves no lock file behind; +//! - a `/run/hyper-suidhelper` or `locks` that is not root-owned, is group- or world-writable, is a +//! symlink, or is not a directory, and a lock name that is a symlink, are all refused with a lock +//! error: the teardown then touches nothing (the jail and socket pins survive) and nothing is +//! created at a symlink's target. +#![cfg(feature = "insecure_mode")] + +use pretty_assertions::assert_eq; +use rstest::rstest; +use serde_json::json; +use std::fs; +use std::path::PathBuf; + +mod setuid_support; +mod support; +use setuid_support::{run_in_fresh_run, Install, MEMBER}; +use support::is_root; + +const ID: &str = "v1"; + +struct Vm { + install: Install, + config: PathBuf, + jail_marker: PathBuf, + pin: PathBuf, +} + +fn vm() -> Vm { + let install = Install::new(); + let srv = install.path().join("srv"); + let root = srv.join("jails/firecracker").join(ID).join("root"); + fs::create_dir_all(&root).unwrap(); + let jail_marker = root.join("marker"); + fs::write(&jail_marker, b"still here").unwrap(); + let socks = srv.join("socks"); + fs::create_dir_all(&socks).unwrap(); + nix::unistd::chown( + &socks, + Some(nix::unistd::Uid::from_raw(MEMBER.uid)), + Some(nix::unistd::Gid::from_raw(MEMBER.gid)), + ) + .unwrap(); + let pin = socks.join(format!("fc-{ID}-api.socket")); + fs::write(&pin, b"").unwrap(); + let config = install.config(&srv); + Vm { + install, + config, + jail_marker, + pin, + } +} + +#[test] +fn a_member_on_a_fresh_run_creates_a_root_only_lock_dir_as_root() { + if !is_root() { + eprintln!("SKIP fresh lock dir: needs root to install setuid"); + return; + } + let vm = vm(); + + let out = run_in_fresh_run( + &vm.install, + &vm.config, + MEMBER, + "umask 000", + &json!({"op": "vm_teardown", "id": ID}), + &[ + "/run/hyper-suidhelper", + "/run/hyper-suidhelper/locks", + "/run/hyper-suidhelper/locks/vm-v1", + ], + ); + + assert_eq!( + ( + out.code, + out.stdout.trim(), + out.state.as_str(), + vm.jail_marker.exists(), + vm.pin.exists(), + ), + ( + Some(0), + r#"{"result":"tore_down"}"#, + "/run/hyper-suidhelper directory 755 0 0\n/run/hyper-suidhelper/locks directory 700 0 0\n", + false, + false, + ), + "{}", + out.stderr + ); +} + +#[rstest] +#[case::member_owned_helper_dir( + "mkdir -m 755 /run/hyper-suidhelper; chown 62010:62001 /run/hyper-suidhelper", + "" +)] +#[case::world_writable_sticky_helper_dir("mkdir -m 1777 /run/hyper-suidhelper", "")] +#[case::group_writable_helper_dir("mkdir -m 775 /run/hyper-suidhelper", "")] +#[case::helper_dir_is_a_file("touch /run/hyper-suidhelper", "")] +#[case::helper_dir_symlinked_to_a_member_dir( + "mkdir -m 755 /run/attacker; mkdir -m 700 /run/attacker/locks; chown 62010 /run/attacker; \ + ln -s /run/attacker /run/hyper-suidhelper", + "/run/attacker/locks/vm-v1" +)] +#[case::member_owned_locks( + "mkdir -m 755 /run/hyper-suidhelper; mkdir -m 700 /run/hyper-suidhelper/locks; \ + chown 62010 /run/hyper-suidhelper/locks", + "" +)] +#[case::group_writable_locks( + "mkdir -m 755 /run/hyper-suidhelper; mkdir -m 730 /run/hyper-suidhelper/locks", + "" +)] +#[case::locks_symlinked_to_a_root_dir( + "mkdir -m 755 /run/hyper-suidhelper; mkdir -m 700 /run/elsewhere; \ + ln -s /run/elsewhere /run/hyper-suidhelper/locks", + "/run/elsewhere/vm-v1" +)] +#[case::lock_name_symlinked( + "mkdir -m 755 /run/hyper-suidhelper; mkdir -m 700 /run/hyper-suidhelper/locks; \ + ln -s /run/victim /run/hyper-suidhelper/locks/vm-v1", + "/run/victim" +)] +fn a_lock_dir_root_does_not_solely_control_is_refused_and_teardown_touches_nothing_as_root( + #[case] setup: &str, + #[case] never_created: &str, +) { + if !is_root() { + eprintln!("SKIP hostile lock dir: needs root to install setuid"); + return; + } + let vm = vm(); + let observe: Vec<&str> = [never_created] + .into_iter() + .filter(|p| !p.is_empty()) + .collect(); + + let out = run_in_fresh_run( + &vm.install, + &vm.config, + MEMBER, + setup, + &json!({"op": "vm_teardown", "id": ID}), + &observe, + ); + + assert_eq!( + ( + out.code, + out.stdout.as_str(), + out.stderr.contains("lock"), + vm.jail_marker.exists(), + vm.pin.exists(), + out.state.as_str(), + ), + (Some(2), "", true, true, true, ""), + "{setup}: {}", + out.stderr + ); +} diff --git a/native/suidhelper/tests/e2e/setuid_support.rs b/native/suidhelper/tests/e2e/setuid_support.rs new file mode 100644 index 00000000..16c9e76c --- /dev/null +++ b/native/suidhelper/tests/e2e/setuid_support.rs @@ -0,0 +1,275 @@ +//! A real setuid-root install of the static insecure-mode helper, driven by a non-root member of +//! its group, optionally inside a private mount namespace whose `/run` is a fresh tmpfs, so the +//! helper's fixed `/run/hyper-suidhelper` lock dir can be staged hostile without touching the +//! host's. + +// Each [[test]] target compiles its own copy of this module and not every target uses every item. +#![allow(dead_code)] + +use assert_fs::TempDir; +use nix::unistd::{Gid, Uid}; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR"); + +pub const INSTALL_GROUP: u32 = 62_001; + +#[derive(Clone, Copy, Debug)] +pub struct Caller { + pub uid: u32, + pub gid: u32, + pub groups: &'static [u32], +} + +pub const MEMBER: Caller = Caller { + uid: 62_010, + gid: INSTALL_GROUP, + groups: &[], +}; + +pub const OTHER_MEMBER: Caller = Caller { + uid: 62_011, + gid: 62_020, + groups: &[INSTALL_GROUP], +}; + +pub const ROOT: Caller = Caller { + uid: 0, + gid: 0, + groups: &[0], +}; + +fn home_of(uid: u32) -> Option { + nix::unistd::User::from_uid(Uid::from_raw(uid)) + .ok() + .flatten() + .map(|user| user.dir) +} + +/// The insecure-mode helper statically linked for musl, the only kind that runs setuid. +pub fn static_insecure_helper() -> PathBuf { + let target = format!("{}-unknown-linux-musl", std::env::consts::ARCH); + let target_dir = Path::new(MANIFEST_DIR).join("../target/setuid-e2e/insecure"); + let mut cargo = Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned())); + cargo + .current_dir(MANIFEST_DIR) + .args([ + "build", + "--quiet", + "-p", + "hyper-suidhelper", + "--bin", + "hyper-suidhelper", + "--features", + "insecure_mode", + "--target", + &target, + "--target-dir", + ]) + .arg(&target_dir); + for coverage_or_target in [ + "RUSTFLAGS", + "CARGO_ENCODED_RUSTFLAGS", + "RUSTC_WRAPPER", + "RUSTC_WORKSPACE_WRAPPER", + "LLVM_PROFILE_FILE", + "CARGO_TARGET_DIR", + "CARGO_BUILD_TARGET_DIR", + "CARGO_LLVM_COV", + "CARGO_LLVM_COV_TARGET_DIR", + ] { + cargo.env_remove(coverage_or_target); + } + let invoker = std::env::var("SUDO_UID") + .ok() + .zip(std::env::var("SUDO_GID").ok()) + .and_then(|(uid, gid)| Some((uid.parse::().ok()?, gid.parse::().ok()?))); + if let Some((uid, gid)) = invoker { + if let Some(home) = home_of(uid) { + cargo.env("HOME", home); + } + // Root's cargo exports its own CARGO_HOME, which the invoker cannot write; + // without it cargo falls back to the invoker's $HOME/.cargo. + cargo.env_remove("CARGO_HOME"); + // Root may have created target/ first (the coverage run does), so hand the + // build its own directories rather than asking the invoker to create them. + fs::create_dir_all(&target_dir).expect("creating the helper's target dir"); + for dir in [ + target_dir.parent().expect("target dir has a parent"), + &target_dir, + ] { + std::os::unix::fs::chown(dir, Some(uid), Some(gid)).expect("chowning target dir"); + } + cargo.uid(uid).gid(gid); + } + assert!( + cargo.status().expect("spawning cargo").success(), + "building the static insecure helper failed" + ); + target_dir.join(&target).join("debug/hyper-suidhelper") +} + +/// A tempdir every caller can traverse, holding a `root:` 4755 copy of the helper. +pub struct Install { + pub dir: TempDir, + pub bin: PathBuf, +} + +impl Install { + pub fn new() -> Self { + let dir = TempDir::new().unwrap(); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let bin = dir.path().join("hyper-suidhelper"); + fs::copy(static_insecure_helper(), &bin).unwrap(); + nix::unistd::chown( + &bin, + Some(Uid::from_raw(0)), + Some(Gid::from_raw(INSTALL_GROUP)), + ) + .unwrap(); + fs::set_permissions(&bin, fs::Permissions::from_mode(0o4755)).unwrap(); + Self { dir, bin } + } + + pub fn path(&self) -> &Path { + self.dir.path() + } + + /// A world-readable, root-owned config naming `work_dir` and a cgroup that does not exist. + pub fn config(&self, work_dir: &Path) -> PathBuf { + let path = self.path().join("config.toml"); + fs::write( + &path, + format!( + "work_dir = \"{}\"\n[jails]\ncgroup = \"hyper-e2e-does-not-exist\"\n", + work_dir.display() + ), + ) + .unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + path + } +} + +/// What one helper invocation left behind. +#[derive(Debug)] +pub struct Outcome { + pub code: Option, + pub stdout: String, + pub stderr: String, + /// `stat -c '%n %F %a %u %g'` of every path the script listed, as seen inside the namespace. + pub state: String, +} + +/// Run `bin` with `request` as `caller` (via `setpriv`, so the setuid bit does the elevating), +/// inside a private mount namespace whose `/run` is an empty tmpfs on which `setup` (a shell +/// snippet, run as root) stages whatever it likes first. `observe` lists paths to `stat` inside +/// the namespace after the helper exits. +pub fn run_in_fresh_run( + install: &Install, + config: &Path, + caller: Caller, + setup: &str, + request: &serde_json::Value, + observe: &[&str], +) -> Outcome { + let out = install.path().join(format!( + "out-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&out).unwrap(); + let groups = caller + .groups + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + let as_caller = if caller.uid == 0 { + String::new() + } else if groups.is_empty() { + format!( + "setpriv --reuid {} --regid {} --clear-groups", + caller.uid, caller.gid + ) + } else { + format!( + "setpriv --reuid {} --regid {} --groups {groups}", + caller.uid, caller.gid + ) + }; + let script = format!( + r#"set -e +mount -t tmpfs -o mode=0755 hyper-e2e-run /run +{setup} +set +e +{as_caller} env -i HYPER_SETUIDHELPER_IS_INSECURE_MODE=1 HYPER_SETUIDHELPER_CONFIG_PATH="$2" "$1" "$3" >"$4/stdout" 2>"$4/stderr" +echo $? >"$4/rc" +for p in {observe}; do stat -c '%n %F %a %u %g' "$p" 2>/dev/null; done >"$4/state" +exit 0 +"#, + observe = observe + .iter() + .map(|p| format!("'{p}'")) + .collect::>() + .join(" "), + ); + let status = Command::new("unshare") + .args(["--mount", "--propagation", "private", "--", "/bin/sh", "-c"]) + .arg(&script) + .arg("sh") + .arg(&install.bin) + .arg(config) + .arg(request.to_string()) + .arg(&out) + .env_clear() + .env("PATH", "/usr/sbin:/usr/bin:/sbin:/bin") + .status() + .unwrap(); + assert!(status.success(), "the namespace script itself failed"); + let read = |name: &str| fs::read_to_string(out.join(name)).unwrap_or_default(); + Outcome { + code: read("rc").trim().parse().ok(), + stdout: read("stdout"), + stderr: read("stderr"), + state: read("state"), + } +} + +/// Run `bin` with `request` as `caller` against the host's own `/run`. +pub fn run_as( + install: &Install, + config: &Path, + caller: Caller, + request: &serde_json::Value, +) -> Outcome { + let Caller { uid, gid, groups } = caller; + let mut cmd = Command::new(&install.bin); + cmd.arg(request.to_string()) + .env_clear() + .env("HYPER_SETUIDHELPER_IS_INSECURE_MODE", "1") + .env("HYPER_SETUIDHELPER_CONFIG_PATH", config); + // SAFETY: the hook runs in the forked child before exec and makes only raw id syscalls. + unsafe { + cmd.pre_exec(move || { + let groups: Vec = groups.iter().copied().map(Gid::from_raw).collect(); + nix::unistd::setgroups(&groups)?; + nix::unistd::setresgid(Gid::from_raw(gid), Gid::from_raw(gid), Gid::from_raw(gid))?; + nix::unistd::setresuid(Uid::from_raw(uid), Uid::from_raw(uid), Uid::from_raw(uid))?; + Ok(()) + }) + }; + let out = cmd.output().unwrap(); + Outcome { + code: out.status.code(), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + state: String::new(), + } +} diff --git a/native/suidhelper/tests/e2e/socket_pin.rs b/native/suidhelper/tests/e2e/socket_pin.rs new file mode 100644 index 00000000..f646268d --- /dev/null +++ b/native/suidhelper/tests/e2e/socket_pin.rs @@ -0,0 +1,373 @@ +//! The socket-pin contract of #135, observed through the binary: `vm_claim_*_socket` hard-links +//! the VM's own socket into `/socks/fc--`, and `vm_teardown` removes exactly +//! that VM's pins under the VM's lock. +//! +//! Laws: +//! - claim is idempotent: re-claiming leaves one pin, still the jail socket's inode, `0600` and +//! owned by the caller; +//! - claim never follows what already sits at the pin name: a planted symlink is replaced, its +//! target untouched; +//! - claim refuses anything at the jail socket name that is not a socket, and then pins nothing; +//! - teardown is scoped: tearing down VM `a` removes `a`'s two pins and jail and nothing else in +//! the socket dir, even for a VM `b` whose id has `a` as a dash-separated prefix; +//! - teardown is idempotent and leaves no lock file behind, and concurrent teardowns of one VM all +//! succeed (they serialize on the lock rather than time out or trip over each other). +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use pretty_assertions::assert_eq; +use proptest::prelude::*; +use rstest::rstest; +use serde_json::json; +use std::fs; +use std::os::unix::fs::{symlink, FileTypeExt, MetadataExt, PermissionsExt}; +use std::os::unix::net::UnixListener; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +mod support; +use support::{is_root, run, stdout_json}; + +const SOCKETS: [(&str, &str); 2] = [ + ("vm_claim_api_socket", "api.socket"), + ("vm_claim_vsock_socket", "vsock.sock"), +]; + +const LOCKS: &str = "/run/hyper-suidhelper/locks"; + +struct Work { + tmp: TempDir, + srv: PathBuf, + config: PathBuf, + listeners: Vec, +} + +impl Work { + fn new() -> Self { + let tmp = TempDir::new().unwrap(); + let srv = tmp.path().join("srv"); + fs::create_dir_all(srv.join("socks")).unwrap(); + let config = tmp.path().join("config.toml"); + fs::write( + &config, + format!( + "work_dir = \"{}\"\n[jails]\ncgroup = \"hyper-e2e-does-not-exist\"\n", + srv.display() + ), + ) + .unwrap(); + fs::set_permissions(&config, fs::Permissions::from_mode(0o644)).unwrap(); + Self { + tmp, + srv, + config, + listeners: Vec::new(), + } + } + + fn jail(&self, id: &str) -> PathBuf { + let root = self.srv.join("jails/firecracker").join(id).join("root"); + fs::create_dir_all(&root).unwrap(); + root + } + + fn socks(&self) -> PathBuf { + self.srv.join("socks") + } + + fn pin(&self, id: &str, socket: &str) -> PathBuf { + self.socks().join(format!("fc-{id}-{socket}")) + } + + fn boot(&mut self, id: &str) { + let jail = self.jail(id); + for (_, socket) in SOCKETS { + self.listeners + .push(UnixListener::bind(jail.join(socket)).unwrap()); + } + } + + fn claim_all(&self, id: &str) { + for (op, _) in SOCKETS { + let out = run(&self.config, json!({"op": op, "id": id})).success(); + assert_eq!(stdout_json(&out)["result"], "claimed", "{op} {id}"); + } + } + + fn teardown(&self, id: &str) { + let out = run(&self.config, json!({"op": "vm_teardown", "id": id})).success(); + assert_eq!(stdout_json(&out)["result"], "tore_down"); + } +} + +fn unique(tag: &str) -> String { + format!("{tag}{}", std::process::id()) +} + +fn ino(path: &Path) -> (u64, u64) { + let meta = fs::symlink_metadata(path).unwrap(); + (meta.dev(), meta.ino()) +} + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn reclaiming_keeps_one_private_pin_on_the_jail_socket_as_root(#[case] (op, socket): (&str, &str)) { + if !is_root() { + eprintln!("SKIP claim idempotence: needs root"); + return; + } + let mut work = Work::new(); + let id = unique("idem"); + work.boot(&id); + + for _ in 0..3 { + let out = run(&work.config, json!({"op": op, "id": id})).success(); + assert_eq!(stdout_json(&out)["result"], "claimed"); + } + + let pin = work.pin(&id, socket); + let meta = fs::symlink_metadata(&pin).unwrap(); + assert_eq!( + ( + ino(&pin), + meta.file_type().is_socket(), + meta.mode() & 0o7777, + meta.uid(), + meta.nlink(), + ), + ( + ino(&work.jail(&id).join(socket)), + true, + 0o600, + nix::unistd::getuid().as_raw(), + 2, + ) + ); + work.teardown(&id); +} + +#[rstest] +#[case::api(SOCKETS[0])] +#[case::vsock(SOCKETS[1])] +fn claim_replaces_a_planted_symlink_pin_without_following_it_as_root( + #[case] (op, socket): (&str, &str), +) { + if !is_root() { + eprintln!("SKIP claim symlink pin: needs root"); + return; + } + let mut work = Work::new(); + let id = unique("sym"); + work.boot(&id); + let victim = work.tmp.path().join("victim"); + fs::write(&victim, b"keep").unwrap(); + fs::set_permissions(&victim, fs::Permissions::from_mode(0o644)).unwrap(); + let victim_before = fs::symlink_metadata(&victim).unwrap(); + symlink(&victim, work.pin(&id, socket)).unwrap(); + + run(&work.config, json!({"op": op, "id": id})).success(); + + let victim_after = fs::symlink_metadata(&victim).unwrap(); + assert_eq!( + ( + fs::read(&victim).unwrap(), + victim_after.mode(), + victim_after.uid(), + victim_after.nlink(), + ino(&work.pin(&id, socket)), + ), + ( + b"keep".to_vec(), + victim_before.mode(), + victim_before.uid(), + 1, + ino(&work.jail(&id).join(socket)), + ) + ); + work.teardown(&id); +} + +#[derive(Debug, Clone, Copy)] +enum Planted { + Regular, + Directory, + Symlink, + Fifo, +} + +fn plant(at: &Path, what: Planted, decoy: &Path) { + match what { + Planted::Regular => fs::write(at, b"x").unwrap(), + Planted::Directory => fs::create_dir(at).unwrap(), + Planted::Symlink => symlink(decoy, at).unwrap(), + Planted::Fifo => { + nix::unistd::mkfifo(at, nix::sys::stat::Mode::from_bits_truncate(0o600)).unwrap() + } + } +} + +#[rstest] +#[case::api_regular(SOCKETS[0], Planted::Regular)] +#[case::api_directory(SOCKETS[0], Planted::Directory)] +#[case::api_symlink_to_a_live_socket(SOCKETS[0], Planted::Symlink)] +#[case::vsock_fifo(SOCKETS[1], Planted::Fifo)] +#[case::vsock_symlink_to_a_live_socket(SOCKETS[1], Planted::Symlink)] +fn claim_of_a_non_socket_is_refused_and_pins_nothing_as_root( + #[case] (op, socket): (&str, &str), + #[case] planted: Planted, +) { + if !is_root() { + eprintln!("SKIP claim non-socket: needs root"); + return; + } + let work = Work::new(); + let id = unique("bad"); + let decoy = work.tmp.path().join("decoy.sock"); + let _decoy = UnixListener::bind(&decoy).unwrap(); + plant(&work.jail(&id).join(socket), planted, &decoy); + + let out = run(&work.config, json!({"op": op, "id": id})).failure(); + + assert_eq!( + ( + String::from_utf8_lossy(&out.get_output().stdout).contains("claimed"), + work.pin(&id, socket).symlink_metadata().is_ok(), + ), + (false, false), + "{planted:?}" + ); + work.teardown(&id); +} + +fn id_pair() -> impl Strategy { + const ID: &str = "[a-zA-Z0-9][a-zA-Z0-9-]{0,12}"; + let suffix = prop_oneof![ + Just("api".to_owned()), + Just("vsock".to_owned()), + Just("api-socket".to_owned()), + "[a-z0-9]{1,6}", + ]; + (ID, suffix, ID, any::()) + .prop_map(|(a, suffix, other, prefixed)| { + let b = if prefixed { + format!("{a}-{suffix}") + } else { + other + }; + (a, b) + }) + .prop_filter("distinct ids", |(a, b)| a != b) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(10))] + + #[test] + fn teardown_removes_exactly_its_own_pins_and_jail_as_root((a, b) in id_pair()) { + if !is_root() { + eprintln!("SKIP teardown scoping: needs root"); + return Ok(()); + } + let tag = std::process::id(); + let (a, b) = (format!("{a}{tag}"), format!("{b}{tag}")); + prop_assume!(a != b); + let mut work = Work::new(); + work.boot(&a); + work.boot(&b); + work.claim_all(&a); + work.claim_all(&b); + let decoys = [ + format!("grpc-{a}.sock"), + format!("fc-{a}-api.socket.old"), + format!("fc-{a}-other.sock"), + format!("fc-{a}"), + ]; + for decoy in &decoys { + fs::write(work.socks().join(decoy), b"").unwrap(); + } + + work.teardown(&a); + + let mut left: Vec = fs::read_dir(work.socks()) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + let mut expected: Vec = decoys + .iter() + .cloned() + .chain(SOCKETS.iter().map(|(_, s)| format!("fc-{b}-{s}"))) + .collect(); + expected.sort(); + prop_assert_eq!(left, expected); + prop_assert!(!work.srv.join("jails/firecracker").join(&a).exists()); + prop_assert!(work.jail(&b).join("api.socket").exists()); + work.teardown(&b); + } +} + +#[rstest] +fn teardown_is_idempotent_and_leaves_no_lock_file_as_root() { + if !is_root() { + eprintln!("SKIP teardown idempotence: needs root"); + return; + } + let mut work = Work::new(); + let id = unique("twice"); + work.boot(&id); + work.claim_all(&id); + + work.teardown(&id); + work.teardown(&id); + fs::remove_dir(work.socks()).unwrap(); + work.teardown(&id); + + assert!( + !Path::new(LOCKS).join(format!("vm-{id}")).exists(), + "teardown must remove its lock file" + ); +} + +#[rstest] +fn concurrent_teardowns_of_one_vm_all_succeed_as_root() { + if !is_root() { + eprintln!("SKIP concurrent teardown: needs root"); + return; + } + let mut work = Work::new(); + let id = unique("race"); + work.boot(&id); + work.claim_all(&id); + let request = json!({"op": "vm_teardown", "id": id}).to_string(); + + let children: Vec<_> = (0..8) + .map(|_| { + support::command(&work.config, &[&request]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() + }) + .collect(); + let outcomes: Vec<(bool, String)> = children + .into_iter() + .map(|child| { + let out = child.wait_with_output().unwrap(); + ( + out.status.success() && String::from_utf8_lossy(&out.stdout).contains("tore_down"), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) + }) + .collect(); + + let failed: Vec<&String> = outcomes + .iter() + .filter(|(ok, _)| !ok) + .map(|(_, e)| e) + .collect(); + assert!(failed.is_empty(), "teardowns failed: {failed:?}"); + assert!(!work.pin(&id, "api.socket").exists()); + assert!(!work.srv.join("jails/firecracker").join(&id).exists()); + assert!(!Path::new(LOCKS).join(format!("vm-{id}")).exists()); +} diff --git a/native/suidhelper/tests/e2e/vm_net.rs b/native/suidhelper/tests/e2e/vm_net.rs new file mode 100644 index 00000000..f3a021f0 --- /dev/null +++ b/native/suidhelper/tests/e2e/vm_net.rs @@ -0,0 +1,549 @@ +//! The network `vm_launch` builds and `vm_teardown` removes, observed from a scratch host +//! namespace with `ip` and `nft` rather than the helper's own rtnetlink code. +//! +//! Laws: +//! - teardown removes everything a launch added to the host: the `vm-veth-`, the +//! guest's `/128` route, its proxy-NDP entry and the VM's namespace; a second teardown +//! changes nothing; +//! - tearing one VM down leaves every other VM's network as it was; +//! - a relaunch replaces the VM's previous network, whatever NetId it held; +//! - a launch that fails while networking (a NetId another VM holds, a missing uplink) +//! leaves no network of its own behind and never damages the VM holding the NetId; +//! - teardown without `[net]` still unlinks the VM, and the next teardown with `[net]` +//! sweeps the proxy entry it could not find; +//! - helpers launching and tearing down distinct VMs at once end in exactly the network +//! of the VMs left running; +//! - `vm_firewall` replaces the port policy whole and leaves the namespace's own table be; +//! - a policy at the per-direction cap, on the edge ports, is one `nft` accepts. +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use pretty_assertions::assert_eq; +use rstest::rstest; +use serde_json::{json, Value}; +use std::collections::BTreeSet; +use std::fs; +use std::net::Ipv6Addr; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicU32, Ordering}; + +mod support; +use hyperrs_linux::net::netns::NetNs; +use support::{is_root, MapperDevice, ScratchNs}; + +const RECORDER_EXIT: i32 = 7; +const REFUSED_EXIT: i32 = 2; +const NODE_PREFIX: &str = "2001:db8::/64"; + +static HOSTS: AtomicU32 = AtomicU32::new(0); + +/// The cookbook's VM address for `net_id`, `:1:0:0:`. +fn guest(net_id: u16) -> Ipv6Addr { + Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 1, 0, 0, net_id) +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct HostNet { + veths: BTreeSet, + guest_routes: BTreeSet, + proxies: BTreeSet, +} + +impl HostNet { + fn of(vms: &[u16]) -> Self { + Self { + veths: vms.iter().map(|id| format!("vm-veth-{id}")).collect(), + guest_routes: vms.iter().copied().map(guest).collect(), + proxies: vms.iter().copied().map(guest).collect(), + } + } +} + +struct Host { + ns: ScratchNs, + tmp: TempDir, + work: PathBuf, + firecracker: PathBuf, + jailer: PathBuf, + kernel: PathBuf, + rootfs: MapperDevice, + tag: u32, +} + +impl Drop for Host { + fn drop(&mut self) { + let Ok(entries) = fs::read_dir("/run/netns") else { + return; + }; + let mine = format!("hyper-vm{}x{}", std::process::id(), self.tag); + for entry in entries.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with(&mine) { + let _ = NetNs::delete(name.as_str().try_into().unwrap()); + } + } + } +} + +impl Host { + fn new() -> Option { + if !is_root() { + return None; + } + let tmp = TempDir::new().unwrap(); + let base = fs::canonicalize(tmp.path()).unwrap(); + let work = base.join("srv"); + fs::create_dir_all(work.join("jails")).unwrap(); + let firecracker = base.join("firecracker"); + fs::write(&firecracker, "#!/bin/true\n").unwrap(); + let jailer = base.join("jailer"); + fs::copy(env!("CARGO_BIN_EXE_fake-jailer"), &jailer).unwrap(); + fs::set_permissions(&jailer, fs::Permissions::from_mode(0o755)).unwrap(); + let kernel = work.join("vmlinux-src"); + fs::write(&kernel, "kernel image").unwrap(); + let backing = base.join("rootfs.img"); + fs::write(&backing, vec![0u8; 1024 * 1024]).unwrap(); + let tag = HOSTS.fetch_add(1, Ordering::Relaxed); + let rootfs = MapperDevice::linear( + &format!("hyper-e2e-vmnet-{}-{tag}", std::process::id()), + &backing, + )?; + Some(Self { + ns: ScratchNs::new(&format!("vmnet{tag}")), + tmp, + work, + firecracker, + jailer, + kernel, + rootfs, + tag, + }) + } + + fn id(&self, vm: &str) -> String { + format!("vm{}x{}{vm}", std::process::id(), self.tag) + } + + fn config(&self, net: Option<&str>) -> PathBuf { + let p = self + .tmp + .path() + .join(format!("config-{}.toml", net.unwrap_or("none"))); + let net = net.map_or(String::new(), |uplink| { + format!("[net]\nuplink = \"{uplink}\"\nipv6_prefix = \"{NODE_PREFIX}\"\n") + }); + let body = format!( + "work_dir = \"{}\"\n[tools]\njailer = \"{}\"\nfirecracker = \"{}\"\n{net}", + self.work.display(), + self.jailer.display(), + self.firecracker.display(), + ); + fs::write(&p, body).unwrap(); + fs::set_permissions(&p, fs::Permissions::from_mode(0o644)).unwrap(); + p + } + + fn launch(&self, cfg: &Path, vm: &str, net_id: u16, firewall: Value) -> i32 { + let request = json!({ + "op": "vm_launch", + "id": self.id(vm), + "uid": 900_001, + "gid": 900_002, + "kernel": self.kernel, + "device": self.rootfs.path(), + "limits": {}, + "net_id": net_id, + "firewall": firewall, + }); + self.ns + .run(cfg, request) + .get_output() + .status + .code() + .unwrap() + } + + /// Run every request at once, each in its own helper, and return their exit codes. + fn concurrently(&self, cfg: &Path, requests: &[Value]) -> Vec { + let children: Vec<_> = requests + .iter() + .map(|request| { + self.ns + .get() + .scoped(|| { + support::command(cfg, &[&request.to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() + }) + .unwrap() + }) + .collect(); + children + .into_iter() + .map(|child| { + let out = child.wait_with_output().unwrap(); + out.status.code().unwrap() + }) + .collect() + } + + fn launch_request(&self, vm: &str, net_id: u16) -> Value { + json!({ + "op": "vm_launch", + "id": self.id(vm), + "uid": 900_001, + "gid": 900_002, + "kernel": self.kernel, + "device": self.rootfs.path(), + "limits": {}, + "net_id": net_id, + "firewall": open(), + }) + } + + fn teardown(&self, cfg: &Path, vm: &str) { + self.ns + .run(cfg, json!({"op": "vm_teardown", "id": self.id(vm)})) + .success(); + } + + fn has_netns(&self, vm: &str) -> bool { + Path::new("/run/netns") + .join(format!("hyper-{}", self.id(vm))) + .exists() + } + + fn ip(&self, args: &[&str]) -> Value { + let out = self + .ns + .get() + .scoped(|| Command::new("ip").args(args).output().unwrap()) + .unwrap(); + assert!(out.status.success(), "ip {args:?}: {out:?}"); + let stdout = String::from_utf8(out.stdout).unwrap(); + if stdout.trim().is_empty() { + return json!([]); + } + serde_json::from_str(&stdout).unwrap() + } + + fn observe(&self) -> HostNet { + let in_vm_range = |addr: Ipv6Addr| u128::from(addr) >> 48 == u128::from(guest(0)) >> 48; + let addrs = |listing: Value, key: &str| -> BTreeSet { + listing + .as_array() + .unwrap() + .iter() + .filter_map(|entry| entry[key].as_str()) + .filter_map(|dst| dst.split('/').next().unwrap().parse().ok()) + .filter(|addr| in_vm_range(*addr)) + .collect() + }; + HostNet { + veths: self + .ip(&["-j", "link", "show"]) + .as_array() + .unwrap() + .iter() + .filter_map(|link| link["ifname"].as_str()) + .filter(|name| name.starts_with("vm-veth-")) + .map(str::to_owned) + .collect(), + guest_routes: addrs(self.ip(&["-j", "-6", "route", "show"]), "dst"), + proxies: addrs(self.ip(&["-j", "-6", "neigh", "show", "proxy"]), "dst"), + } + } + + fn nft_rules(&self, vm: &str, table: &str) -> Option> { + let ns = NetNs::open( + format!("hyper-{}", self.id(vm)) + .as_str() + .try_into() + .unwrap(), + ) + .ok()?; + let out = ns + .scoped(|| { + Command::new("nft") + .args(["-j", "list", "table", "inet", table]) + .output() + .unwrap() + }) + .unwrap(); + if !out.status.success() { + return None; + } + let listing: Value = serde_json::from_slice(&out.stdout).unwrap(); + Some( + listing["nftables"] + .as_array() + .unwrap() + .iter() + .filter_map(|item| item.get("rule")) + .map(|rule| rule["expr"].clone()) + .collect(), + ) + } + + /// Every destination-port match in `table`, as `(protocol, first, last)`. + fn dports(&self, vm: &str, table: &str) -> Vec<(String, u64, u64)> { + self.nft_rules(vm, table) + .expect("the table is listed") + .iter() + .flat_map(|expr| expr.as_array().unwrap().clone()) + .filter_map(|statement| { + let m = statement.get("match")?; + let payload = m["left"].get("payload")?; + (payload["field"] == "dport").then(|| { + let protocol = payload["protocol"].as_str().unwrap().to_owned(); + match &m["right"] { + Value::Number(port) => { + (protocol, port.as_u64().unwrap(), port.as_u64().unwrap()) + } + range => ( + protocol, + range["range"][0].as_u64().unwrap(), + range["range"][1].as_u64().unwrap(), + ), + } + }) + }) + .collect() + } +} + +fn open() -> Value { + json!({"inbound": [], "outbound_blocked": []}) +} + +macro_rules! require { + () => { + match Host::new() { + Some(host) => host, + None => { + eprintln!("SKIP: needs root and losetup/dmsetup"); + return; + } + } + }; +} + +#[rstest] +#[case::first(0)] +#[case::past_one_octet(64)] +#[case::last(16_383)] +fn teardown_leaves_no_trace_of_the_vm_network_and_is_idempotent_as_root(#[case] net_id: u16) { + let host = require!(); + let cfg = host.config(Some("lo")); + + assert_eq!(host.launch(&cfg, "a", net_id, open()), RECORDER_EXIT); + let launched = (host.observe(), host.has_netns("a")); + host.teardown(&cfg, "a"); + let torn_down = (host.observe(), host.has_netns("a")); + host.teardown(&cfg, "a"); + + assert_eq!( + (launched, torn_down, (host.observe(), host.has_netns("a"))), + ( + (HostNet::of(&[net_id]), true), + (HostNet::default(), false), + (HostNet::default(), false), + ) + ); +} + +#[rstest] +fn tearing_one_vm_down_leaves_its_neighbours_network_alone_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + + for (vm, net_id) in [("a", 1), ("b", 2), ("c", 3)] { + assert_eq!(host.launch(&cfg, vm, net_id, open()), RECORDER_EXIT); + } + host.teardown(&cfg, "b"); + + assert_eq!( + (host.observe(), ["a", "b", "c"].map(|vm| host.has_netns(vm))), + (HostNet::of(&[1, 3]), [true, false, true]) + ); +} + +#[rstest] +fn a_relaunch_onto_another_net_id_replaces_the_old_network_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + + assert_eq!(host.launch(&cfg, "a", 1, open()), RECORDER_EXIT); + assert_eq!(host.launch(&cfg, "a", 2, open()), RECORDER_EXIT); + + assert_eq!(host.observe(), HostNet::of(&[2])); +} + +#[rstest] +fn a_launch_onto_a_held_net_id_fails_and_leaves_the_holder_intact_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + + assert_eq!(host.launch(&cfg, "a", 5, open()), RECORDER_EXIT); + let refused = host.launch(&cfg, "b", 5, open()); + + assert_eq!( + ( + refused, + host.observe(), + host.has_netns("a"), + host.has_netns("b") + ), + (REFUSED_EXIT, HostNet::of(&[5]), true, false) + ); +} + +#[rstest] +fn a_launch_whose_uplink_is_missing_leaves_no_network_behind_as_root() { + let host = require!(); + let cfg = host.config(Some("nope0")); + + let refused = host.launch(&cfg, "a", 4, open()); + + assert_eq!( + (refused, host.observe(), host.has_netns("a")), + (REFUSED_EXIT, HostNet::default(), false) + ); +} + +#[rstest] +fn a_teardown_without_net_config_unlinks_and_the_next_one_sweeps_its_proxy_as_root() { + let host = require!(); + let (with_net, without_net) = (host.config(Some("lo")), host.config(None)); + + assert_eq!(host.launch(&with_net, "a", 6, open()), RECORDER_EXIT); + assert_eq!(host.launch(&with_net, "b", 7, open()), RECORDER_EXIT); + host.teardown(&without_net, "a"); + let unswept = host.observe(); + host.teardown(&with_net, "never-launched"); + + assert_eq!( + (unswept, host.observe(), host.has_netns("a")), + ( + HostNet { + proxies: [guest(6), guest(7)].into(), + ..HostNet::of(&[7]) + }, + HostNet::of(&[7]), + false + ) + ); +} + +#[rstest] +fn setting_the_firewall_replaces_the_policy_whole_and_keeps_the_namespace_table_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + let launched_with = json!({ + "inbound": [{"protocol": "tcp", "ports": [22, 22]}], + "outbound_blocked": [{"protocol": "tcp", "ports": [25, 25]}], + }); + assert_eq!(host.launch(&cfg, "a", 8, launched_with), RECORDER_EXIT); + let namespace_table = host.nft_rules("a", "hyper"); + + host.ns + .run( + &cfg, + json!({"op": "vm_firewall", "id": host.id("a"), "firewall": { + "inbound": [{"protocol": "udp", "ports": [6000, 6010]}], + "outbound_blocked": [], + }}), + ) + .success(); + + assert_eq!( + ( + host.dports("a", "hyper-firewall"), + host.nft_rules("a", "hyper") == namespace_table, + ), + (vec![("udp".to_owned(), 6000, 6010)], true) + ); +} + +#[rstest] +fn a_policy_at_the_rule_cap_on_the_edge_ports_is_installed_whole_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + let edges = [[1, 1], [65_535, 65_535], [1, 65_535]]; + let rules = |protocol: &str| -> Vec<(String, u64, u64)> { + edges + .iter() + .map(|[first, last]| (*first, *last)) + .chain((0..125).map(|i| (1000 + 2 * i, 1001 + 2 * i))) + .map(|(first, last)| (protocol.to_owned(), first, last)) + .collect() + }; + let request = |rules: &[(String, u64, u64)]| -> Vec { + rules + .iter() + .map(|(protocol, first, last)| json!({"protocol": protocol, "ports": [first, last]})) + .collect() + }; + let (inbound, outbound) = (rules("tcp"), rules("udp")); + + let launched = host.launch( + &cfg, + "a", + 9, + json!({"inbound": request(&inbound), "outbound_blocked": request(&outbound)}), + ); + + assert_eq!( + (launched, host.dports("a", "hyper-firewall")), + (RECORDER_EXIT, [inbound, outbound].concat()) + ); +} + +#[rstest] +fn concurrent_launches_and_teardowns_end_in_exactly_the_running_vms_network_as_root() { + let host = require!(); + let cfg = host.config(Some("lo")); + let first: Vec<(String, u16)> = (0..6).map(|i| (format!("a{i}"), 10 + i)).collect(); + let second: Vec<(String, u16)> = (0..6).map(|i| (format!("b{i}"), 20 + i)).collect(); + + let launched = host.concurrently( + &cfg, + &first + .iter() + .map(|(vm, net_id)| host.launch_request(vm, *net_id)) + .collect::>(), + ); + let mixed = host.concurrently( + &cfg, + &first + .iter() + .step_by(2) + .map(|(vm, _)| json!({"op": "vm_teardown", "id": host.id(vm)})) + .chain( + second + .iter() + .map(|(vm, net_id)| host.launch_request(vm, *net_id)), + ) + .collect::>(), + ); + let running: Vec = first + .iter() + .skip(1) + .step_by(2) + .chain(&second) + .map(|(_, net_id)| *net_id) + .collect(); + + assert_eq!( + (launched, mixed, host.observe()), + ( + vec![RECORDER_EXIT; 6], + [vec![0; 3], vec![RECORDER_EXIT; 6]].concat(), + HostNet::of(&running) + ) + ); +} diff --git a/native/suidhelper/tests/e2e/vm_net_traffic.rs b/native/suidhelper/tests/e2e/vm_net_traffic.rs new file mode 100644 index 00000000..e5e89790 --- /dev/null +++ b/native/suidhelper/tests/e2e/vm_net_traffic.rs @@ -0,0 +1,504 @@ +//! Real connections across a whole node: `host_init`'s table on the host and `vm_launch`'s +//! namespaces, routes and tables for two VMs, with an internet (`wan`) beyond the uplink. +//! +//! Firecracker's `tap0` is swapped for a veth of the same name whose peer is the guest's +//! `eth0`, holding what `vm_launch` gave the tap, so every rule that matches on `tap0` +//! sees guest traffic. Laws, per the networking cookbook: +//! - the guest reaches the internet over IPv4 (double NAT) and IPv6 (routed /128); +//! - inbound, exactly the ports in the VM's open ranges reach it, both ends inclusive, for +//! TCP and UDP alike; +//! - outbound, exactly the ports in its blocked ranges never leave it; +//! - VMs reach each other over their public IPv6 addresses only on the ports the +//! receiving VM opened; +//! - DNS to the gateway reaches the resolver, and nothing else on the gateway answers; +//! - `vm_firewall` takes effect on the next connection. +#![cfg(feature = "insecure_mode")] + +use assert_fs::TempDir; +use hyperrs_linux::net::link::DuplicateAddrDetection; +use hyperrs_linux::net::netns::NetNs; +use hyperrs_linux::net::route::Route; +use hyperrs_linux::net::{IfIndex, IfName, IpNet, Rtnl}; +use pretty_assertions::assert_eq; +use rstest::rstest; +use serde_json::{json, Value}; +use std::fs; +use std::net::{Ipv6Addr, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::thread; +use std::time::Duration; + +mod support; +use socket2::{Domain, Protocol, Socket, Type}; +use support::{is_root, MapperDevice, ScratchNs}; + +const RECORDER_EXIT: i32 = 7; +const DROPPED_AFTER: Duration = Duration::from_secs(2); +const GUEST_TCP: [u16; 4] = [6999, 7000, 7002, 7003]; +const GUEST_UDP: [u16; 2] = [7100, 7101]; +const WAN_TCP: [u16; 5] = [53, 7199, 7200, 7201, 7202]; +/// Bound to one address each, so a reply leaves from the address the request reached. +const WAN_UDP: [&str; 3] = ["1.1.1.1:53", "198.51.100.1:7202", "[2001:db8:ffff::1]:7202"]; + +static LABS: AtomicU32 = AtomicU32::new(0); + +#[derive(Debug, Clone, Copy)] +enum Vm { + A, + B, +} + +impl Vm { + fn net_id(self) -> u16 { + match self { + Self::A => 1, + Self::B => 2, + } + } + + fn guest_v6(self) -> Ipv6Addr { + Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 1, 0, 0, self.net_id()) + } + + fn firewall(self) -> Value { + match self { + Self::A => json!({ + "inbound": [ + {"protocol": "tcp", "ports": [7000, 7002]}, + {"protocol": "udp", "ports": [7100, 7100]}, + ], + "outbound_blocked": [{"protocol": "tcp", "ports": [7200, 7201]}], + }), + Self::B => json!({ + "inbound": [{"protocol": "tcp", "ports": [7000, 7000]}], + "outbound_blocked": [], + }), + } + } +} + +/// How the internet's router reaches the VM range. +#[derive(Debug, Clone, Copy)] +enum Upstream { + /// Routed to the host's link-local address. + Routed, + /// On-link, so the router solicits each guest address and the host must answer by proxy. + OnLink, +} + +#[derive(Debug, Clone, Copy)] +enum Src { + Guest(Vm), + Wan, +} + +#[derive(Debug, Clone, Copy)] +enum Proto { + Tcp, + Udp, + /// An ICMP echo; the port is ignored. + Ping, +} + +struct Lab { + host: ScratchNs, + wan: ScratchNs, + guests: [ScratchNs; 2], + cfg: PathBuf, + tag: u32, + _tmp: TempDir, + _rootfs: MapperDevice, + _listeners: Vec, +} + +impl Drop for Lab { + fn drop(&mut self) { + for vm in [Vm::A, Vm::B] { + let _ = NetNs::delete(self.netns(vm).as_str().try_into().unwrap()); + } + } +} + +impl Lab { + fn new(upstream: Upstream) -> Option { + if !is_root() { + return None; + } + let tag = LABS.fetch_add(1, Ordering::Relaxed); + let tmp = TempDir::new().unwrap(); + let base = fs::canonicalize(tmp.path()).unwrap(); + let work = base.join("srv"); + fs::create_dir_all(work.join("jails")).unwrap(); + let firecracker = base.join("firecracker"); + fs::write(&firecracker, "#!/bin/true\n").unwrap(); + let jailer = base.join("jailer"); + fs::copy(env!("CARGO_BIN_EXE_fake-jailer"), &jailer).unwrap(); + fs::set_permissions(&jailer, fs::Permissions::from_mode(0o755)).unwrap(); + fs::write(work.join("vmlinux"), "kernel image").unwrap(); + let backing = base.join("rootfs.img"); + fs::write(&backing, vec![0u8; 1024 * 1024]).unwrap(); + let rootfs = MapperDevice::linear( + &format!("hyper-e2e-traffic-{}-{tag}", std::process::id()), + &backing, + )?; + let cfg = base.join("config.toml"); + fs::write( + &cfg, + format!( + "work_dir = \"{}\"\n[tools]\njailer = \"{}\"\nfirecracker = \"{}\"\n\ + [net]\nuplink = \"if0\"\nipv6_prefix = \"2001:db8::/64\"\n", + work.display(), + jailer.display(), + firecracker.display() + ), + ) + .unwrap(); + fs::set_permissions(&cfg, fs::Permissions::from_mode(0o644)).unwrap(); + + let mut lab = Self { + host: ScratchNs::new(&format!("traffic{tag}-host")), + wan: ScratchNs::new(&format!("traffic{tag}-wan")), + guests: [ + ScratchNs::new(&format!("traffic{tag}-ga")), + ScratchNs::new(&format!("traffic{tag}-gb")), + ], + cfg, + tag, + _tmp: tmp, + _rootfs: rootfs, + _listeners: Vec::new(), + }; + lab.wire_the_internet(upstream); + lab.host.run(&lab.cfg, json!({"op": "host_init"})).success(); + for vm in [Vm::A, Vm::B] { + let launched = lab.host.run( + &lab.cfg, + json!({ + "op": "vm_launch", + "id": lab.id(vm), + "uid": 900_001, + "gid": 900_002, + "kernel": work.join("vmlinux"), + "device": lab._rootfs.path(), + "limits": {}, + "net_id": vm.net_id(), + "firewall": vm.firewall(), + }), + ); + assert_eq!(launched.get_output().status.code(), Some(RECORDER_EXIT)); + lab.plug_in_the_guest(vm); + } + + let tcp = |ns: &ScratchNs, port| { + ns.get() + .scoped(|| TcpListener::bind((Ipv6Addr::UNSPECIFIED, port))) + .unwrap() + .unwrap() + }; + let echo = |ns: &ScratchNs, at: SocketAddr| { + let socket = ns.get().scoped(|| UdpSocket::bind(at)).unwrap().unwrap(); + thread::spawn(move || { + let mut buf = [0u8; 64]; + while let Ok((n, from)) = socket.recv_from(&mut buf) { + let _ = socket.send_to(&buf[..n], from); + } + }); + }; + let mut listeners: Vec = WAN_TCP.iter().map(|p| tcp(&lab.wan, *p)).collect(); + WAN_UDP + .iter() + .for_each(|at| echo(&lab.wan, at.parse().unwrap())); + for (guest, vm) in lab.guests.iter().zip([Vm::A, Vm::B]) { + listeners.extend(GUEST_TCP.iter().map(|p| tcp(guest, *p))); + GUEST_UDP + .iter() + .for_each(|p| echo(guest, guest_port(vm, *p))); + } + lab._listeners = listeners; + Some(lab) + } + + fn id(&self, vm: Vm) -> String { + format!("vm{}t{}{vm:?}", std::process::id(), self.tag).to_lowercase() + } + + fn netns(&self, vm: Vm) -> String { + format!("hyper-{}", self.id(vm)) + } + + fn guest(&self, vm: Vm) -> &NetNs { + match vm { + Vm::A => self.guests[0].get(), + Vm::B => self.guests[1].get(), + } + } + + fn wire_the_internet(&self, upstream: Upstream) { + let host = self.host.get().rtnl().unwrap(); + let wan = self.wan.get().rtnl().unwrap(); + host.create_veth(&name("if0"), &name("wan"), self.wan.get()) + .unwrap(); + up(&host, "lo", &[]); + let if0 = up(&host, "if0", &["203.0.113.2/24", "fe80::a/64"]); + up( + &wan, + "lo", + &["198.51.100.1/32", "1.1.1.1/32", "2001:db8:ffff::1/128"], + ); + let wan_link = up(&wan, "wan", &["203.0.113.1/24", "fe80::b/64"]); + let vm_range = match upstream { + Upstream::Routed => route("2001:db8:0:0:1::/80", Some("fe80::a"), wan_link), + Upstream::OnLink => route("2001:db8::/64", None, wan_link), + }; + for (rtnl, route) in [ + (&host, route("0.0.0.0/0", Some("203.0.113.1"), if0)), + (&host, route("::/0", Some("fe80::b"), if0)), + (&wan, vm_range), + ] { + rtnl.add_route(&route).unwrap(); + } + self.host.get().enable_forwarding().unwrap(); + } + + fn plug_in_the_guest(&self, vm: Vm) { + let ns = NetNs::open(self.netns(vm).as_str().try_into().unwrap()).unwrap(); + let inner = ns.rtnl().unwrap(); + inner + .link(&name("tap0")) + .unwrap() + .unwrap() + .delete() + .unwrap(); + inner + .create_veth(&name("tap0"), &name("eth0"), self.guest(vm)) + .unwrap(); + let tap = up(&inner, "tap0", &["172.16.0.1/30", "fe80::1/64"]); + inner + .add_route(&route(&format!("{}/128", vm.guest_v6()), None, tap)) + .unwrap(); + + let guest = self.guest(vm).rtnl().unwrap(); + up(&guest, "lo", &[]); + let eth0 = up( + &guest, + "eth0", + &[ + "172.16.0.2/30", + &format!("{}/128", vm.guest_v6()), + "fe80::2/64", + ], + ); + for (dst, via) in [("0.0.0.0/0", "172.16.0.1"), ("::/0", "fe80::1")] { + guest.add_route(&route(dst, Some(via), eth0)).unwrap(); + } + } + + fn set_firewall(&self, vm: Vm, firewall: Value) { + self.host + .run( + &self.cfg, + json!({"op": "vm_firewall", "id": self.id(vm), "firewall": firewall}), + ) + .success(); + } + + fn reaches(&self, from: Src, proto: Proto, to: SocketAddr) -> bool { + let ns = match from { + Src::Guest(vm) => self.guest(vm), + Src::Wan => self.wan.get(), + }; + match proto { + Proto::Tcp => ns + .scoped(|| TcpStream::connect_timeout(&to, DROPPED_AFTER).is_ok()) + .unwrap(), + Proto::Udp => { + let local: SocketAddr = match to { + SocketAddr::V4(_) => "0.0.0.0:0".parse().unwrap(), + SocketAddr::V6(_) => "[::]:0".parse().unwrap(), + }; + let socket = ns.scoped(|| UdpSocket::bind(local)).unwrap().unwrap(); + socket.set_read_timeout(Some(DROPPED_AFTER)).unwrap(); + socket.send_to(b"ping", to).unwrap(); + let mut buf = [0u8; 64]; + socket + .recv_from(&mut buf) + .is_ok_and(|(n, _)| &buf[..n] == b"ping") + } + Proto::Ping => ns.scoped(|| pings(to)).unwrap(), + } + } +} + +/// Whether an echo request to `to` gets its reply back. +fn pings(to: SocketAddr) -> bool { + const ID: u16 = 0x4859; + let (domain, protocol, request, reply) = match to { + SocketAddr::V4(_) => (Domain::IPV4, Protocol::ICMPV4, 8, 0), + SocketAddr::V6(_) => (Domain::IPV6, Protocol::ICMPV6, 128, 129), + }; + let socket = Socket::new(domain, Type::RAW, Some(protocol)).unwrap(); + socket.set_read_timeout(Some(DROPPED_AFTER)).unwrap(); + let mut echo = [request, 0, 0, 0, (ID >> 8) as u8, ID as u8, 0, 1]; + if matches!(to, SocketAddr::V4(_)) { + let sum = echo + .chunks(2) + .map(|w| u32::from(u16::from_be_bytes([w[0], w[1]]))) + .sum::(); + echo[2..4].copy_from_slice(&(!(((sum & 0xffff) + (sum >> 16)) as u16)).to_be_bytes()); + } + let target = SocketAddr::new(to.ip(), 0); + socket.send_to(&echo, &target.into()).unwrap(); + let mut packet = [0u8; 1500]; + while let Ok(n) = std::io::Read::read(&mut &socket, &mut packet) { + let icmp = match to { + SocketAddr::V4(_) => &packet[usize::from(packet[0] & 0x0f) * 4..n], + SocketAddr::V6(_) => &packet[..n], + }; + if icmp.len() >= 8 && icmp[0] == reply && u16::from_be_bytes([icmp[4], icmp[5]]) == ID { + return true; + } + } + false +} + +fn name(s: &str) -> IfName { + s.parse().unwrap() +} + +fn up(rtnl: &Rtnl, link: &str, addrs: &[&str]) -> IfIndex { + let link = rtnl.link(&name(link)).unwrap().unwrap(); + link.disable_addr_gen().unwrap(); + for addr in addrs { + link.add_addr(addr.parse().unwrap(), DuplicateAddrDetection::Disabled) + .unwrap(); + } + link.set_up().unwrap(); + link.index() +} + +fn route(dst: &str, via: Option<&str>, dev: IfIndex) -> Route { + match dst.parse::().unwrap() { + IpNet::V4(dst) => Route::V4 { + dst, + via: via.map(|via| via.parse().unwrap()), + dev, + }, + IpNet::V6(dst) => Route::V6 { + dst, + via: via.map(|via| via.parse().unwrap()), + dev, + }, + } +} + +fn guest_port(vm: Vm, port: u16) -> SocketAddr { + SocketAddr::from((vm.guest_v6(), port)) +} + +macro_rules! require { + ($upstream:expr) => { + match Lab::new($upstream) { + Some(lab) => lab, + None => { + eprintln!("SKIP: needs root and losetup/dmsetup"); + return; + } + } + }; +} + +#[rstest] +#[case::guest_to_the_internet_over_ipv4(Src::Guest(Vm::A), Proto::Tcp, "198.51.100.1:7202", true)] +#[case::guest_to_the_internet_over_ipv6( + Src::Guest(Vm::A), + Proto::Tcp, + "[2001:db8:ffff::1]:7202", + true +)] +#[case::below_the_blocked_range(Src::Guest(Vm::A), Proto::Tcp, "198.51.100.1:7199", true)] +#[case::first_blocked_port(Src::Guest(Vm::A), Proto::Tcp, "198.51.100.1:7200", false)] +#[case::last_blocked_port(Src::Guest(Vm::A), Proto::Tcp, "198.51.100.1:7201", false)] +#[case::blocked_over_ipv6(Src::Guest(Vm::A), Proto::Tcp, "[2001:db8:ffff::1]:7200", false)] +#[case::another_vm_blocks_nothing(Src::Guest(Vm::B), Proto::Tcp, "198.51.100.1:7200", true)] +#[case::below_the_open_range(Src::Wan, Proto::Tcp, "[2001:db8::1:0:0:1]:6999", false)] +#[case::first_open_port(Src::Wan, Proto::Tcp, "[2001:db8::1:0:0:1]:7000", true)] +#[case::last_open_port(Src::Wan, Proto::Tcp, "[2001:db8::1:0:0:1]:7002", true)] +#[case::above_the_open_range(Src::Wan, Proto::Tcp, "[2001:db8::1:0:0:1]:7003", false)] +#[case::open_udp_port(Src::Wan, Proto::Udp, "[2001:db8::1:0:0:1]:7100", true)] +#[case::closed_udp_port(Src::Wan, Proto::Udp, "[2001:db8::1:0:0:1]:7101", false)] +#[case::tcp_rule_opens_no_udp(Src::Wan, Proto::Udp, "[2001:db8::1:0:0:2]:7100", false)] +#[case::vm_to_vm_on_an_open_port(Src::Guest(Vm::A), Proto::Tcp, "[2001:db8::1:0:0:2]:7000", true)] +#[case::vm_to_vm_on_a_closed_port(Src::Guest(Vm::A), Proto::Tcp, "[2001:db8::1:0:0:2]:7003", false)] +#[case::vm_to_vm_over_the_transit_range(Src::Guest(Vm::A), Proto::Tcp, "10.200.0.10:7000", false)] +#[case::udp_to_the_internet_over_ipv4(Src::Guest(Vm::A), Proto::Udp, "198.51.100.1:7202", true)] +#[case::udp_to_the_internet_over_ipv6( + Src::Guest(Vm::A), + Proto::Udp, + "[2001:db8:ffff::1]:7202", + true +)] +#[case::dns_over_udp_reaches_the_resolver(Src::Guest(Vm::A), Proto::Udp, "172.16.0.1:53", true)] +#[case::ping_to_the_internet_over_ipv4(Src::Guest(Vm::A), Proto::Ping, "198.51.100.1:0", true)] +#[case::ping_to_the_internet_over_ipv6( + Src::Guest(Vm::A), + Proto::Ping, + "[2001:db8:ffff::1]:0", + true +)] +#[case::inbound_ping_is_not_an_open_port(Src::Wan, Proto::Ping, "[2001:db8::1:0:0:1]:0", false)] +#[case::the_gateway_does_not_answer_ping(Src::Guest(Vm::A), Proto::Ping, "172.16.0.1:0", false)] +#[case::dns_over_tcp_reaches_the_resolver(Src::Guest(Vm::A), Proto::Tcp, "172.16.0.1:53", true)] +#[case::other_gateway_ports_are_silent(Src::Guest(Vm::A), Proto::Tcp, "172.16.0.1:7202", false)] +#[case::guest_to_the_host_uplink(Src::Guest(Vm::A), Proto::Tcp, "203.0.113.2:7202", false)] +fn a_launched_node_carries_exactly_what_the_cookbook_allows_as_root( + #[case] from: Src, + #[case] proto: Proto, + #[case] to: &str, + #[case] reachable: bool, +) { + let lab = require!(Upstream::Routed); + + assert_eq!(lab.reaches(from, proto, to.parse().unwrap()), reachable); +} + +#[rstest] +fn a_new_firewall_governs_the_next_connection_as_root() { + let lab = require!(Upstream::Routed); + let before = + [7000, 7003].map(|port| lab.reaches(Src::Wan, Proto::Tcp, guest_port(Vm::A, port))); + + lab.set_firewall( + Vm::A, + json!({"inbound": [{"protocol": "tcp", "ports": [7003, 7003]}], "outbound_blocked": []}), + ); + + assert_eq!( + ( + before, + [7000, 7003].map(|port| lab.reaches(Src::Wan, Proto::Tcp, guest_port(Vm::A, port))), + lab.reaches( + Src::Guest(Vm::A), + Proto::Tcp, + "198.51.100.1:7200".parse().unwrap() + ), + ), + ([true, false], [false, true], true) + ); +} + +#[rstest] +#[case::open_port(7000, true)] +#[case::closed_port(7003, false)] +fn an_on_link_upstream_reaches_a_guest_through_proxy_ndp_as_root( + #[case] port: u16, + #[case] reachable: bool, +) { + let lab = require!(Upstream::OnLink); + + assert_eq!( + lab.reaches(Src::Wan, Proto::Tcp, guest_port(Vm::B, port)), + reachable + ); +} diff --git a/native/suidhelper/tests/net_addressing.rs b/native/suidhelper/tests/net_addressing.rs new file mode 100644 index 00000000..4f82837c --- /dev/null +++ b/native/suidhelper/tests/net_addressing.rs @@ -0,0 +1,150 @@ +//! Laws of a node's address plan, over every prefix the config accepts rather than the +//! cookbook's one `/64`: +//! - a node prefix parses exactly when it is global unicast (`2000::/3`) and `/64` or +//! shorter, and parsing clears its host bits; +//! - `:1::/80` and every guest address sit inside the node prefix, each guest +//! address maps back to its NetId, and distinct NetIds get distinct addresses; +//! - `Blocked::overlapping` agrees with an interval-intersection oracle, never pairs an +//! IPv4 range with an IPv6 one, and names the first overlapping range. + +use hyper_suidhelper::net::{Blocked, NetId, NodePrefix}; +use hyperrs_linux::net::IpNet; +use proptest::prelude::*; +use std::net::{Ipv4Addr, Ipv6Addr}; +use test_strategy::proptest; + +fn mask(bits: u32, len: u8) -> u128 { + match u32::from(len) { + 0 => 0, + len => u128::MAX << (bits - len) & (u128::MAX >> (128 - bits)), + } +} + +fn net_id(id: u16) -> NetId { + id.try_into().unwrap() +} + +/// `[first, last]` of the addresses `net` covers, as integers. +fn interval(net: IpNet) -> (u128, u128, bool) { + match net { + IpNet::V4(net) => { + let m = mask(32, net.prefix.get()); + let first = u128::from(u32::from(net.addr)) & m; + (first, first | (!m & u128::from(u32::MAX)), false) + } + IpNet::V6(net) => { + let m = mask(128, net.prefix.get()); + let first = u128::from(net.addr) & m; + (first, first | !m, true) + } + } +} + +fn shares_an_address(a: IpNet, b: IpNet) -> bool { + let ((a0, a1, a6), (b0, b1, b6)) = (interval(a), interval(b)); + a6 == b6 && a0 <= b1 && b0 <= a1 +} + +fn global_unicast_prefix() -> impl Strategy { + (any::(), 3u8..=64).prop_map(|(bits, len)| { + let addr = (bits & !(0b111 << 125)) | (0b001 << 125); + (Ipv6Addr::from(addr), len) + }) +} + +fn ip_net() -> impl Strategy { + prop_oneof![ + (any::(), 0u8..=32).prop_map(|(a, l)| format!("{}/{l}", Ipv4Addr::from(a))), + (any::(), 0u8..=128).prop_map(|(a, l)| format!("{}/{l}", Ipv6Addr::from(a))), + (0u8..=255, 0u8..=255, 8u8..=24).prop_map(|(a, b, l)| format!("10.{a}.{b}.0/{l}")), + (0u16..4, 0u8..=128).prop_map(|(a, l)| format!("2001:db8:{a}::/{l}")), + ] + .prop_map(|s| s.parse().unwrap()) +} + +#[proptest] +fn a_node_prefix_parses_exactly_when_global_unicast_and_at_most_64_bits( + addr: u128, + #[strategy(0u8..=128)] len: u8, +) { + let s = format!("{}/{len}", Ipv6Addr::from(addr)); + let global = len >= 3 && addr >> 125 == 0b001; + let parsed = s.parse::(); + + prop_assert_eq!(parsed.is_ok(), global && len <= 64, "{}", s); + if let Ok(prefix) = parsed { + prop_assert_eq!(u128::from(prefix.get().addr), addr & mask(128, len)); + prop_assert_eq!(prefix.get().prefix.get(), len); + } +} + +#[proptest] +fn every_guest_address_sits_in_the_node_prefix_and_maps_back_to_its_net_id( + #[strategy(global_unicast_prefix())] node: (Ipv6Addr, u8), + #[strategy(0u16..NetId::COUNT)] id: u16, +) { + let (addr, len) = node; + let prefix: NodePrefix = format!("{addr}/{len}").parse().unwrap(); + let network = u128::from(prefix.get().addr); + let range = prefix.vm_range(); + let guest = prefix.guest(net_id(id)); + + prop_assert_eq!(range.prefix.get(), 80); + prop_assert_eq!(u128::from(range.addr) & mask(128, len), network); + prop_assert_eq!(u128::from(guest) & mask(128, 80), u128::from(range.addr)); + prop_assert_eq!(u128::from(guest) & mask(128, len), network); + prop_assert_eq!(prefix.guest_net_id(guest), Some(net_id(id))); +} + +#[proptest] +fn distinct_net_ids_get_distinct_guest_addresses_under_any_prefix( + #[strategy(global_unicast_prefix())] node: (Ipv6Addr, u8), + #[strategy(0u16..NetId::COUNT)] a: u16, + #[strategy(0u16..NetId::COUNT)] b: u16, +) { + prop_assume!(a != b); + let (addr, len) = node; + let prefix: NodePrefix = format!("{addr}/{len}").parse().unwrap(); + + prop_assert_ne!(prefix.guest(net_id(a)), prefix.guest(net_id(b))); +} + +#[proptest] +fn no_address_outside_the_vm_range_maps_to_a_net_id( + #[strategy(global_unicast_prefix())] node: (Ipv6Addr, u8), + addr: u128, +) { + let (prefix_addr, len) = node; + let prefix: NodePrefix = format!("{prefix_addr}/{len}").parse().unwrap(); + + if let Some(id) = prefix.guest_net_id(Ipv6Addr::from(addr)) { + prop_assert_eq!(u128::from(prefix.guest(id)), addr); + } +} + +#[proptest] +fn overlapping_names_the_first_blocked_range_sharing_an_address( + #[strategy(prop::collection::vec(ip_net(), 0..8))] ranges: Vec, + #[strategy(ip_net())] net: IpNet, +) { + let blocked = + Blocked::try_from(ranges.iter().map(ToString::to_string).collect::>()).unwrap(); + + prop_assert_eq!( + blocked.overlapping(net), + ranges + .iter() + .copied() + .find(|range| shares_an_address(net, *range)) + ); +} + +#[proptest] +fn overlap_is_symmetric(#[strategy(ip_net())] a: IpNet, #[strategy(ip_net())] b: IpNet) { + let blocked = |net: IpNet| Blocked::try_from(vec![net.to_string()]).unwrap(); + + prop_assert_eq!( + blocked(b).overlapping(a).is_some(), + blocked(a).overlapping(b).is_some() + ); +} diff --git a/native/suidhelper/tests/request_wire_properties.rs b/native/suidhelper/tests/request_wire_properties.rs new file mode 100644 index 00000000..6ee9c900 --- /dev/null +++ b/native/suidhelper/tests/request_wire_properties.rs @@ -0,0 +1,269 @@ +//! The node's JSON encoder (Jason) and this parser must read every request the same way, so +//! [`Request::from_args`] is held to JSON semantics and to exact numeric limits. +//! +//! Laws: +//! - JSON semantics, not bytes: any key order and any insignificant whitespace, at any depth, +//! parses exactly when the canonical form does, and routes to the same op; +//! - every op name routes to its own variant, never to a sibling with the same fields (the two +//! socket claims share `{id}`); +//! - a key repeated in any object, at any depth, is refused even when both values agree, so no +//! two JSON parsers can disagree on which value wins; +//! - bounded integers are accepted exactly up to their documented maximum and refused past it; +//! zero is refused exactly where a field is non-zero. + +use hyper_suidhelper::cli::{Request, RequestError}; +use proptest::prelude::*; +use rstest::rstest; +use serde_json::{json, Map, Value}; +use std::ffi::OsString; +use test_strategy::proptest; + +const OPS: &[&str] = &[ + "vm_launch", + "vm_claim_api_socket", + "vm_claim_vsock_socket", + "vm_teardown", + "vm_firewall", + "img_activate", + "dm_remove", + "thin_pool_create", + "thin_volume_create", + "thin_destroy", + "thin_fork_delta", + "host_init", + "version", +]; + +fn parse_raw(raw: &str) -> Result { + Request::from_args([OsString::from(raw)].into_iter()) +} + +fn parse(request: &Value) -> Result { + parse_raw(&request.to_string()) +} + +fn fixture() -> Vec<(String, Value)> { + let fixture: Map = + serde_json::from_str(include_str!("fixtures/requests.json")).unwrap(); + fixture.into_iter().collect() +} + +fn request(which: usize) -> Value { + let mut all = fixture(); + let len = all.len(); + all.swap_remove(which % len).1 +} + +fn routes_to(request: &Request, op: &str) -> bool { + op == match request { + Request::VmLaunch(_) => "vm_launch", + Request::VmClaimApiSocket(_) => "vm_claim_api_socket", + Request::VmClaimVsockSocket(_) => "vm_claim_vsock_socket", + Request::VmTeardown(_) => "vm_teardown", + Request::VmFirewall(_) => "vm_firewall", + Request::ImgActivate(_) => "img_activate", + Request::DmRemove(_) => "dm_remove", + Request::ThinPoolCreate(_) => "thin_pool_create", + Request::ThinVolumeCreate(_) => "thin_volume_create", + Request::ThinDestroy(_) => "thin_destroy", + Request::ThinForkDelta(_) => "thin_fork_delta", + Request::HostInit(_) => "host_init", + Request::Version(_) => "version", + } +} + +/// One generated seed drives every shuffle and spacing choice of a rendering. +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + self.0 >> 33 + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } + + fn space(&mut self) -> &'static str { + [" ", "\n", "\t", "\r\n", "", " "][self.below(6)] + } +} + +fn render(value: &Value, rng: &mut Lcg) -> String { + match value { + Value::Object(map) => { + let mut entries: Vec<(&String, &Value)> = map.iter().collect(); + for i in (1..entries.len()).rev() { + entries.swap(i, rng.below(i + 1)); + } + let body: Vec = entries + .into_iter() + .map(|(k, v)| { + let key = Value::String(k.clone()); + let (a, b, c) = (rng.space(), rng.space(), rng.space()); + format!("{a}{key}{b}:{c}{}", render(v, rng)) + }) + .collect(); + format!("{{{}{}}}", body.join(","), rng.space()) + } + Value::Array(items) => { + let body: Vec = items + .iter() + .map(|v| format!("{}{}", rng.space(), render(v, rng))) + .collect(); + format!("[{}{}]", body.join(","), rng.space()) + } + scalar => scalar.to_string(), + } +} + +fn object_paths(value: &Value, at: String, out: &mut Vec) { + match value { + Value::Object(map) => { + out.push(at.clone()); + for (k, v) in map { + object_paths(v, format!("{at}/{k}"), out); + } + } + Value::Array(items) => { + for (i, v) in items.iter().enumerate() { + object_paths(v, format!("{at}/{i}"), out); + } + } + _ => {} + } +} + +/// `value` rendered with the first key of the object at `path` written twice, same value both. +fn with_duplicate_key(value: &Value, path: &str) -> Option { + const MARKER: &str = "__duplicate_marker__"; + let (key, repeated) = value.pointer(path)?.as_object()?.iter().next()?; + let duplicate = format!("{}:{}", Value::String(key.clone()), repeated); + let mut marked = value.clone(); + marked + .pointer_mut(path)? + .as_object_mut()? + .insert(MARKER.into(), json!(0)); + Some( + marked + .to_string() + .replacen(&format!("\"{MARKER}\":0"), &duplicate, 1), + ) +} + +#[rstest] +fn every_op_routes_to_its_own_variant() { + for (name, request) in fixture() { + let parsed = parse(&request).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!( + routes_to(&parsed, request["op"].as_str().unwrap()), + "{name} routed to another op" + ); + } +} + +#[rstest] +fn the_fixture_names_every_op() { + let mut named: Vec = fixture() + .into_iter() + .map(|(_, r)| r["op"].as_str().unwrap().to_owned()) + .collect(); + named.sort(); + named.dedup(); + let mut ops = OPS.to_vec(); + ops.sort(); + assert_eq!(named, ops); +} + +#[proptest] +fn key_order_and_whitespace_never_change_the_parse(which: usize, seed: u64) { + let request = request(which); + let raw = render(&request, &mut Lcg(seed)); + let parsed = parse_raw(&raw); + prop_assert!(parsed.is_ok(), "{raw}: {:?}", parsed.err()); + prop_assert!( + routes_to(&parsed.unwrap(), request["op"].as_str().unwrap()), + "{raw}" + ); +} + +#[proptest] +fn a_repeated_key_in_any_object_is_refused(which: usize, pick: usize) { + let request = request(which); + let mut paths = Vec::new(); + object_paths(&request, String::new(), &mut paths); + let path = &paths[pick % paths.len()]; + prop_assume!(!request + .pointer(path) + .unwrap() + .as_object() + .unwrap() + .is_empty()); + let raw = with_duplicate_key(&request, path).unwrap(); + prop_assert!( + matches!(parse_raw(&raw), Err(RequestError::Json(_))), + "accepted a repeated key at {path:?}: {raw}" + ); +} + +fn launch_with(field: &str, value: Value) -> Value { + let mut request = fixture() + .into_iter() + .find(|(name, _)| name == "vm_launch") + .unwrap() + .1; + request[field] = value; + request +} + +fn volume_with_thin_id(id: Value) -> Value { + json!({"op": "thin_volume_create", "pool": "hyper-pool", "thin_id": id, + "name": "hyper-v1", "origin": "/dev/mapper/hyper-img"}) +} + +#[proptest] +fn a_net_id_is_accepted_exactly_below_the_transit_capacity(#[strategy(0u32..=70_000)] id: u32) { + let accepted = parse(&launch_with("net_id", json!(id))).is_ok(); + prop_assert_eq!(accepted, id < 16_384, "net_id {}", id); +} + +#[proptest] +fn a_thin_id_is_accepted_exactly_within_24_bits( + #[strategy(prop_oneof![0u64..1 << 25, (1u64 << 24) - 4..(1 << 24) + 4, any::()])] id: u64, +) { + let accepted = parse(&volume_with_thin_id(json!(id))).is_ok(); + prop_assert_eq!(accepted, id < 1 << 24, "thin_id {}", id); +} + +#[rstest] +#[case::uid_max(launch_with("uid", json!(u32::MAX)), true)] +#[case::uid_past_u32(launch_with("uid", json!(u64::from(u32::MAX) + 1)), false)] +#[case::gid_zero_parses_and_is_left_to_the_range_check(launch_with("gid", json!(0)), true)] +#[case::thin_id_max(volume_with_thin_id(json!((1u64 << 24) - 1)), true)] +#[case::limits_empty(launch_with("limits", json!({})), true)] +#[case::limits_null_entries(launch_with("limits", json!({"memory_max": null, "cpu_max": null})), true)] +#[case::memory_max_u64(launch_with("limits", json!({"memory_max": u64::MAX})), true)] +#[case::memory_max_zero(launch_with("limits", json!({"memory_max": 0})), false)] +#[case::cpu_quota_zero(launch_with("limits", json!({"cpu_max": {"quota_us": 0, "period_us": 1}})), false)] +#[case::cpu_period_zero(launch_with("limits", json!({"cpu_max": {"quota_us": 1, "period_us": 0}})), false)] +#[case::cpu_max_missing_period(launch_with("limits", json!({"cpu_max": {"quota_us": 1}})), false)] +#[case::vm_id_64_chars(launch_with("id", json!("a".repeat(64))), true)] +#[case::vm_id_65_chars(launch_with("id", json!("a".repeat(65))), false)] +#[case::vm_id_inner_dash(launch_with("id", json!("v-1")), true)] +#[case::vm_id_non_ascii_digit(launch_with("id", json!("v\u{0661}")), false)] +#[case::device_traversing_mapper(launch_with("device", json!("/dev/mapper/../sda1")), false)] +fn bounded_fields_are_accepted_exactly_up_to_their_limit( + #[case] request: Value, + #[case] accepted: bool, +) { + assert_eq!( + parse(&request).is_ok(), + accepted, + "{request}: {:?}", + parse(&request).err() + ); +} diff --git a/test/controls/accumulator_rebase_properties_test.exs b/test/controls/accumulator_rebase_properties_test.exs new file mode 100644 index 00000000..47d1dcc9 --- /dev/null +++ b/test/controls/accumulator_rebase_properties_test.exs @@ -0,0 +1,51 @@ +defmodule Controls.AccumulatorRebasePropertiesTest do + @moduledoc """ + Laws for `Controls.Accumulator.rebase/1`, the reboot hook (#134): the + counter's source is known to have been recreated at zero. + + * rebasing never changes what has accrued but not yet been flushed — a + reboot between flushes loses nothing; + * rebasing is idempotent; + * after a rebase, the first reading accrues in full, whatever the old + counter last read (above or below it) and whether or not a flush ran. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Controls.Accumulator + alias Unit.Time + + defp readings, do: list_of(integer(0..1_000_000_000), min_length: 1) + + defp fold(acc, values), do: Enum.reduce(values, acc, &Accumulator.observe(&2, Time.us(&1))) + + defp total_us(acc), do: Time.as_us(Accumulator.total(acc)) + + property "a rebase keeps everything accrued since the last flush" do + check all(values <- readings()) do + acc = fold(Accumulator.new(Time.zero()), values) + assert total_us(Accumulator.rebase(acc)) == total_us(acc) + end + end + + property "rebasing twice is rebasing once" do + check all(values <- readings(), next <- integer(0..1_000_000_000)) do + once = values |> then(&fold(Accumulator.new(Time.zero()), &1)) |> Accumulator.rebase() + twice = Accumulator.rebase(once) + + assert total_us(Accumulator.observe(twice, Time.us(next))) == + total_us(Accumulator.observe(once, Time.us(next))) + end + end + + property "the first reading after a rebase accrues in full" do + check all(values <- readings(), flush? <- boolean(), next <- integer(0..1_000_000_000)) do + acc = fold(Accumulator.new(Time.zero()), values) + acc = if flush?, do: Accumulator.flush(acc), else: acc + + rebased = acc |> Accumulator.rebase() |> Accumulator.observe(Time.us(next)) + + assert total_us(rebased) == total_us(acc) + next + end + end +end diff --git a/test/e2e/hostile_guest_network_test.exs b/test/e2e/hostile_guest_network_test.exs new file mode 100644 index 00000000..8b113533 --- /dev/null +++ b/test/e2e/hostile_guest_network_test.exs @@ -0,0 +1,86 @@ +defmodule Hyper.E2e.HostileGuestNetworkTest do + @moduledoc """ + A root shell inside a real guest, turned against the host it runs on. + + Laws (docs/cookbook/networking.md, Security): + + * a guest opens no connection to any address the host holds, over IPv4 or + IPv6, on any interface; + * a guest never reaches the provider's metadata service; + * while it still reaches the internet, so the refusals above are the + firewall and not a guest without a network. + """ + use ExUnit.Case, async: false + + import Hyper.E2e + + @moduletag :integration + @moduletag timeout: :timer.minutes(10) + + @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19") + @connect_timeout_s 3 + + setup_all do + {:ok, img_id} = Hyper.Img.OciLoader.load(@image) + {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) + on_exit(fn -> Hyper.Node.stop_image_vm(vm) end) + {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"]) + %{vm: vm} + end + + defp host_addresses do + {:ok, ifaddrs} = :inet.getifaddrs() + + for {_name, opts} <- ifaddrs, + {:addr, addr} <- opts, + not loopback?(addr), + not link_local?(addr), + uniq: true, + do: addr + end + + defp loopback?({127, _, _, _}), do: true + defp loopback?({0, 0, 0, 0, 0, 0, 0, 1}), do: true + defp loopback?(_), do: false + + defp link_local?({169, 254, _, _}), do: true + defp link_local?({a, _, _, _, _, _, _, _}), do: Bitwise.band(a, 0xFFC0) == 0xFE80 + defp link_local?(_), do: false + + defp guest_tcp(vm, addr, port) do + {:ok, %{exit_code: code}} = + await_exec(vm, [ + "/usr/bin/nc", + "-z", + "-w", + "#{@connect_timeout_s}", + to_string(:inet.ntoa(addr)), + "#{port}" + ]) + + code + end + + test "the guest opens no connection to any host address", %{vm: vm} do + {:ok, listener} = + :gen_tcp.listen(0, [:inet6, {:ipv6_v6only, false}, active: false, reuseaddr: true]) + + on_exit(fn -> :gen_tcp.close(listener) end) + {:ok, port} = :inet.port(listener) + addrs = host_addresses() + assert addrs != [] + + for addr <- addrs, do: guest_tcp(vm, addr, port) + + assert :gen_tcp.accept(listener, 1_000) == {:error, :timeout}, + "a guest connection reached the host on one of #{inspect(addrs)}" + end + + test "the guest never reaches the metadata service", %{vm: vm} do + assert guest_tcp(vm, {169, 254, 169, 254}, 80) != 0 + end + + test "the guest still reaches the internet", %{vm: vm} do + assert guest_tcp(vm, {1, 1, 1, 1}, 53) == 0 + end +end diff --git a/test/e2e/vm_firewall_test.exs b/test/e2e/vm_firewall_test.exs new file mode 100644 index 00000000..d0464982 --- /dev/null +++ b/test/e2e/vm_firewall_test.exs @@ -0,0 +1,96 @@ +defmodule Hyper.E2e.VmFirewallTest do + @moduledoc """ + `Hyper.set_vm_firewall/2` against two real guests on one node. + + Laws (docs/cookbook/networking.md, Security): + + * a VM's inbound ports are closed until the operator opens them, even to + another VM on the same host reaching it over its public IPv6 address; + * opening a port admits new connections, and closing it again refuses them; + * an outbound-blocked port never leaves the VM, and unblocking it restores it. + """ + use ExUnit.Case, async: false + + import Hyper.E2e + + alias Hyper.Vm.Firewall + + @moduletag :integration + @moduletag timeout: :timer.minutes(10) + + @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19") + @port 8080 + + setup_all do + {:ok, img_id} = Hyper.Img.OciLoader.load(@image) + vms = for _ <- 1..2, do: boot(img_id) + on_exit(fn -> Enum.each(vms, &Hyper.Node.stop_image_vm/1) end) + [client, server] = vms + %{client: client, server: server, server_ip6: ip6(server)} + end + + defp boot(img_id) do + {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) + {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"]) + vm + end + + defp ip6(vm) do + {:ok, %{stdout: cmdline}} = await_exec(vm, ["/bin/cat", "/proc/cmdline"]) + [_, ip6] = Regex.run(~r/\bhyper\.ip6=(\S+)/, cmdline) + ip6 + end + + defp listen(vm) do + {:ok, %{exit_code: 0}} = + await_exec(vm, [ + "/bin/sh", + "-c", + "[ -e /run/listening ] || { touch /run/listening; " <> + "(setsid sh -c 'while :; do nc -l -p #{@port}; sleep 0.1; done' " <> + "/dev/null 2>&1 &); }" + ]) + + :ok + end + + defp reaches?(vm, host, port) do + {:ok, %{exit_code: code}} = + await_exec(vm, ["/usr/bin/nc", "-z", "-w", "3", host, "#{port}"]) + + code == 0 + end + + defp set!(vm, inbound, outbound_blocked) do + {:ok, firewall} = Firewall.new(inbound, outbound_blocked) + assert Hyper.set_vm_firewall(Hyper.id(vm), firewall) == :ok + end + + test "an inbound port opens and closes for another VM on the host", ctx do + listen(ctx.server) + + closed = reaches?(ctx.client, ctx.server_ip6, @port) + set!(ctx.server, [{:tcp, @port..@port//1}], []) + opened = reaches?(ctx.client, ctx.server_ip6, @port) + set!(ctx.server, [], []) + reclosed = reaches?(ctx.client, ctx.server_ip6, @port) + + assert {closed, opened, reclosed} == {false, true, false} + end + + test "a guest reaches its own listener, so a refusal is the firewall", ctx do + listen(ctx.server) + + assert reaches?(ctx.server, "127.0.0.1", @port) + end + + test "an outbound-blocked port never leaves the VM until unblocked", ctx do + before = reaches?(ctx.client, "1.1.1.1", 53) + set!(ctx.client, [], [{:tcp, 53..53//1}]) + blocked = reaches?(ctx.client, "1.1.1.1", 53) + set!(ctx.client, [], []) + unblocked = reaches?(ctx.client, "1.1.1.1", 53) + + assert {before, blocked, unblocked} == {true, false, true} + end +end diff --git a/test/hyper/cluster/scheduler_properties_test.exs b/test/hyper/cluster/scheduler_properties_test.exs new file mode 100644 index 00000000..202f429f --- /dev/null +++ b/test/hyper/cluster/scheduler_properties_test.exs @@ -0,0 +1,196 @@ +defmodule Hyper.Cluster.SchedulerPropertiesTest do + @moduledoc """ + Laws for placement (`Hyper.Cluster.Scheduler`). + + * `colocation_score/2` equals an independent oracle — the bytes of the + VM's layers that are mounted on the node — is invariant under + reordering the layers, and is unchanged by adding a layer the node does + not hold; + * `candidates/2` lists every fitting node exactly once, most colocated + bytes first; + * `place/3` walks candidates in rank order, confirming each at most once: + it stops at the first confirmation (never confirming a second node, so a + VM is never duplicated), stops at the first node lost mid-call (which + may have booted the VM) with `:node_unreachable`, skips refusals, and + answers `:no_capacity` only when every candidate refused. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Cluster.Budget + alias Hyper.Cluster.Scheduler + alias Hyper.Node.Budget.NodeState + alias Hyper.Vm.Instance.Spec + alias Unit.{Bandwidth, Information} + + @moduletag :capture_log + + setup do + if Process.whereis(Budget.name()) do + :ok + else + _ = Application.ensure_all_started(:horde) + start_supervised!(Budget) + :ok + end + end + + defp layer_ids, do: list_of(string(:alphanumeric, min_length: 1, max_length: 6), max_length: 8) + + defp sized_layers do + gen all( + ids <- uniq_list_of(string(:alphanumeric, min_length: 1, max_length: 6), max_length: 8), + sizes <- list_of(integer(0..1_000_000), length: length(ids)) + ) do + Enum.zip(ids, Enum.map(sizes, &Information.bytes/1)) + end + end + + property "colocation score is the byte sum of the VM's layers the node has mounted" do + check all(layers <- sized_layers(), mounted <- layer_ids()) do + state = node_state(:oracle@nowhere, mounted) + mounted_set = MapSet.new(mounted) + + oracle = + for {id, size} <- layers, MapSet.member?(mounted_set, id), reduce: 0 do + acc -> acc + Information.as_bytes(size) + end + + assert Scheduler.colocation_score(state, layers) == oracle + assert Scheduler.colocation_score(state, Enum.reverse(layers)) == oracle + end + end + + property "a layer the node does not hold never changes its score" do + check all(layers <- sized_layers(), mounted <- layer_ids(), size <- integer(1..1_000_000)) do + state = node_state(:oracle@nowhere, mounted) + foreign = {"not-mounted-" <> Enum.join(mounted, "-"), Information.bytes(size)} + + assert Scheduler.colocation_score(state, [foreign | layers]) == + Scheduler.colocation_score(state, layers) + end + end + + property "candidates are every fitting node once, most colocated bytes first" do + check all(count <- integer(1..5), bytes <- uniq_list_of(integer(1..1_000), length: count)) do + nodes = Enum.map(1..count, &:"cand_#{&1}@nowhere") + states = Enum.zip(nodes, bytes) + layers = Enum.map(states, fn {node, b} -> {layer_for(node), Information.mib(b)} end) + + ranked = with_advertised(nodes, fn -> Scheduler.candidates(spec(), layers) end) + + expected = states |> Enum.sort_by(&elem(&1, 1), :desc) |> Enum.map(&elem(&1, 0)) + assert ranked == expected + end + end + + property "place confirms at most one node, in rank order, stopping at the first ok or loss" do + check all(outcomes <- list_of(member_of([:ok, :refuse, :lost]), min_length: 1, max_length: 5)) do + nodes = Enum.map(1..length(outcomes), &:"walk_#{&1}@nowhere") + plan = Map.new(Enum.zip(nodes, outcomes)) + layers = nodes |> Enum.with_index() |> Enum.map(&rank_layer/1) + test = self() + + attempt = fn node -> + send(test, {:attempt, node}) + + case Map.fetch!(plan, node) do + :ok -> {:ok, {:booted_on, node}} + :refuse -> {:error, :mem_exhausted} + :lost -> {:error, {:lost_mid_call, :noconnection}} + end + end + + result = with_advertised(nodes, fn -> Scheduler.place(spec(), layers, attempt) end) + attempted = drain_attempts() + + {walked, rest} = Enum.split_while(Enum.zip(nodes, outcomes), &(elem(&1, 1) == :refuse)) + + {expected_attempts, expected_result} = + case rest do + [{node, :ok} | _] -> + {Enum.map(walked, &elem(&1, 0)) ++ [node], {:ok, {node, {:booted_on, node}}}} + + [{node, :lost} | _] -> + {Enum.map(walked, &elem(&1, 0)) ++ [node], {:error, :node_unreachable}} + + [] -> + {nodes, {:error, :no_capacity}} + end + + assert attempted == expected_attempts + assert result == expected_result + end + end + + defp drain_attempts(acc \\ []) do + receive do + {:attempt, node} -> drain_attempts([node | acc]) + after + 0 -> Enum.reverse(acc) + end + end + + defp rank_layer({node, i}), do: {layer_for(node), Information.mib(1_000 - i)} + + defp layer_for(node), do: "layer-" <> Atom.to_string(node) + + defp with_advertised(nodes, fun) do + keys = + Enum.map(nodes, fn node -> + key = {:node, {__MODULE__, node}} + + {:ok, _} = + Horde.Registry.register(Budget.name(), key, node_state(node, [layer_for(node)])) + + key + end) + + await_fitting(length(nodes)) + + try do + fun.() + after + Enum.each(keys, &Horde.Registry.unregister(Budget.name(), &1)) + await_fitting(0) + end + end + + defp await_fitting(n, tries \\ 500) do + fitting = Enum.count(Budget.all_states(), &NodeState.fits?(&1, spec())) + + cond do + fitting == n -> :ok + tries == 0 -> flunk("budget registry never settled at #{n} fitting nodes (saw #{fitting})") + true -> await_fitting(n, tries - 1) + end + end + + # 64 TiB of memory: no real node's own advertisement fits, so only the + # states this suite registers are candidates. + defp spec do + %Spec{ + vcpus: 1, + mem: Information.tib(64), + disk: Information.gib(1), + disk_bw: Bandwidth.mibps(1), + net_bw: Bandwidth.mibps(1) + } + end + + defp node_state(node, layers) do + %NodeState{ + node: node, + mem_free: Information.tib(128), + disk_free: Information.gib(100), + cpu_load: 0.0, + cpu_capacity: 8, + cpu_max_load: 1.0, + disk_bw_load: Bandwidth.zero(), + disk_bw_ceiling: Bandwidth.gibps(1), + net_bw_load: Bandwidth.zero(), + net_bw_ceiling: Bandwidth.gibps(1), + layers: layers + } + end +end diff --git a/test/hyper/grpc/page_forged_token_properties_test.exs b/test/hyper/grpc/page_forged_token_properties_test.exs new file mode 100644 index 00000000..47fd2cc7 --- /dev/null +++ b/test/hyper/grpc/page_forged_token_properties_test.exs @@ -0,0 +1,70 @@ +defmodule Hyper.Grpc.PageForgedTokenPropertiesTest do + @moduledoc """ + `Hyper.Grpc.Page` under a hostile ListVms caller, who controls both + `page_token` (any string) and `page_size` (any int32), over the real + `Hyper.Grpc.Pageable.Vm`. + + Laws: + + * total: no token string and no page size makes `paginate/4` raise — it + answers a page or `:bad_page_token`; + * a forged token cannot read outside keyset order: whatever cursor `c` it + decodes to, the page is exactly the first `min(size, max)` VMs (default + for a non-positive size) whose id sorts strictly after `c`, so a forged + cursor neither skips nor re-lists a VM relative to that position; + * the next token a forged page returns resumes exactly after that page. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Grpc.Page + alias Hyper.Grpc.Pageable.Vm, as: Pageable + + defp vms do + gen all( + ids <- uniq_list_of(string(:printable, min_length: 1, max_length: 12), max_length: 60), + nodes <- list_of(member_of([:a@h, :b@h, :c@h]), length: length(ids)) + ) do + Enum.zip(ids, nodes) + end + end + + defp int32, do: one_of([integer(-5..1_200), member_of([-2_147_483_648, 2_147_483_647])]) + + defp effective(size) when size <= 0, do: Pageable.default_page_size() + defp effective(size), do: min(size, Pageable.max_page_size()) + + property "any token string and int32 size answers a page or :bad_page_token, never a raise" do + check all( + entries <- vms(), + size <- int32(), + token <- one_of([string(:printable), string(:utf8), binary()]) + ) do + case Page.paginate(entries, size, token, Pageable) do + {:ok, {page, next}} -> assert is_list(page) and is_binary(next) + {:error, :bad_page_token} -> :ok + end + end + end + + property "a forged cursor pages exactly the VMs sorting after it, and resumes after the page" do + check all(entries <- vms(), cursor <- binary(min_length: 1, max_length: 14), size <- int32()) do + token = Base.url_encode64(cursor, padding: false) + {:ok, {page, next}} = Page.paginate(entries, size, token, Pageable) + + after_cursor = entries |> Enum.filter(fn {id, _} -> id > cursor end) |> Enum.sort() + assert page == Enum.take(after_cursor, effective(size)) + + case next do + "" -> + assert length(after_cursor) <= effective(size) + + _ -> + {:ok, {following, _}} = Page.paginate(entries, size, next, Pageable) + + assert following == + after_cursor |> Enum.drop(length(page)) |> Enum.take(effective(size)) + end + end + end +end diff --git a/test/hyper/node/fire_vmm/agent/relay_budget_scope_test.exs b/test/hyper/node/fire_vmm/agent/relay_budget_scope_test.exs new file mode 100644 index 00000000..d55c2857 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent/relay_budget_scope_test.exs @@ -0,0 +1,97 @@ +defmodule Hyper.Node.FireVMM.Agent.RelayBudgetScopeTest do + @moduledoc """ + Scope of the relay's 16 MiB guest-to-host reply cap (#132): + + - it is per connection, not per relay: successive connections each get a + fresh budget, so a VM that has answered more than the cap in total over + several execs is still served; + - it caps only the guest-to-host direction: the host can send the guest + more than the cap on one connection. + """ + use ExUnit.Case, async: true + + alias Hyper.Node.FireVMM.Agent.Relay + alias Unit.Information + + @cap Information.as_bytes(Information.mib(16)) + @recv_chunk Information.as_bytes(Information.mib(2)) + + setup do + vsock_path = tmp_path("vsock") + listen_path = tmp_path("listen") + + {:ok, fc} = :socket.open(:local, :stream) + :ok = :socket.bind(fc, %{family: :local, path: vsock_path}) + :ok = :socket.listen(fc, 16) + + start_supervised!( + {Relay, %{vm_id: "vscope", vsock_uds: vsock_path, listen_path: listen_path}} + ) + + on_exit(fn -> + :socket.close(fc) + File.rm(vsock_path) + end) + + {:ok, %{fc: fc, listen_path: listen_path}} + end + + defp tmp_path(label), + do: Path.join(System.tmp_dir!(), "rscope-#{label}-#{System.unique_integer([:positive])}.sock") + + defp guest(fc, fun) do + parent = self() + + spawn_link(fn -> + {:ok, conn} = :socket.accept(fc) + {:ok, "CONNECT " <> _} = :socket.recv(conn) + :ok = :socket.send(conn, "OK 5\n") + fun.(conn, parent) + end) + end + + defp client(listen_path) do + {:ok, sock} = :socket.open(:local, :stream) + :ok = :socket.connect(sock, %{family: :local, path: listen_path}) + sock + end + + defp drain(sock, acc \\ []) do + case :socket.recv(sock, 0, 5_000) do + {:ok, data} -> drain(sock, [acc | data]) + {:error, reason} -> {reason, IO.iodata_to_binary(acc)} + end + end + + defp recv_exactly(sock, n, acc \\ []) + defp recv_exactly(_sock, 0, acc), do: IO.iodata_to_binary(acc) + + defp recv_exactly(sock, n, acc) do + {:ok, data} = :socket.recv(sock, min(n, @recv_chunk), 5_000) + recv_exactly(sock, n - byte_size(data), [acc | data]) + end + + test "each connection gets a fresh budget", %{fc: fc, listen_path: listen_path} do + per_conn = :crypto.strong_rand_bytes(div(@cap * 3, 4)) + + for _ <- 1..3 do + guest(fc, fn conn, _ -> + :ok = :socket.send(conn, per_conn) + :socket.close(conn) + end) + + assert drain(client(listen_path)) == {:closed, per_conn} + end + end + + test "the host can send the guest more than the cap", %{fc: fc, listen_path: listen_path} do + upload = :crypto.strong_rand_bytes(@cap + Information.as_bytes(Information.mib(4))) + + guest(fc, fn conn, parent -> + send(parent, {:guest_got, recv_exactly(conn, byte_size(upload))}) + end) + + :ok = :socket.send(client(listen_path), upload) + assert_receive {:guest_got, ^upload}, 10_000 + end +end diff --git a/test/hyper/node/fire_vmm/agent/relay_cap_properties_test.exs b/test/hyper/node/fire_vmm/agent/relay_cap_properties_test.exs new file mode 100644 index 00000000..1ba3617a --- /dev/null +++ b/test/hyper/node/fire_vmm/agent/relay_cap_properties_test.exs @@ -0,0 +1,220 @@ +defmodule Hyper.Node.FireVMM.Agent.RelayCapPropertiesTest do + @moduledoc """ + The relay's guest-to-host reply cap (#132), against a hostile guest that + chooses how many bytes to stream and how to chunk them: + + - **never over the cap** — the client never receives more than the cap, + however the guest chunks its stream; + - **transparent under the cap** — a guest that stays within the cap is + relayed byte-for-byte, and the connection stays open; + - **cumulative, not per-chunk** — the budget is spent across the whole + connection, so many small chunks overflow it exactly like one big one; + - **prefix integrity** — what the client receives is always a prefix of what + the guest sent: the cap truncates, it never reorders or corrupts; + - **the cap is inclusive** — a guest may send exactly the cap; one byte more + closes the connection after exactly the cap was delivered; + - **overflow tears down both sides** — the guest-side (vsock) connection is + closed too, so an overflowing guest cannot pin a host-side connection; + - **overflow is per connection** — it never disturbs another connection or + the relay's ability to accept new ones. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Node.FireVMM.Agent.Relay + alias Unit.Information + + @cap Information.as_bytes(Information.mib(16)) + @quiet_ms 300 + + defp tmp_path(label) do + Path.join(System.tmp_dir!(), "relaycap-#{label}-#{System.unique_integer([:positive])}.sock") + end + + setup do + vsock_path = tmp_path("vsock") + listen_path = tmp_path("listen") + + {:ok, fc} = :socket.open(:local, :stream) + :ok = :socket.bind(fc, %{family: :local, path: vsock_path}) + :ok = :socket.listen(fc, 16) + + {:ok, relay} = + Relay.start_link(%{vm_id: "vcap", vsock_uds: vsock_path, listen_path: listen_path}) + + true = Process.unlink(relay) + + on_exit(fn -> + if Process.alive?(relay), do: GenServer.stop(relay) + :socket.close(fc) + File.rm(vsock_path) + File.rm(listen_path) + end) + + %{fc: fc, listen_path: listen_path} + end + + defp guest(fc, script) do + test = self() + + spawn(fn -> + {:ok, conn} = :socket.accept(fc) + {:ok, "CONNECT " <> _} = :socket.recv(conn, 0, 2_000) + :ok = :socket.send(conn, "OK 5\n") + script.(conn) + send(test, {:guest_sees, guest_fate(conn)}) + end) + end + + defp guest_fate(conn) do + case :socket.recv(conn, 0, 3_000) do + {:ok, _} -> guest_fate(conn) + {:error, reason} -> reason + end + end + + defp connect(listen_path) do + {:ok, client} = :socket.open(:local, :stream) + :ok = :socket.connect(client, %{family: :local, path: listen_path}) + client + end + + defp drain(sock, acc) do + case :socket.recv(sock, 0, @quiet_ms) do + {:ok, data} -> drain(sock, [acc | data]) + {:error, :closed} -> {:closed, IO.iodata_to_binary(acc)} + {:error, :timeout} -> {:open, IO.iodata_to_binary(acc)} + end + end + + defp payload(total, seed) do + _ = :rand.seed(:exsss, seed) + :rand.bytes(total) + end + + defp chunked(bin, sizes), do: chunked(bin, sizes, sizes) + + defp chunked(<<>>, _sizes, _all), do: [] + defp chunked(bin, [], all), do: chunked(bin, all, all) + + defp chunked(bin, [n | sizes], all) do + take = min(n, byte_size(bin)) + <> = bin + [chunk | chunked(rest, sizes, all)] + end + + defp send_chunks(conn, chunks), do: Enum.reduce_while(chunks, :ok, &send_chunk(conn, &1, &2)) + + defp send_chunk(conn, chunk, :ok) do + case :socket.send(conn, chunk) do + :ok -> {:cont, :ok} + {:error, _} -> {:halt, :refused} + end + end + + property "the client gets a prefix of the guest's stream, never more than the cap", + %{fc: fc, listen_path: listen_path} do + check all( + total <- + one_of([ + integer(0..65_536), + integer((@cap - Information.as_bytes(Information.mib(1)))..@cap), + integer((@cap + 1)..(@cap + Information.as_bytes(Information.mib(2)))) + ]), + sizes <- + list_of(integer(512..Information.as_bytes(Information.mib(4))), min_length: 1), + seed <- {integer(1..1_000_000), integer(1..1_000_000), integer(1..1_000_000)}, + max_runs: 12 + ) do + sent = payload(total, seed) + chunks = chunked(sent, sizes) + guest(fc, fn conn -> send_chunks(conn, chunks) end) + client = connect(listen_path) + + {fate, got} = drain(client, []) + :socket.close(client) + + assert byte_size(got) <= @cap, "relayed #{byte_size(got)}B past the #{@cap}B cap" + assert got == binary_part(sent, 0, byte_size(got)), "received bytes are not a prefix" + + if total <= @cap do + assert {fate, byte_size(got)} == {:open, total} + else + assert fate == :closed, "#{total}B guest stream left the connection open" + end + end + end + + test "many small chunks overflow the cap the same as one big chunk", + %{fc: fc, listen_path: listen_path} do + chunk = :binary.copy(<<7>>, 4_096) + n = div(@cap, 4_096) + 1 + + guest(fc, fn conn -> send_chunks(conn, List.duplicate(chunk, n)) end) + client = connect(listen_path) + + assert {:closed, got} = drain(client, []) + assert byte_size(got) <= @cap + end + + test "exactly the cap is delivered and kept open; the next byte closes after the cap", + %{fc: fc, listen_path: listen_path} do + guest = + guest(fc, fn conn -> + :ok = :socket.send(conn, :binary.copy(<<1>>, @cap)) + + receive do + :one_more -> :ok = :socket.send(conn, <<2>>) + end + end) + + client = connect(listen_path) + assert {:open, got} = drain(client, []) + assert byte_size(got) == @cap + + send(guest, :one_more) + assert {:closed, ""} = drain(client, []) + assert_receive {:guest_sees, :closed}, 5_000 + end + + test "overflow closes the guest-side connection, not just the client's", + %{fc: fc, listen_path: listen_path} do + guest(fc, fn conn -> send_chunks(conn, [:binary.copy(<<3>>, @cap + 1)]) end) + client = connect(listen_path) + + assert {:closed, got} = drain(client, []) + assert byte_size(got) <= @cap + assert_receive {:guest_sees, :closed}, 5_000 + end + + test "one connection overflowing leaves a concurrent connection and the relay serving", + %{fc: fc, listen_path: listen_path} do + guest(fc, &echo/1) + calm = connect(listen_path) + assert ping(calm, "before") + + guest(fc, fn conn -> send_chunks(conn, [:binary.copy(<<4>>, @cap + 1)]) end) + flood = connect(listen_path) + assert {:closed, _} = drain(flood, []) + + assert ping(calm, "after") + guest(fc, &echo/1) + assert ping(connect(listen_path), "new") + end + + defp echo(conn) do + case :socket.recv(conn, 0, 10_000) do + {:ok, data} -> + :ok = :socket.send(conn, data) + echo(conn) + + {:error, _} -> + :ok + end + end + + defp ping(sock, msg) do + :ok = :socket.send(sock, msg) + :socket.recv(sock, byte_size(msg), 2_000) == {:ok, msg} + end +end diff --git a/test/hyper/node/fire_vmm/agent/relay_dialer_properties_test.exs b/test/hyper/node/fire_vmm/agent/relay_dialer_properties_test.exs new file mode 100644 index 00000000..be851083 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent/relay_dialer_properties_test.exs @@ -0,0 +1,95 @@ +defmodule Hyper.Node.FireVMM.Agent.RelayDialerPropertiesTest do + @moduledoc """ + `RelayDialer.dial/3`'s handshake refusals (#132), over arbitrary reply lines + the vsock UDS can answer `CONNECT` with: + + - **length bound** — a line whose content (before the LF) is 64 bytes or more + is always `{:error, :vsock_line_too_long}`, whatever its bytes; a shorter + one is never refused for length; + - **exact acceptance** — a short line is accepted iff it starts with the + exact bytes `"OK "`; any other short line is `{:error, {:vsock_no_ok, + line}}` carrying the line verbatim (case, leading space, missing space and + NUL/non-UTF-8 bytes do not pass); + - **no over-read** — on acceptance every byte after the LF is left on the + socket for the caller. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Node.FireVMM.Agent.RelayDialer + + @max_line_bytes 64 + + setup do + path = Path.join(System.tmp_dir!(), "dialprop-#{System.unique_integer([:positive])}.sock") + {:ok, srv} = :socket.open(:local, :stream) + :ok = :socket.bind(srv, %{family: :local, path: path}) + :ok = :socket.listen(srv, 4) + + on_exit(fn -> + :socket.close(srv) + File.rm(path) + end) + + %{path: path, srv: srv} + end + + defp answer(srv, bytes) do + spawn_link(fn -> + {:ok, conn} = :socket.accept(srv) + {:ok, "CONNECT 1024\n"} = :socket.recv(conn, 0, 2_000) + _ = :socket.send(conn, bytes) + Process.sleep(:infinity) + end) + end + + defp no_lf(gen), do: map(gen, &:binary.replace(&1, "\n", "x", [:global])) + + defp line do + one_of([ + no_lf(binary(max_length: @max_line_bytes + 8)), + map(no_lf(binary(max_length: @max_line_bytes)), &("OK " <> &1)), + member_of(["ok 5", " OK 5", "OK5", "OK", "OK\t5", "OK\0 5", <<"OK", 0xC3, " 5">>]), + map( + integer((@max_line_bytes - 2)..(@max_line_bytes + 2)), + &("OK " <> :binary.copy("5", &1 - 3)) + ) + ]) + end + + property "a line is refused for length iff it is 64 bytes or more, else decided by its OK prefix", + %{path: path, srv: srv} do + check all(line <- line(), tail <- binary(max_length: 32), max_runs: 200) do + answer(srv, line <> "\n" <> tail) + result = RelayDialer.dial(path, 1024, 1_000) + + cond do + byte_size(line) >= @max_line_bytes -> + assert result == {:error, :vsock_line_too_long} + + String.starts_with?(line, "OK ") -> + assert {:ok, sock} = result + + if tail != "" do + assert :socket.recv(sock, byte_size(tail), 1_000) == {:ok, tail} + end + + :socket.close(sock) + + true -> + assert result == {:error, {:vsock_no_ok, line}} + end + end + end + + property "an unterminated line of 64 bytes or more is refused without waiting for the deadline", + %{path: path, srv: srv} do + check all(junk <- no_lf(binary(min_length: @max_line_bytes, max_length: 4_096)), max_runs: 30) do + answer(srv, junk) + {us, result} = :timer.tc(fn -> RelayDialer.dial(path, 1024, 5_000) end) + + assert result == {:error, :vsock_line_too_long} + assert us < 1_000_000, "waited #{div(us, 1_000)}ms on an already-overlong line" + end + end +end diff --git a/test/hyper/node/fire_vmm/agent_relay_test.exs b/test/hyper/node/fire_vmm/agent_relay_test.exs new file mode 100644 index 00000000..60b56884 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent_relay_test.exs @@ -0,0 +1,145 @@ +defmodule Hyper.Node.FireVMM.AgentRelayTest do + @moduledoc """ + `Agent.exec/3` through a real `Relay` and a fake Firecracker vsock that + splices each `CONNECT` to an in-process gRPC guest: + + - a reply under the relay's 16 MiB cap round-trips intact: the cap bounds + a hostile guest, it must not truncate or reject an honest large reply; + - a reply over the cap is an error that is not `:timeout`, and the relay + keeps serving: the next exec on the same VM succeeds; + - concurrent execs on one VM each get their own reply, never another + call's. + + `async: false`: `Hyper.Cfg.Toml.put_cache/1` is process-global. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Agent.V1.{ExecResponse, HealthResponse} + alias Hyper.Node.FireVMM.Agent + alias Hyper.Node.FireVMM.Agent.Relay + alias Unit.Information + + @vm_id "vrelayexec" + + defmodule Guest do + @moduledoc false + use GRPC.Server, service: Hyper.Agent.V1.GuestAgent.Service + + def exec(%{argv: ["/bin/head", bytes]}, _stream), + do: %ExecResponse{exit_code: 0, stdout: :binary.copy("y", String.to_integer(bytes))} + + def exec(%{argv: ["/bin/echo" | rest]}, _stream) do + Process.sleep(:rand.uniform(50)) + %ExecResponse{exit_code: 0, stdout: Enum.join(rest, " ")} + end + + def health(_req, _stream), do: %HealthResponse{ok: true} + end + + setup_all do + {:ok, _} = Application.ensure_all_started(:grpc) + {:ok, _} = Application.ensure_all_started(:cowboy) + original = Hyper.Cfg.Toml.reload() + tmp = Path.join(System.tmp_dir!(), "agent-relay-#{System.unique_integer([:positive])}") + :ok = Hyper.Cfg.Toml.put_cache(Map.put(original, "work_dir", tmp)) + dir = Hyper.Cfg.Dirs.socket_dir() + File.mkdir_p!(dir) + + guest_path = Path.join(dir, "guest-#{@vm_id}.sock") + vsock_path = Path.join(dir, "vsock-#{@vm_id}.sock") + {:ok, _pid, _port} = GRPC.Server.start(Guest, 0, adapter_opts: [ip: {:local, guest_path}]) + + {:ok, fc} = :socket.open(:local, :stream) + :ok = :socket.bind(fc, %{family: :local, path: vsock_path}) + :ok = :socket.listen(fc, 64) + firecracker = spawn(fn -> fake_firecracker(fc, guest_path) end) + + {:ok, relay} = + Relay.start_link(%{ + vm_id: @vm_id, + vsock_uds: vsock_path, + listen_path: Agent.relay_socket_path(@vm_id) + }) + + true = Process.unlink(relay) + + on_exit(fn -> + GenServer.stop(relay) + Process.exit(firecracker, :kill) + GRPC.Server.stop(Guest) + Hyper.Cfg.Toml.put_cache(original) + File.rm_rf(tmp) + end) + + :ok + end + + defp fake_firecracker(listen, guest_path) do + with {:ok, vsock} <- :socket.accept(listen) do + splice(vsock, guest_path) + fake_firecracker(listen, guest_path) + end + end + + defp splice(vsock, guest_path) do + spawn(fn -> + {:ok, "CONNECT " <> _} = :socket.recv(vsock) + :ok = :socket.send(vsock, "OK 5\n") + {:ok, guest} = :socket.open(:local, :stream) + :ok = :socket.connect(guest, %{family: :local, path: guest_path}) + _ = spawn(fn -> pipe(guest, vsock) end) + pipe(vsock, guest) + end) + end + + defp pipe(from, to) do + with {:ok, data} <- :socket.recv(from), + :ok <- :socket.send(to, data) do + pipe(from, to) + else + _ -> + :socket.close(from) + :socket.close(to) + end + end + + test "an honest reply under the cap round-trips intact" do + for mib <- [1, 5, 15] do + bytes = Information.as_bytes(Information.mib(mib)) + + assert {:ok, %{exit_code: 0, stdout: stdout}} = + Agent.exec(@vm_id, ["/bin/head", "#{bytes}"], timeout: 20_000) + + assert {mib, byte_size(stdout), stdout == :binary.copy("y", bytes)} == {mib, bytes, true} + end + end + + test "a reply over the cap is a non-timeout error and the relay keeps serving" do + over = Information.as_bytes(Information.mib(17)) + + assert {:error, reason} = Agent.exec(@vm_id, ["/bin/head", "#{over}"], timeout: 20_000) + assert reason != :timeout + + assert {:ok, %{stdout: "still here"}} = + Agent.exec(@vm_id, ["/bin/echo", "still", "here"], timeout: 5_000) + end + + property "concurrent execs on one VM each get their own reply" do + check all( + words <- + uniq_list_of(string(:alphanumeric, min_length: 1), min_length: 2, max_length: 16), + max_runs: 10 + ) do + results = + words + |> Task.async_stream(&Agent.exec(@vm_id, ["/bin/echo", &1], timeout: 10_000), + max_concurrency: length(words), + timeout: 15_000 + ) + |> Enum.map(fn {:ok, {:ok, %{stdout: out}}} -> out end) + + assert results == words + end + end +end diff --git a/test/hyper/node/fire_vmm/jailer_socket_pin_properties_test.exs b/test/hyper/node/fire_vmm/jailer_socket_pin_properties_test.exs new file mode 100644 index 00000000..50b65b17 --- /dev/null +++ b/test/hyper/node/fire_vmm/jailer_socket_pin_properties_test.exs @@ -0,0 +1,115 @@ +defmodule Hyper.Node.FireVMM.JailerSocketPinPropertiesTest do + @moduledoc """ + The BEAM-side half of the socket-pin contract from #135. + + Laws: + + * wire contract with the helper: for every work dir and VM id, `host_socket/1` + and `host_vsock/1` are exactly the pins the helper's claim ops create, + `/socks/fc--api.socket` and `/socks/fc--vsock.sock` + (Rust: `Config::socket_dir` joined with `SocketPin::name`); + * the BEAM never dials a jail path: neither pin lies under `chroot_base/0`; + * naming is injective: no two `(id, socket)` pairs, nor a pin and a relay + socket, share a path, so one VM's teardown or claim can never hit another's; + * reclaim sweeps exactly what Hyper names: after a reboot every Firecracker pin + and relay socket for any set of VMs is removed, and every look-alike survives. + + `async: false` because the work dir is set through `Hyper.Cfg.Toml.put_cache/1`, + which is process-global. + """ + + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Node.FireVMM.{Agent, Jailer} + alias Hyper.Node.Reclaim + + setup do + original = Hyper.Cfg.Toml.reload() + on_exit(fn -> Hyper.Cfg.Toml.put_cache(original) end) + {:ok, original: original} + end + + defp vm_id do + StreamData.string(Enum.concat(?a..?z, ?2..?7), min_length: 1, max_length: 16) + |> StreamData.map(&("v" <> &1)) + end + + defp work_dir do + StreamData.string(:alphanumeric, min_length: 1, max_length: 12) + |> StreamData.list_of(min_length: 1, max_length: 4) + |> StreamData.map(&Path.join(["/", "srv-prop" | &1])) + end + + defp with_work_dir(original, dir), + do: Hyper.Cfg.Toml.put_cache(Map.put(original, "work_dir", dir)) + + property "the BEAM dials exactly the pins the helper creates, never a jail path", + %{original: original} do + check all(dir <- work_dir(), id <- vm_id()) do + with_work_dir(original, dir) + + assert {Jailer.host_socket(id), Jailer.host_vsock(id)} == + {"#{dir}/socks/fc-#{id}-api.socket", "#{dir}/socks/fc-#{id}-vsock.sock"} + + chroot = Hyper.Cfg.Dirs.chroot_base() <> "/" + refute String.starts_with?(Jailer.host_socket(id), chroot) + refute String.starts_with?(Jailer.host_vsock(id), chroot) + end + end + + property "no two VMs or socket kinds share a socket path", %{original: original} do + with_work_dir(original, "/srv/hyper") + + check all(ids <- StreamData.uniq_list_of(vm_id(), min_length: 2, max_length: 12)) do + paths = + Enum.flat_map(ids, fn id -> + [Jailer.host_socket(id), Jailer.host_vsock(id), Agent.relay_socket_path(id)] + end) + + assert length(Enum.uniq(paths)) == 3 * length(ids) + end + end + + property "reclaim removes every pin and relay socket Hyper names and spares look-alikes", + %{original: original} do + check all( + ids <- StreamData.uniq_list_of(vm_id(), min_length: 1, max_length: 6), + max_runs: 40 + ) do + tmp = Path.join(System.tmp_dir!(), "hyper-pin-#{System.unique_integer([:positive])}") + with_work_dir(original, tmp) + dir = Hyper.Cfg.Dirs.socket_dir() + File.mkdir_p!(dir) + + named = + Enum.flat_map(ids, fn id -> + [Jailer.host_socket(id), Jailer.host_vsock(id), Agent.relay_socket_path(id)] + end) + + look_alikes = + Enum.flat_map(ids, fn id -> + [ + "fc-#{id}-api.socket.bak", + "fc-#{id}-vsock.sock~", + "fc-#{id}", + "#{id}-api.socket", + "fc-#{id}-api.sock", + "grpc-#{id}.socket" + ] + end) + |> Enum.map(&Path.join(dir, &1)) + + for path <- named ++ look_alikes, do: File.write!(path, "") + + try do + Reclaim.reclaim_sockets() + + assert dir |> File.ls!() |> Enum.map(&Path.join(dir, &1)) |> Enum.sort() == + Enum.sort(look_alikes) + after + File.rm_rf!(tmp) + end + end + end +end diff --git a/test/hyper/node/fire_vmm/vm_linux/provider_trust_properties_test.exs b/test/hyper/node/fire_vmm/vm_linux/provider_trust_properties_test.exs new file mode 100644 index 00000000..a91b08f6 --- /dev/null +++ b/test/hyper/node/fire_vmm/vm_linux/provider_trust_properties_test.exs @@ -0,0 +1,138 @@ +defmodule Hyper.Node.FireVMM.VmLinux.ProviderTrustPropertiesTest do + @moduledoc """ + Kernel trust in `Hyper.Node.FireVMM.VmLinux.Provider`: a cached kernel is + only booted when no jail could have rewritten it. + + Laws: + + * `usable?/2` oracle: for every permission mode (setuid/setgid/sticky + included) and every expected owner, a regular file is usable exactly when + it carries no group- or other-write bit and is owned by root or that + owner; with no known owner (`nil`) only root qualifies; + * a non-regular path (directory, FIFO, dangling or live symlink) is never + usable, and judging it never blocks; + * `install_state/3` oracle: with each build independently faithful, stale, + missing, loosened, or replaced by a symlink to a faithful copy, the state + is `:ok` iff every build is faithful, `:not_installed` iff none is, and + `:bad_install` otherwise. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Node.FireVMM.VmLinux.{Manifest, Provider} + + setup do + dir = Path.join(System.tmp_dir!(), "vmlinux-trust-#{System.unique_integer([:positive])}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf!(dir) end) + {:ok, dir: dir, me: File.stat!(dir).uid} + end + + property "usable?/2 holds exactly for root- or owner-owned files no group or other can write", + %{dir: dir, me: me} do + path = Path.join(dir, "vmlinux") + File.write!(path, "kernel") + + check all( + mode <- integer(0..0o7777), + owner <- member_of([me, me + 1, 0, nil]), + max_runs: 400 + ) do + File.chmod!(path, mode) + %File.Stat{mode: actual, uid: uid} = File.stat!(path) + + expected = + Bitwise.band(actual, 0o022) == 0 and (uid == 0 or uid == owner) + + assert Provider.usable?(path, owner) == expected, + "mode #{Integer.to_string(actual, 8)}, owner #{inspect(owner)}" + end + end + + test "a non-regular path is never usable and judging it never blocks", %{dir: dir, me: me} do + trusted = Path.join(dir, "trusted") + File.write!(trusted, "kernel") + File.chmod!(trusted, 0o644) + + fifo = Path.join(dir, "fifo") + {_, 0} = System.cmd("mkfifo", ["-m", "0644", fifo]) + + subdir = Path.join(dir, "subdir") + File.mkdir_p!(subdir) + File.chmod!(subdir, 0o755) + + live = Path.join(dir, "live-link") + File.ln_s!(trusted, live) + dangling = Path.join(dir, "dangling-link") + File.ln_s!(Path.join(dir, "nowhere"), dangling) + + verdicts = + for {name, path} <- [fifo: fifo, directory: subdir, live_link: live, dangling: dangling] do + task = Task.async(fn -> Provider.usable?(path, me) end) + {name, Task.yield(task, 2_000) || Task.shutdown(task, :brutal_kill)} + end + + assert verdicts == [ + fifo: {:ok, false}, + directory: {:ok, false}, + live_link: {:ok, false}, + dangling: {:ok, false} + ] + end + + defp build_condition, do: member_of([:faithful, :stale, :missing, :loose, :symlinked]) + + property "install_state/3 is :ok iff every build is faithful and :not_installed iff none is", + %{dir: dir, me: me} do + builds = Manifest.builds_for(:x86_64) + assert length(builds) > 1 + + check all(conditions <- list_of(build_condition(), length: length(builds)), max_runs: 100) do + case_dir = Path.join(dir, "case-#{System.unique_integer([:positive])}") + File.mkdir_p!(case_dir) + staged = Enum.zip_with(builds, conditions, &stage(case_dir, &1, &2)) + faithful = Enum.count(conditions, &(&1 == :faithful)) + + expected = + cond do + faithful == length(builds) -> :ok + faithful == 0 -> {:error, :not_installed} + true -> {:error, :bad_install} + end + + assert Provider.install_state(case_dir, staged, me) == expected, inspect(conditions) + end + end + + defp stage(dir, build, condition) do + content = "kernel #{build.asset}" + sha = Base.encode16(:crypto.hash(:sha256, content), case: :lower) + path = Path.join(dir, build.asset) + + case condition do + :faithful -> + write(path, content, 0o644) + + :stale -> + write(path, "previous release #{build.asset}", 0o644) + + :missing -> + :ok + + :loose -> + write(path, content, 0o664) + + :symlinked -> + real = Path.join(dir, "real-#{build.asset}") + write(real, content, 0o644) + File.ln_s!(real, path) + end + + %{build | sha256: sha} + end + + defp write(path, content, mode) do + File.write!(path, content) + File.chmod!(path, mode) + end +end diff --git a/test/hyper/node/id_pool_properties_test.exs b/test/hyper/node/id_pool_properties_test.exs new file mode 100644 index 00000000..3ad7ad7e --- /dev/null +++ b/test/hyper/node/id_pool_properties_test.exs @@ -0,0 +1,93 @@ +defmodule Hyper.Node.IdPoolPropertiesTest do + @moduledoc """ + Laws for the two per-VM id pools, `Hyper.Node.Users` (uid/gid) and + `Hyper.Node.Net` (network slot), which share one bump-pointer + FIFO + free-queue design (#134). Each id is a VM identity: handing a held id to a + second VM is a uid / netns collision between two guests. + + Over any interleaving of claims, explicit releases, owner binds and owner + deaths by well-behaved callers (each held id returned exactly once): + + * no id is ever handed out while it is still held; + * every id handed out lies in the configured range; + * a claim is refused (`:exhausted`) exactly when every id in the range is + held — a returned id is always reusable, so nothing leaks; + * a returned id is not reissued while a never-issued id remains. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Node.{Net, Users} + + @pools [users: {Users, {:new}, :free}, net: {Net, :claim, :release}] + + defp op do + one_of([ + constant(:claim), + tuple({constant(:release), integer(0..1_000)}), + tuple({constant(:bind_and_die), integer(0..1_000)}) + ]) + end + + for {name, {mod, claim_msg, release_msg}} <- @pools do + property "#{name}: never reissues a held id, never leaks one, stays in range" do + mod = unquote(mod) + claim_msg = unquote(Macro.escape(claim_msg)) + release_tag = unquote(release_msg) + + check all( + min <- integer(0..60_000), + span <- integer(0..12), + ops <- list_of(op(), max_length: 80) + ) do + max = min + span + pool = new_pool(mod, min, max) + + Enum.reduce(ops, {pool, [], MapSet.new()}, fn + :claim, {pool, held, ever} -> + {:reply, reply, pool} = mod.handle_call(claim_msg, {self(), make_ref()}, pool) + + case reply do + {:ok, id} -> + refute id in held, "#{inspect(mod)} reissued held id #{id}" + assert id in min..max + fresh_left? = MapSet.size(ever) < span + 1 + + if fresh_left?, + do: refute(MapSet.member?(ever, id), "reused #{id} while fresh ids remained") + + {pool, held ++ [id], MapSet.put(ever, id)} + + {:error, :exhausted} -> + assert length(held) == span + 1, "refused with only #{length(held)} held" + {pool, held, ever} + end + + {:release, _}, {pool, [], ever} -> + {pool, [], ever} + + {:release, pick}, {pool, held, ever} -> + id = Enum.at(held, rem(pick, length(held))) + {:noreply, pool} = mod.handle_cast({release_tag, id}, pool) + {pool, List.delete(held, id), ever} + + {:bind_and_die, _}, {pool, [], ever} -> + {pool, [], ever} + + {:bind_and_die, pick}, {pool, held, ever} -> + id = Enum.at(held, rem(pick, length(held))) + owner = spawn(fn -> :ok end) + {:reply, :ok, pool} = mod.handle_call({:bind, id, owner}, {self(), make_ref()}, pool) + ref = pool.owners |> Enum.find(fn {_ref, bound} -> bound == id end) |> elem(0) + {:noreply, pool} = mod.handle_info({:DOWN, ref, :process, owner, :normal}, pool) + {pool, List.delete(held, id), ever} + end) + end + end + end + + defp new_pool(Users, min, max), do: %Users.State{max: max, next: min} + + defp new_pool(Net, min, max), + do: %Net.State{host: :unused_by_the_id_pool, max: max, next: min} +end diff --git a/test/hyper/node/try_run_contract_test.exs b/test/hyper/node/try_run_contract_test.exs new file mode 100644 index 00000000..bd07bf81 --- /dev/null +++ b/test/hyper/node/try_run_contract_test.exs @@ -0,0 +1,98 @@ +defmodule Hyper.Node.TryRunContractTest do + @moduledoc """ + `Hyper.Node.try_run/3`'s no-leak contract (#134), against this node's real + `Hyper.Node.Budget.Hard` ledger: every VM `start_fun` boots is either + admitted (and held against its pid) or torn down through `stop_fun` — never + both, never neither; concurrent placements never admit past the node's hard + headroom; and once the admitted VMs die the headroom returns exactly to + where it started, so a refusal or an admission leaves no reservation behind. + """ + use ExUnit.Case, async: false + + alias Hyper.Node.Budget.Hard + alias Hyper.Vm.Instance.Spec + alias Unit.{Bandwidth, Information} + + @moduletag :capture_log + @moduletag :integration + @racers 6 + + test "concurrent placements never over-admit, and every refused VM is stopped" do + before = Hard.headroom() + spec = spec(Information.bytes(div(Information.as_bytes(before.mem), 3) + 1)) + test = self() + + stop = fn pid -> + send(test, {:stopped, pid}) + Process.exit(pid, :kill) + :ok + end + + results = + 1..@racers + |> Enum.map(fn _ -> + Task.async(fn -> + vm = spawn(fn -> Process.sleep(:infinity) end) + {vm, Hyper.Node.try_run(spec, fn -> {:ok, vm} end, stop)} + end) + end) + |> Task.await_many() + + stopped = drain_stopped() + {admitted, refused} = Enum.split_with(results, &match?({_, {:ok, _}}, &1)) + + assert length(admitted) in 1..2, inspect(results) + assert Enum.map(refused, &elem(&1, 0)) |> Enum.sort() == Enum.sort(stopped) + assert Enum.all?(admitted, fn {vm, {:ok, pid}} -> vm == pid and Process.alive?(vm) end) + refute Enum.any?(refused, fn {vm, _} -> Process.alive?(vm) end) + + Enum.each(admitted, fn {vm, _} -> Process.exit(vm, :kill) end) + assert eventually(fn -> Hard.headroom() == before end), "a reservation outlived its VM" + end + + test "a VM that failed to boot is never stopped, and reserves nothing" do + before = Hard.headroom() + test = self() + + assert Hyper.Node.try_run( + spec(Information.mib(1)), + fn -> {:error, :kernel_missing} end, + fn pid -> send(test, {:stopped, pid}) end + ) == {:error, :kernel_missing} + + refute_received {:stopped, _} + assert Hard.headroom() == before + end + + defp drain_stopped(acc \\ []) do + receive do + {:stopped, pid} -> drain_stopped([pid | acc]) + after + 0 -> acc + end + end + + defp eventually(fun, tries \\ 100) do + cond do + fun.() -> + true + + tries == 0 -> + false + + true -> + Process.sleep(20) + eventually(fun, tries - 1) + end + end + + defp spec(mem) do + %Spec{ + vcpus: 0.01, + mem: mem, + disk: Information.bytes(1), + disk_bw: Bandwidth.zero(), + net_bw: Bandwidth.zero() + } + end +end diff --git a/test/hyper/suid_helper/wire_properties_test.exs b/test/hyper/suid_helper/wire_properties_test.exs new file mode 100644 index 00000000..e07ab156 --- /dev/null +++ b/test/hyper/suid_helper/wire_properties_test.exs @@ -0,0 +1,87 @@ +defmodule Hyper.SuidHelper.WirePropertiesTest do + @moduledoc """ + The node and the helper agree on the wire, checked against the real stamped helper binary + (the one `Hyper.SuidHelper.Expected` was captured from), not a fixture. + + Laws: + + * every VM id `Hyper.Vm.Id.generate/0` mints is a VM id the helper parses, for both socket + claims; + * every firewall `Hyper.Vm.Firewall.new/2` accepts, up to `max_rules/0` in each direction + and over the whole `1..65535` port space, becomes a `vm_firewall` request the helper + parses; + * a request the helper refuses to parse surfaces as `{:error, {2, "invalid request: ..."}}`, + never as success or as another error. + + The helper runs unprivileged and not setuid here: a request it parses gets past the caller + gate (the binary's group is the runner's) and then fails raising euid, before touching + anything. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.SuidHelper + alias Hyper.Vm.Firewall + + @helper "native/target/*-unknown-linux-musl/release/hyper-suidhelper" + |> Path.wildcard() + |> List.first() + + @moduletag skip: if(@helper, do: false, else: "no stamped helper; run mix compile") + + setup do + previous = Application.get_env(:hyper, Hyper.Cfg.Tools, []) + + Application.put_env( + :hyper, + Hyper.Cfg.Tools, + Keyword.put(previous, :suidhelper, Path.expand(@helper)) + ) + + on_exit(fn -> Application.put_env(:hyper, Hyper.Cfg.Tools, previous) end) + end + + defp parsed?({:error, {2, "invalid request" <> _}}), do: false + defp parsed?({:error, {2, _message}}), do: true + + defp rule do + gen all( + protocol <- member_of([:tcp, :udp]), + first <- one_of([constant(1), constant(65_535), integer(1..65_535)]), + last <- one_of([constant(first), constant(65_535), integer(first..65_535)]) + ) do + {protocol, first..last//1} + end + end + + defp rules do + one_of([ + list_of(rule(), max_length: 8), + list_of(rule(), length: Firewall.max_rules()) + ]) + end + + property "every minted VM id is one the helper parses" do + check all(id <- repeatedly(&Hyper.Vm.Id.generate/0), max_runs: 25) do + for claim <- [&SuidHelper.Vm.claim_api_socket/1, &SuidHelper.Vm.claim_vsock_socket/1] do + result = claim.(id) + assert parsed?(result), "#{id}: #{inspect(result)}" + end + end + end + + property "every firewall the node accepts is one the helper parses" do + check all(inbound <- rules(), outbound <- rules(), max_runs: 25) do + {:ok, firewall} = Firewall.new(inbound, outbound) + result = SuidHelper.Vm.firewall(Hyper.Vm.Id.generate(), firewall) + assert parsed?(result), inspect(result) + end + end + + test "a request the helper cannot parse is an invalid-request error, not success" do + for id <- ["v_1", "-v1", "", String.duplicate("a", 65), "v1/../x", "vé"] do + assert {:error, {2, "invalid request" <> _}} = SuidHelper.Vm.claim_api_socket(id), + inspect(id) + end + end +end diff --git a/test/mix/tasks/firecracker.install_test.exs b/test/mix/tasks/firecracker.install_test.exs new file mode 100644 index 00000000..8bd9ea3f --- /dev/null +++ b/test/mix/tasks/firecracker.install_test.exs @@ -0,0 +1,72 @@ +defmodule Mix.Tasks.Firecracker.InstallTest do + @moduledoc """ + `mix firecracker.install` against the real pinned release. + + Laws: + + * whatever already sits at `/firecracker` or `/jailer` - a + symlink or a hard link to someone else's file - is replaced, never + written through: the file it named keeps its bytes; + * each installed binary is a fresh regular file, mode `0755`, that runs as + the pinned version; + * reinstalling over a finished install succeeds and yields the same bytes. + """ + use ExUnit.Case, async: false + + @moduletag :external + @moduletag timeout: 300_000 + + setup do + previous = Mix.shell() + Mix.shell(Mix.Shell.Process) + prefix = Path.join(System.tmp_dir!(), "fc-install-#{System.unique_integer([:positive])}") + File.mkdir_p!(prefix) + + on_exit(fn -> + Mix.shell(previous) + File.rm_rf!(prefix) + end) + + {:ok, prefix: prefix} + end + + defp install(prefix), do: Mix.Tasks.Firecracker.Install.run(["--prefix", prefix]) + + defp assert_fresh_binary(path, victims) do + stat = File.lstat!(path) + assert stat.type == :regular + assert Bitwise.band(stat.mode, 0o7777) == 0o755 + refute stat.inode in Enum.map(victims, &File.stat!(&1).inode) + {out, 0} = System.cmd(path, ["--version"], stderr_to_stdout: true) + assert out =~ "v1.16.0" + end + + test "planted links at the install paths are replaced, never written through", %{ + prefix: prefix + } do + sym_victim = Path.join(prefix, "sym-victim") + hard_victim = Path.join(prefix, "hard-victim") + File.write!(sym_victim, "operator data") + File.write!(hard_victim, "other data") + File.ln_s!(sym_victim, Path.join(prefix, "firecracker")) + File.ln!(hard_victim, Path.join(prefix, "jailer")) + + install(prefix) + + assert File.read!(sym_victim) == "operator data" + assert File.read!(hard_victim) == "other data" + assert_fresh_binary(Path.join(prefix, "firecracker"), [sym_victim, hard_victim]) + assert_fresh_binary(Path.join(prefix, "jailer"), [sym_victim, hard_victim]) + end + + test "reinstalling over a finished install succeeds with the same bytes", %{prefix: prefix} do + install(prefix) + first = for b <- ~w(firecracker jailer), do: File.read!(Path.join(prefix, b)) + + install(prefix) + second = for b <- ~w(firecracker jailer), do: File.read!(Path.join(prefix, b)) + + assert :crypto.hash(:sha256, first) == :crypto.hash(:sha256, second) + for b <- ~w(firecracker jailer), do: assert_fresh_binary(Path.join(prefix, b), []) + end +end diff --git a/test/redist/targz_replace_test.exs b/test/redist/targz_replace_test.exs new file mode 100644 index 00000000..fda49d16 --- /dev/null +++ b/test/redist/targz_replace_test.exs @@ -0,0 +1,95 @@ +defmodule Redist.TargzReplaceTest do + @moduledoc """ + `Redist.Targz.install/3` into a `dest_dir` that already holds files - the + reinstall path `mix firecracker.install` relies on to never keep an old or + planted binary. + + Laws: + + * the archive wins: every entry lands with the archive's bytes, whatever + regular file previously sat at that path (stale release or plant); + * a planted symlink at an entry path is never written through - the + file it points at is unchanged, whether the install succeeds or refuses; + * an archive symlink cannot be used to write outside `dest_dir`. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Redist.Support.HttpServer + alias Redist.Targz + + setup do + {:ok, _} = Application.ensure_all_started(:req) + tmp = Path.join(System.tmp_dir!(), "targz-replace-#{System.unique_integer([:positive])}") + File.mkdir_p!(tmp) + on_exit(fn -> File.rm_rf!(tmp) end) + {:ok, server: HttpServer.start(), tmp: tmp} + end + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) + + defp targz(tmp, entries) do + path = Path.join(tmp, "build-#{System.unique_integer([:positive])}.tar.gz") + :ok = :erl_tar.create(String.to_charlist(path), entries, [:compressed]) + bytes = File.read!(path) + File.rm!(path) + bytes + end + + defp install(server, bytes, dest) do + url = HttpServer.put_file(server, "a-#{System.unique_integer([:positive])}.tar.gz", bytes) + Targz.install(url, sha256(bytes), dest) + end + + property "every entry lands with the archive's bytes over whatever file was there", + %{server: server, tmp: tmp} do + check all( + fresh <- binary(min_length: 1, max_length: 512), + stale <- binary(max_length: 512), + max_runs: 25 + ) do + dest = Path.join(tmp, "dest-#{System.unique_integer([:positive])}") + File.mkdir_p!(Path.join(dest, "release")) + File.write!(Path.join(dest, "release/firecracker"), stale) + + bytes = targz(tmp, [{~c"release/firecracker", fresh}]) + + assert install(server, bytes, dest) == :ok + assert File.read!(Path.join(dest, "release/firecracker")) == fresh + end + end + + test "a planted symlink at an entry path is never written through", %{ + server: server, + tmp: tmp + } do + victim = Path.join(tmp, "victim") + File.write!(victim, "operator data") + dest = Path.join(tmp, "dest") + File.mkdir_p!(dest) + File.ln_s!(victim, Path.join(dest, "firecracker")) + + _ = install(server, targz(tmp, [{~c"firecracker", "archive bytes"}]), dest) + + assert File.read!(victim) == "operator data" + end + + test "an archive symlink cannot be followed out of dest_dir", %{server: server, tmp: tmp} do + outside = Path.join(tmp, "outside") + File.mkdir_p!(outside) + src = Path.join(tmp, "src") + File.mkdir_p!(src) + File.ln_s!(outside, Path.join(src, "link")) + + archive = Path.join(tmp, "evil.tar.gz") + {:ok, tar} = :erl_tar.open(String.to_charlist(archive), [:write, :compressed]) + :ok = :erl_tar.add(tar, String.to_charlist(Path.join(src, "link")), ~c"link", []) + :ok = :erl_tar.add(tar, "pwned", ~c"link/escape.txt", []) + :ok = :erl_tar.close(tar) + + got = install(server, File.read!(archive), Path.join(tmp, "dest")) + + assert match?({:error, _}, got) + assert File.ls!(outside) == [] + end +end diff --git a/test/sys/tmp_remove_properties_test.exs b/test/sys/tmp_remove_properties_test.exs new file mode 100644 index 00000000..91ddb7cf --- /dev/null +++ b/test/sys/tmp_remove_properties_test.exs @@ -0,0 +1,136 @@ +defmodule Sys.TmpRemovePropertiesTest do + @moduledoc """ + Cleanup laws of `Sys.Tmp.with_tempdir/2` against trees an unpacked image can + leave behind (nested dirs of any mode down to 0000, files of any mode, and + symlinks pointing out of the tree at dirs and files with restrictive modes): + + * the temp dir is always gone afterwards, on return and on raise; + * nothing reachable only through a symlink is removed, and no symlink + target outside the tree has its mode changed (cleanup makes the tree + traversable without ever chmodding through a link); + * if `fun` swaps the temp dir itself for a symlink to an outside dir, + cleanup removes the link and leaves the outside dir and its contents + as they were. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + import Bitwise + + alias Sys.Tmp + + @moduletag :tmp_dir + + @dir_modes [0o000, 0o100, 0o200, 0o300, 0o400, 0o500, 0o555, 0o700] + @file_modes [0o000, 0o400, 0o644, 0o4755] + + defp tree do + leaf = + StreamData.one_of([ + StreamData.tuple({StreamData.constant(:file), StreamData.member_of(@file_modes)}), + StreamData.member_of([{:link, :outside_dir}, {:link, :outside_file}, {:link, :dangling}]) + ]) + + StreamData.tree(leaf, fn child -> + StreamData.tuple( + {StreamData.constant(:dir), StreamData.member_of(@dir_modes), + StreamData.list_of(child, max_length: 3)} + ) + end) + end + + defp outside(tmp) do + root = Path.join(tmp, "o#{System.unique_integer([:positive])}") + dir = Path.join(root, "dir") + File.mkdir_p!(dir) + File.write!(Path.join(dir, "keep"), "keep") + file = Path.join(root, "file") + File.write!(file, "outside") + File.chmod!(file, 0o400) + File.chmod!(dir, 0o500) + %{root: root, dir: dir, file: file} + end + + defp build!({:file, mode}, path, _out) do + File.write!(path, "x") + File.chmod!(path, mode) + end + + defp build!({:link, :outside_dir}, path, out), do: File.ln_s!(out.dir, path) + defp build!({:link, :outside_file}, path, out), do: File.ln_s!(out.file, path) + defp build!({:link, :dangling}, path, out), do: File.ln_s!(Path.join(out.root, "nope"), path) + + defp build!({:dir, mode, children}, path, out) do + File.mkdir!(path) + + children + |> Enum.with_index() + |> Enum.each(fn {child, i} -> build!(child, Path.join(path, "n#{i}"), out) end) + + File.chmod!(path, mode) + end + + defp assert_outside_intact(out) do + assert band(File.lstat!(out.dir).mode, 0o7777) == 0o500 + assert band(File.lstat!(out.file).mode, 0o7777) == 0o400 + assert File.read!(Path.join(out.dir, "keep")) == "keep" + assert File.read!(out.file) == "outside" + refute File.exists?(Path.join(out.root, "nope")) + end + + defp release(out) do + File.chmod!(out.dir, 0o700) + File.chmod!(out.file, 0o600) + end + + property "any image-shaped tree is removed and nothing behind a symlink is touched", %{ + tmp_dir: tmp + } do + check all(shape <- tree(), raise? <- StreamData.boolean(), max_runs: 150) do + out = outside(tmp) + + work = fn dir -> + build!(shape, Path.join(dir, "rootfs"), out) + if raise?, do: raise("boom"), else: dir + end + + dir = + if raise? do + holder = self() + + assert_raise RuntimeError, fn -> + Tmp.with_tempdir("tmp-prop", fn dir -> + send(holder, {:dir, dir}) + work.(dir) + end) + end + + assert_receive {:dir, dir} + dir + else + Tmp.with_tempdir("tmp-prop", work) + end + + assert {:error, :enoent} = File.lstat(dir), inspect(shape) + assert_outside_intact(out) + release(out) + end + end + + test "a temp dir swapped for a symlink to an outside dir is unlinked, not followed", %{ + tmp_dir: tmp + } do + out = outside(tmp) + + dir = + Tmp.with_tempdir("tmp-prop", fn dir -> + File.rmdir!(dir) + File.ln_s!(out.dir, dir) + dir + end) + + assert {:error, :enoent} = File.lstat(dir) + assert_outside_intact(out) + release(out) + end +end