diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh index 326e3caa..310791c0 100755 --- a/.github/scripts/provision-kvm-host.sh +++ b/.github/scripts/provision-kvm-host.sh @@ -138,12 +138,16 @@ sudo chown "$(id -u):$(id -g)" /srv/hyper mkdir -p /srv/hyper/layers # firecracker.install writes into /opt/firecracker as the invoking user, then -# the helper requires both binaries root-owned and not group/world-writable. +# the helper requires both binaries root-owned and not group/world-writable, +# and every parent directory too: the jailer is handed firecracker by path. +# The hosted runner image leaves /opt itself writable beyond root. +sudo chown root:root /opt +sudo chmod go-w /opt sudo mkdir -p /opt/firecracker sudo chown "$(id -u)" /opt/firecracker mix firecracker.install -sudo chown root:root /opt/firecracker/firecracker /opt/firecracker/jailer -sudo chmod 0755 /opt/firecracker/firecracker /opt/firecracker/jailer +sudo chown root:root /opt/firecracker /opt/firecracker/firecracker /opt/firecracker/jailer +sudo chmod 0755 /opt/firecracker /opt/firecracker/firecracker /opt/firecracker/jailer # With passwordless sudo the task installs setuid-root itself, for the runner's group. mix suidhelper.install --group "$(id -gn)" diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index 178b7cbb..a1b3377d 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -428,6 +428,8 @@ config :hyper, Hyper.Cfg.Cluster, - Must be owned by `root`. - Must be only writable by `root`. - Must not be a symlink. + - For `firecracker` only: every directory above it must also be owned by + `root` and writable only by `root`, since the jailer is handed it by path. #### Range diff --git a/docs/cookbook/install.md b/docs/cookbook/install.md index 5e5f611f..24da7746 100644 --- a/docs/cookbook/install.md +++ b/docs/cookbook/install.md @@ -368,9 +368,20 @@ mix firecracker.install # installs to /opt/firecracker The task installs the binaries under their bare basenames (`firecracker`, `jailer` – the setuid helper rejects version-stamped names), marks them executable, and prints the `[tools]` snippet for `/etc/hyper/config.toml`. -After installing, make both binaries root-owned and not group- or -world-writable (the task prints the exact `chown` command); the helper refuses -them otherwise. +After installing, make both binaries **and their directory** root-owned and +not group- or world-writable (the task prints the exact commands); the helper +refuses them otherwise: + +```sh +sudo chown root:root /opt/firecracker /opt/firecracker/firecracker /opt/firecracker/jailer +sudo chmod 0755 /opt/firecracker /opt/firecracker/firecracker /opt/firecracker/jailer +``` + +Every parent directory (`/opt` included) must be root-owned and not group- or +world-writable too. + +Hosts installed before 0.2 left `/opt/firecracker` owned by the node user; +every VM launch there fails until the directory is chowned as above. ### SUID Helper diff --git a/lib/hyper/firecracker/api/transport.ex b/lib/hyper/firecracker/api/transport.ex index 34291a73..dcf4d83e 100644 --- a/lib/hyper/firecracker/api/transport.ex +++ b/lib/hyper/firecracker/api/transport.ex @@ -2,8 +2,10 @@ defmodule Hyper.Firecracker.Api.Transport do @moduledoc """ `oapi_generator` client backend for the Firecracker API. Each generated operation calls `request/1` with a plain map describing the call; this issues - it over the target daemon's HTTP-over-Unix-socket API via `Req` and decodes the - typed response. + it over the target daemon's HTTP-over-Unix-socket API on a raw `:gen_tcp` + socket and decodes the typed response. The daemon is untrusted: the status + line, headers and body are size-capped, and the whole call shares one 30 s + deadline. The per-VM socket path arrives as `opts[:socket_path]` (injected by `Hyper.Node.FireVMM.Client`). Request bodies are generated structs encoded by @@ -12,8 +14,11 @@ defmodule Hyper.Firecracker.Api.Transport do own generated `decode/1` (also from `Hyper.Firecracker.Api.Codec`). Returns: `:ok` (204 / empty), `{:ok, decoded}` (2xx with a typed body), - `{:error, {:api, status, fault_message}}` (non-2xx), or - `{:error, {:transport, reason}}` (socket/connection failure). + `{:error, {:api, status, fault_message}}` (non-2xx), + `{:error, {:unexpected_body, status}}` (a typed reply that is not a JSON + object), `{:error, {:body_too_large, max_bytes}}`, or + `{:error, {:transport, reason}}` (socket failure, deadline, or an oversized + head). """ use OpenTelemetryDecorator @@ -22,60 +27,252 @@ defmodule Hyper.Firecracker.Api.Transport do @max_body_bytes Information.as_bytes(Information.mib(1)) + # A hostile daemon controls every byte of the reply, so the head is bounded + # as tightly as the body: the length of each status, header, chunk-size and + # trailer line, and how many of them there may be. + @max_line_bytes Information.as_bytes(Information.kib(8)) + @max_headers 100 + + # One deadline for the whole call, below `Hyper.Node.FireVMM.Client`'s 35 s + # call cap, so a daemon drip-feeding bytes cannot keep the client busy. + @call_budget_ms Unit.Time.as_ms(Unit.Time.s(30)) + @type result :: :ok | {:ok, term()} | {:error, {:api, pos_integer(), String.t() | nil} | {:transport, term()} - | {:body_too_large, pos_integer()}} + | {:body_too_large, pos_integer()} + | {:unexpected_body, pos_integer()}} @spec request(map()) :: result() @decorate with_span("Hyper.Firecracker.Api.Transport.request", include: [:method, :url]) def request(%{method: method, url: url, opts: opts} = info) do - socket_path = Keyword.fetch!(opts, :socket_path) + deadline = System.monotonic_time(:millisecond) + @call_budget_ms - req = - Req.new( - base_url: "http://localhost", - unix_socket: socket_path, - retry: false, - receive_timeout: 30_000 - ) + case call(Keyword.fetch!(opts, :socket_path), encode(method, url, info), deadline) do + {:ok, status, headers, body} -> + handle(status, decode(headers, body), Map.get(info, :response, [])) - req_opts = [method: method, url: url, into: &collect_capped/2] + {:error, :body_too_large} -> + {:error, {:body_too_large, @max_body_bytes}} - req_opts = - if Map.has_key?(info, :body), do: Keyword.put(req_opts, :json, info.body), else: req_opts + {:error, reason} -> + {:error, {:transport, reason}} + end + end - req_opts = if q = info[:query], do: Keyword.put(req_opts, :params, q), else: req_opts + defp encode(method, url, info) do + target = if q = info[:query], do: url <> "?" <> URI.encode_query(q), else: url - case Req.request(req, req_opts) do - {:ok, %Req.Response{status: status, body: body} = response} -> - if Req.Response.get_private(response, :body_too_large), - do: {:error, {:body_too_large, @max_body_bytes}}, - else: handle(status, body, Map.get(info, :response, [])) + {headers, body} = + case Map.fetch(info, :body) do + {:ok, body} -> + json = Jason.encode_to_iodata!(body) - {:error, reason} -> - {:error, {:transport, reason}} + {[ + "content-type: application/json\r\naccept: application/json\r\n", + "content-length: #{IO.iodata_length(json)}\r\n" + ], json} + + :error -> + {[], []} + end + + [ + method |> Atom.to_string() |> String.upcase(), + " ", + target, + " HTTP/1.1\r\nhost: localhost\r\n", + headers, + "\r\n", + body + ] + end + + defp call(socket_path, request, deadline) do + opts = [:binary, active: false, packet: :raw, send_timeout: @call_budget_ms] + + with {:ok, sock} <- :gen_tcp.connect({:local, socket_path}, 0, opts, remaining(deadline)) do + try do + with :ok <- :gen_tcp.send(sock, request), + {:ok, status, headers} <- read_head(sock, deadline), + {:ok, body} <- read_body(sock, status, headers, deadline) do + {:ok, status, headers, body} + end + after + :gen_tcp.close(sock) + end + end + end + + defp read_head(sock, deadline) do + :ok = :inet.setopts(sock, packet: :http_bin, packet_size: @max_line_bytes) + + case recv(sock, 0, deadline) do + {:ok, {:http_response, _version, status, _reason}} -> + read_headers(sock, status, %{}, 0, deadline) + + {:ok, _other} -> + {:error, :malformed_status_line} + + {:error, :emsgsize} -> + {:error, :line_too_long} + + {:error, _} = err -> + err + end + end + + defp read_headers(_sock, _status, _headers, count, _deadline) when count > @max_headers, + do: {:error, :too_many_headers} + + defp read_headers(sock, status, headers, count, deadline) do + case recv(sock, 0, deadline) do + {:ok, {:http_header, _, name, _, value}} -> + headers = Map.put(headers, header_name(name), value) + read_headers(sock, status, headers, count + 1, deadline) + + {:ok, :http_eoh} -> + {:ok, status, headers} + + {:ok, _other} -> + {:error, :malformed_header} + + {:error, :emsgsize} -> + {:error, :line_too_long} + + {:error, _} = err -> + err + end + end + + defp header_name(name) when is_atom(name), do: name |> Atom.to_string() |> String.downcase() + defp header_name(name), do: String.downcase(name) + + defp read_body(_sock, status, _headers, _deadline) + when status in 100..199 or status in [204, 304], + do: {:ok, ""} + + defp read_body(sock, _status, headers, deadline) do + :ok = :inet.setopts(sock, packet: :raw) + + cond do + chunked?(headers) -> read_chunks(sock, [], 0, deadline) + length = headers["content-length"] -> read_sized(sock, length, deadline) + true -> read_to_close(sock, [], 0, deadline) end end - @spec collect_capped({:data, binary()}, {Req.Request.t(), Req.Response.t()}) :: - {:cont | :halt, {Req.Request.t(), Req.Response.t()}} - defp collect_capped({:data, data}, {req, %Req.Response{body: body} = resp}) do - if byte_size(body) + byte_size(data) > @max_body_bytes do - {:halt, {req, %{Req.Response.put_private(resp, :body_too_large, true) | body: ""}}} + defp chunked?(%{"transfer-encoding" => coding}), + do: coding |> String.downcase() |> String.contains?("chunked") + + defp chunked?(_headers), do: false + + defp read_sized(sock, length, deadline) do + case Integer.parse(length) do + {0, ""} -> {:ok, ""} + {n, ""} when n > @max_body_bytes -> {:error, :body_too_large} + {n, ""} when n > 0 -> recv(sock, n, deadline) + _ -> {:error, :bad_content_length} + end + end + + defp read_to_close(sock, acc, size, deadline) do + case recv(sock, 0, deadline) do + {:ok, data} when size + byte_size(data) > @max_body_bytes -> {:error, :body_too_large} + {:ok, data} -> read_to_close(sock, [acc, data], size + byte_size(data), deadline) + {:error, :closed} -> {:ok, IO.iodata_to_binary(acc)} + {:error, _} = err -> err + end + end + + defp read_chunks(sock, acc, size, deadline) do + with {:ok, line} <- read_line(sock, deadline), + {:ok, chunk_size} <- chunk_size(line) do + cond do + chunk_size == 0 -> + with :ok <- skip_trailers(sock, 0, deadline), do: {:ok, IO.iodata_to_binary(acc)} + + size + chunk_size > @max_body_bytes -> + {:error, :body_too_large} + + true -> + with {:ok, chunk} <- recv(sock, chunk_size, deadline), + :ok <- expect_crlf(sock, deadline) do + read_chunks(sock, [acc, chunk], size + chunk_size, deadline) + end + end + end + end + + defp chunk_size(line) do + [hex | _extensions] = String.split(line, ";", parts: 2) + + case Integer.parse(String.trim(hex), 16) do + {n, ""} when n >= 0 -> {:ok, n} + _ -> {:error, :malformed_chunk} + end + end + + defp expect_crlf(sock, deadline) do + case recv(sock, 2, deadline) do + {:ok, "\r\n"} -> :ok + {:ok, _other} -> {:error, :malformed_chunk} + {:error, _} = err -> err + end + end + + defp skip_trailers(_sock, count, _deadline) when count > @max_headers, + do: {:error, :too_many_headers} + + defp skip_trailers(sock, count, deadline) do + case read_line(sock, deadline) do + {:ok, "\r\n"} -> :ok + {:ok, _trailer} -> skip_trailers(sock, count + 1, deadline) + {:error, _} = err -> err + end + end + + defp read_line(sock, deadline) do + :ok = :inet.setopts(sock, packet: :line, packet_size: @max_line_bytes) + line = recv(sock, 0, deadline) + :ok = :inet.setopts(sock, packet: :raw) + + case line do + {:error, :emsgsize} -> {:error, :line_too_long} + other -> other + end + end + + defp recv(sock, length, deadline), do: :gen_tcp.recv(sock, length, remaining(deadline)) + + defp remaining(deadline), do: max(deadline - System.monotonic_time(:millisecond), 0) + + @spec decode(%{String.t() => binary()}, binary()) :: term() + defp decode(headers, body) do + with true <- body != "" and json?(headers), + {:ok, decoded} <- Jason.decode(body) do + decoded else - {:cont, {req, %{resp | body: body <> data}}} + _ -> body end end + defp json?(%{"content-type" => type}) do + [media | _params] = String.split(type, ";", parts: 2) + String.downcase(String.trim(media)) == "application/json" + end + + defp json?(_headers), do: false + @spec handle(pos_integer(), term(), [{pos_integer(), term()}]) :: result() defp handle(status, body, responses) when status in 200..299 do case List.keyfind(responses, status, 0) do {^status, :null} -> :ok - {^status, {module, :t}} -> {:ok, module.decode(body)} + {^status, {module, :t}} when is_map(body) -> {:ok, module.decode(body)} + {^status, {_module, :t}} -> {:error, {:unexpected_body, status}} _other -> if body in [nil, ""], do: :ok, else: {:ok, body} end end diff --git a/lib/hyper/node/vmlinux.ex b/lib/hyper/node/vmlinux.ex index 1444da6e..74728ee7 100644 --- a/lib/hyper/node/vmlinux.ex +++ b/lib/hyper/node/vmlinux.ex @@ -60,8 +60,23 @@ defmodule Hyper.Node.Vmlinux do defp present(path) do cond do not File.regular?(path) -> {:error, {:vmlinux_missing, path}} - not Provider.usable?(path) -> {:error, {:vmlinux_untrusted, path}} + not Provider.usable?(resolve(path)) -> {:error, {:vmlinux_untrusted, path}} true -> :ok end end + + # The helper canonicalizes the kernel path before vetting it, so a symlink is + # judged by its target here too. Past the hop bound the link itself is + # returned, which `Provider.usable?/1` refuses. + @max_link_hops 40 + + defp resolve(path, hops \\ @max_link_hops) + defp resolve(path, 0), do: path + + defp resolve(path, hops) do + case :file.read_link_all(path) do + {:ok, target} -> resolve(Path.expand(to_string(target), Path.dirname(path)), hops - 1) + {:error, _} -> path + end + end end diff --git a/lib/mix/tasks/firecracker.install.ex b/lib/mix/tasks/firecracker.install.ex index 4fba41dd..80c2a13b 100644 --- a/lib/mix/tasks/firecracker.install.ex +++ b/lib/mix/tasks/firecracker.install.ex @@ -145,10 +145,11 @@ defmodule Mix.Tasks.Firecracker.Install do Mix.shell().info(""" Almost done. Run these as root so the setuid helper will accept the binaries - (it refuses any jailer/firecracker not owned by root): + (it refuses any jailer/firecracker not owned by root, or in a directory + anyone but root can write): - sudo chown root:root #{fc} #{jailer} - sudo chmod 0755 #{fc} #{jailer} + sudo chown root:root #{prefix} #{fc} #{jailer} + sudo chmod 0755 #{prefix} #{fc} #{jailer} Then add to /etc/hyper/config.toml (file: root-owned, mode 0644): diff --git a/lib/redist/targz.ex b/lib/redist/targz.ex index 546a4788..44f81531 100644 --- a/lib/redist/targz.ex +++ b/lib/redist/targz.ex @@ -45,15 +45,32 @@ defmodule Redist.Targz do end defp extract(tar_path, dest_dir) do - File.mkdir_p!(dest_dir) + tar = String.to_charlist(tar_path) - case :erl_tar.extract( - String.to_charlist(tar_path), - [:compressed, {:cwd, String.to_charlist(dest_dir)}] - ) do - :ok -> :ok - {:error, {name, :unsafe_path}} -> {:error, {:unsafe_tar_entry, to_string(name)}} - {:error, reason} -> {:error, {:extract_failed, reason}} + with :ok <- refuse_symlinks(tar) do + File.mkdir_p!(dest_dir) + + case :erl_tar.extract(tar, [:compressed, {:cwd, String.to_charlist(dest_dir)}]) do + :ok -> :ok + {:error, {name, :unsafe_path}} -> {:error, {:unsafe_tar_entry, to_string(name)}} + {:error, reason} -> {:error, {:extract_failed, reason}} + end + end + end + + # A chain of individually in-tree links (`s -> .`, `t -> s/..`) followed by + # an entry written through it spins `:erl_tar.extract/2` forever. No release + # this installs ships a symlink, so none is accepted. + defp refuse_symlinks(tar) do + case :erl_tar.table(tar, [:compressed, :verbose]) do + {:ok, entries} -> + case Enum.find(entries, &(elem(&1, 1) == :symlink)) do + nil -> :ok + entry -> {:error, {:unsafe_tar_entry, to_string(elem(entry, 0))}} + end + + {:error, reason} -> + {:error, {:extract_failed, reason}} end end diff --git a/native/.cargo/mutants.toml b/native/.cargo/mutants.toml index 0aa0d229..29e89e7d 100644 --- a/native/.cargo/mutants.toml +++ b/native/.cargo/mutants.toml @@ -43,6 +43,8 @@ exclude_re = [ # Equivalent: the operands never share a bit (no OpenFlags carries O_EXCL without O_CREAT, # nor O_DIRECTORY or O_CLOEXEC; the RESOLVE_* flags are distinct bits). "hyperrs-fs/src/(dir|file)\\.rs:.*replace \\| with \\^ in SafeFile<.*>::(openat|open_at|open_raw)$", + # Equivalent: S_ISUID and S_ISGID share no bits. + "replace \\| with \\^ in KernelImage::is_set_id$", # Equivalent: the dst-nat chain runs after conntrack and before routing at -100 or 100. "suidhelper/src/net/mod\\.rs:\\d+:\\d+: delete -$", ] diff --git a/native/hyperrs-firecracker/Cargo.toml b/native/hyperrs-firecracker/Cargo.toml index 61383f03..2763e61b 100644 --- a/native/hyperrs-firecracker/Cargo.toml +++ b/native/hyperrs-firecracker/Cargo.toml @@ -27,3 +27,7 @@ pretty_assertions = "1" proptest = "1" rstest = "0.28" test-strategy = "0.4" + +[[test]] +name = "kernel_trust" +path = "tests/kernel_trust.rs" diff --git a/native/hyperrs-firecracker/src/jailer/chroot.rs b/native/hyperrs-firecracker/src/jailer/chroot.rs index bc906dc6..ecc182d7 100644 --- a/native/hyperrs-firecracker/src/jailer/chroot.rs +++ b/native/hyperrs-firecracker/src/jailer/chroot.rs @@ -113,8 +113,9 @@ impl KernelImage { source, })?; let real: SafePathBuf = real.try_into()?; - let file: SafeFile = - SafeFile::open(&real, OpenFlags::RDONLY).map_err(|source| KernelError::Open { + let file = SafeFile::::open(&real, OpenFlags::PATH) + .and_then(|vetted| vetted.reopen(OpenFlags::RDONLY)) + .map_err(|source| KernelError::Open { path: real.as_path().to_path_buf(), source, })?; @@ -132,6 +133,11 @@ impl KernelImage { } Ok(Self(file)) } + + fn is_set_id(&self) -> nix::Result { + let set_id = Mode::S_ISUID | Mode::S_ISGID; + Ok(fstat(self.0.as_raw_fd())?.st_mode & set_id.bits() != 0) + } } /// The overal layout of the jail path. See [`JailLayout::new`]. @@ -239,19 +245,23 @@ impl JailRoot { /// Put the kernel image in the chroot jail. pub fn stage_kernel(&self, kernel: &KernelImage) -> Result<(), dir::DirError> { - self.0 - .linkat(&kernel.0, KERNEL_NAME) - .or_else(|errno| -> Result<(), dir::DirError> { - if errno != Errno::EXDEV { - return Err(errno.into()); - } - - let dest = self.0.mkfileat(KERNEL_NAME, KERNEL_MODE)?; - let mut src = File::from(kernel.0.as_fd().try_clone_to_owned()?); - src.rewind()?; - io::copy(&mut src, &mut File::from(OwnedFd::from(dest)))?; - Ok(()) - }) + // A root-made hard link would hand the jail a set-id inode that `protected_hardlinks` + // refuses unprivileged users; the copy is written at `KERNEL_MODE` instead. + if kernel.is_set_id()? { + return self.copy_kernel(kernel); + } + match self.0.linkat(&kernel.0, KERNEL_NAME) { + Err(Errno::EXDEV) => self.copy_kernel(kernel), + linked => Ok(linked?), + } + } + + fn copy_kernel(&self, kernel: &KernelImage) -> Result<(), dir::DirError> { + let dest = self.0.mkfileat(KERNEL_NAME, KERNEL_MODE)?; + let mut src = File::from(kernel.0.as_fd().try_clone_to_owned()?); + src.rewind()?; + io::copy(&mut src, &mut File::from(OwnedFd::from(dest)))?; + Ok(()) } /// Add the rootfs block device into the chroot jail. @@ -538,6 +548,55 @@ mod tests { assert_eq!(err.errno(), Some(Errno::EEXIST), "got {err:?}"); } + #[rstest] + fn a_staged_kernel_never_carries_a_set_id_bit( + jail: Jail, + #[values(0o4755, 0o2755, 0o6755)] mode: u32, + #[values(false, true)] cross_device: bool, + ) { + let src_dir = if cross_device { + match TempDir::new_in("/dev/shm") { + Ok(d) => d, + Err(_) => return, + } + } else { + TempDir::new_in(jail.base()).unwrap() + }; + let src = src_dir.child("vmlinux"); + src.write_str("kernel").unwrap(); + if is_root() { + std::os::unix::fs::chown(&src, Some(0), Some(0)).unwrap(); + } + fs::set_permissions(&src, fs::Permissions::from_mode(mode)).unwrap(); + + jail.open_root() + .stage_kernel(&KernelImage::open(&strict(&src)).unwrap()) + .unwrap(); + + let staged = fs::symlink_metadata(jail.root_dir.join("vmlinux")) + .unwrap() + .mode(); + assert_eq!( + staged & 0o6000, + 0, + "staged vmlinux is {staged:o} from a {mode:o} source" + ); + } + + #[rstest] + fn a_kernel_without_a_set_id_bit_is_hard_linked_on_one_filesystem(jail: Jail) { + let src = jail.kernel_source("vmlinux-src", "kernel"); + + jail.open_root() + .stage_kernel(&KernelImage::open(&strict(&src)).unwrap()) + .unwrap(); + + assert_eq!( + fs::metadata(jail.root_dir.join("vmlinux")).unwrap().ino(), + fs::metadata(&src).unwrap().ino() + ); + } + #[rstest] fn stage_kernel_never_changes_the_source(jail: Jail) { let src = jail.kernel_source("vmlinux-src", "kernel"); diff --git a/native/hyperrs-firecracker/src/jailer/mod.rs b/native/hyperrs-firecracker/src/jailer/mod.rs index e3985066..40293f2c 100644 --- a/native/hyperrs-firecracker/src/jailer/mod.rs +++ b/native/hyperrs-firecracker/src/jailer/mod.rs @@ -111,3 +111,173 @@ impl Jailer<'_> { self.command().exec_as_root(token) } } + +#[cfg(test)] +mod tests { + //! The jailer runs as root with the argv built here, so the laws are about what it is told: + //! + //! - argv round-trip: the flags before `--` parse back, pair by pair, into exactly the VM id, + //! uid, gid, chroot base, parent cgroup, netns and cgroup limits the invocation was built + //! from; `--` occurs once and is followed by nothing but `--api-sock `; + //! - `--exec-file` is the vetted Firecracker's own path, which `RootSafeBinPathBuf` only + //! accepts when no one but root can rewrite a directory above it. + + use super::chroot::JailLayout; + use super::{CgroupLimits, CpuMax, Jailer}; + use crate::VmId; + use assert_fs::TempDir; + use hyperrs_fs::path::{IsAbsolute, SafePathBuf, StrictComponent, StrictComponents}; + use hyperrs_process::binary::{RootSafeBin, RootSafeBinPathBuf}; + use nix::unistd::{Gid, Uid}; + use proptest::prelude::*; + use std::collections::BTreeMap; + use std::ffi::OsString; + use std::fs; + use std::num::NonZeroU64; + use std::os::unix::fs::PermissionsExt; + use std::path::{Path, PathBuf}; + use test_strategy::proptest; + + type Strict = SafePathBuf; + + const BASE: &str = "/srv/jails"; + const API: &str = "/api.socket"; + const NETNS: &str = "/run/netns/hyper-x"; + + fn strict(p: impl AsRef) -> Strict { + p.as_ref().to_path_buf().try_into().unwrap() + } + + fn is_root() -> bool { + nix::unistd::geteuid().is_root() + } + + fn root_binary(dir: &Path, name: &str, body: &str) -> PathBuf { + let p = dir.join(name); + fs::write(&p, body).unwrap(); + fs::set_permissions(&p, fs::Permissions::from_mode(0o755)).unwrap(); + std::os::unix::fs::chown(&p, Some(0), Some(0)).unwrap(); + p + } + + struct Bins { + _dir: TempDir, + dir: PathBuf, + jailer: RootSafeBin<"jailer">, + firecracker: RootSafeBinPathBuf<"firecracker">, + } + + fn bins() -> Bins { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); + let jailer = root_binary(&dir, "jailer", "jailer"); + let jailer = RootSafeBin::open(jailer.as_path().try_into().unwrap()).unwrap(); + let firecracker = root_binary(&dir, "firecracker", "vetted"); + let firecracker = + RootSafeBinPathBuf::open(firecracker.as_path().try_into().unwrap()).unwrap(); + Bins { + _dir: tmp, + dir, + jailer, + firecracker, + } + } + + fn argv(bins: &Bins, id: &VmId, uid: u32, gid: u32, limits: CgroupLimits) -> Vec { + let (base, api, netns) = (strict(BASE), strict(API), strict(NETNS)); + let layout = JailLayout::new(&base, id); + Jailer::builder() + .jailer(&bins.jailer) + .firecracker(&bins.firecracker) + .layout(&layout) + .parent_cgroup(StrictComponent::from_static("hyper")) + .uid(Uid::from_raw(uid)) + .gid(Gid::from_raw(gid)) + .limits(limits) + .api_sock(&api) + .netns(&netns) + .build() + .command() + .get_args() + .map(|a| a.to_os_string()) + .collect() + } + + fn limits() -> impl Strategy { + let nz = || (1u64..=u64::MAX).prop_map(|n| NonZeroU64::new(n).unwrap()); + ( + proptest::option::of(nz()), + proptest::option::of((nz(), nz()).prop_map(|(quota_us, period_us)| CpuMax { + quota_us, + period_us, + })), + ) + .prop_map(|(memory_max, cpu_max)| CgroupLimits { + memory_max, + cpu_max, + }) + } + + #[proptest(cases = 128)] + fn argv_round_trips_every_input_and_ends_with_only_the_api_socket_as_root( + #[strategy("[A-Za-z0-9][A-Za-z0-9-]{0,63}")] id: String, + uid: u32, + gid: u32, + #[strategy(limits())] limits: CgroupLimits, + ) { + if !is_root() { + eprintln!( + "SKIP argv_round_trips_every_input_and_ends_with_only_the_api_socket_as_root" + ); + return Ok(()); + } + let bins = bins(); + let id: VmId = id.parse().unwrap(); + + let args = argv(&bins, &id, uid, gid, limits); + + let split = args.iter().position(|a| a == "--").expect("no `--`"); + prop_assert_eq!(args.iter().filter(|a| *a == "--").count(), 1); + prop_assert_eq!( + &args[split + 1..], + &[OsString::from("--api-sock"), API.into()][..] + ); + let flags = &args[..split]; + prop_assert_eq!(flags.len() % 2, 0, "{:?}", flags); + let mut got: BTreeMap> = BTreeMap::new(); + for pair in flags.chunks(2) { + let flag = pair[0].to_str().unwrap().to_owned(); + prop_assert!(flag.starts_with("--"), "{:?}", pair); + prop_assert!(!pair[1].to_string_lossy().starts_with('-'), "{:?}", pair); + got.entry(flag).or_default().push(pair[1].clone()); + } + + let mut want: BTreeMap> = [ + ("--id", id.to_string().into()), + ("--exec-file", bins.dir.join("firecracker").into_os_string()), + ("--uid", uid.to_string().into()), + ("--gid", gid.to_string().into()), + ("--chroot-base-dir", BASE.into()), + ("--cgroup-version", "2".into()), + ("--parent-cgroup", "hyper".into()), + ("--netns", NETNS.into()), + ] + .into_iter() + .map(|(k, v)| (k.to_owned(), vec![v])) + .collect(); + let cgroups: Vec = limits + .memory_max + .map(|m| format!("memory.max={m}").into()) + .into_iter() + .chain( + limits + .cpu_max + .map(|c| format!("cpu.max={} {}", c.quota_us, c.period_us).into()), + ) + .collect(); + if !cgroups.is_empty() { + want.insert("--cgroup".to_owned(), cgroups); + } + prop_assert_eq!(got, want); + } +} diff --git a/native/hyperrs-firecracker/tests/kernel_trust.rs b/native/hyperrs-firecracker/tests/kernel_trust.rs new file mode 100644 index 00000000..0b6e1136 --- /dev/null +++ b/native/hyperrs-firecracker/tests/kernel_trust.rs @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: MIT +//! Trust contract of the guest kernel handed to a jail: [`KernelImage::open`] decides whether a +//! host file may become a VM's `vmlinux`, and [`JailRoot::stage_kernel`] puts it in the jail. +//! +//! Laws: +//! - refusal oracle: for every owner-readable mode, a kernel is accepted exactly when it carries +//! no group- or other-write bit, and a refusal is always `GroupOrOtherWritable`; +//! - only root or the caller may own an accepted kernel: as root, every non-zero owner uid is +//! refused with `Owner { uid }` naming exactly that uid; +//! - a non-regular source (directory, FIFO, character device, socket) is refused, and refusing it +//! never blocks; +//! - staging round-trips bytes: the jail's `vmlinux` equals the opened source, both by hard link +//! on one filesystem and by copy across filesystems (`EXDEV`), however many jails one opened +//! image is staged into; +//! - staging is bound to the inode that was opened and vetted, never the path: replacing the +//! path afterwards cannot put the replacement into the jail; +//! - a staged kernel is never writable by group or other, so a jail cannot rewrite the kernel +//! other VMs boot. + +use assert_fs::prelude::*; +use assert_fs::TempDir; +use hyperrs_firecracker::jailer::chroot::{JailLayout, JailRoot, KernelError, KernelImage}; +use hyperrs_firecracker::VmId; +use hyperrs_fs::file::ValidationError; +use hyperrs_fs::path::{IsAbsolute, SafePathBuf, StrictComponents}; +use nix::sys::stat::Mode; +use nix::unistd::{geteuid, mkfifo}; +use proptest::prelude::*; +use std::fs; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::os::unix::net::UnixListener; +use std::path::{Path, PathBuf}; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; +use test_strategy::proptest; + +type Strict = SafePathBuf; + +const REFUSAL_DEADLINE: Duration = Duration::from_secs(3); + +fn strict(p: impl AsRef) -> Strict { + p.as_ref().to_path_buf().try_into().unwrap() +} + +fn write_kernel(path: &Path, bytes: &[u8], mode: u32) { + fs::write(path, bytes).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(mode)).unwrap(); +} + +fn jail_root(base: &Path, id: &str) -> (JailRoot, PathBuf) { + let id: VmId = id.parse().unwrap(); + let root = JailLayout::new(&strict(base), &id).create_root().unwrap(); + ( + root, + base.join("firecracker").join(id.to_string()).join("root"), + ) +} + +/// Staging hard-links by fd (`AT_EMPTY_PATH`), which needs `CAP_DAC_READ_SEARCH` before Linux +/// 6.10; the helper always stages elevated, so these laws are proven as root. +fn skip_unless_root() -> bool { + let skip = !geteuid().is_root(); + if skip { + eprintln!("SKIP: needs root"); + } + skip +} + +fn shm_dir() -> Option { + let shm = Path::new("/dev/shm"); + let other_fs = fs::metadata(shm).ok()?.dev() != fs::metadata(std::env::temp_dir()).ok()?.dev(); + other_fs.then(|| TempDir::new_in(shm).unwrap()) +} + +#[proptest(cases = 256)] +fn kernel_is_accepted_exactly_when_no_group_or_other_write_bit_is_set( + #[strategy(0u32..=0o7777)] bits: u32, +) { + let mode = bits | 0o400; + let tmp = TempDir::new().unwrap(); + let src = tmp.child("vmlinux"); + write_kernel(&src, b"kernel", mode); + let actual_mode = fs::metadata(&src).unwrap().mode() & 0o7777; + + let got = KernelImage::open(&strict(&src)); + + if actual_mode & 0o022 == 0 { + prop_assert!( + got.is_ok(), + "mode {:o} refused: {:?}", + actual_mode, + got.err() + ); + } else { + prop_assert!( + matches!( + got, + Err(KernelError::Open { + source: ValidationError::GroupOrOtherWritable, + .. + }) + ), + "mode {:o} not refused as group/other-writable: {:?}", + actual_mode, + got.err() + ); + } +} + +#[proptest(cases = 64)] +fn kernel_owned_by_any_non_root_uid_is_refused_naming_that_uid_as_root( + #[strategy(1u32..u32::MAX - 1)] uid: u32, +) { + if skip_unless_root() { + return Ok(()); + } + let tmp = TempDir::new().unwrap(); + let src = tmp.child("vmlinux"); + write_kernel(&src, b"kernel", 0o644); + std::os::unix::fs::chown(&src, Some(uid), None).unwrap(); + + match KernelImage::open(&strict(&src)) { + Err(KernelError::Owner { uid: refused, .. }) => prop_assert_eq!(refused.as_raw(), uid), + other => prop_assert!(false, "uid {} not refused as owner: {:?}", uid, other.err()), + } +} + +/// Runs `open` on another thread so a blocking `open(2)` fails the test instead of hanging it; +/// `unblock` is then given the chance to release the stuck thread. +fn open_within_deadline(path: &Path, unblock: impl FnOnce()) -> Option> { + let (tx, rx) = mpsc::channel(); + let target = strict(path); + let worker = thread::spawn(move || { + let _ = tx.send( + KernelImage::open(&target) + .map(drop) + .map_err(|e| e.to_string()), + ); + }); + let got = rx.recv_timeout(REFUSAL_DEADLINE).ok(); + if got.is_none() { + unblock(); + } + let _ = worker.join(); + got +} + +#[test] +fn a_fifo_kernel_is_refused_without_blocking() { + let tmp = TempDir::new().unwrap(); + let fifo = tmp.child("vmlinux"); + mkfifo(fifo.path(), Mode::from_bits_truncate(0o644)).unwrap(); + + let fifo_path = fifo.to_path_buf(); + let got = open_within_deadline(fifo.path(), move || { + let _ = fs::OpenOptions::new() + .write(true) + .custom_flags(nix::libc::O_NONBLOCK) + .open(&fifo_path); + }); + + match got { + None => panic!("KernelImage::open blocked on a FIFO for over {REFUSAL_DEADLINE:?}"), + Some(result) => assert!(result.is_err(), "a FIFO was accepted as a kernel"), + } +} + +#[test] +fn non_regular_kernel_sources_are_refused() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.child("dir"); + dir.create_dir_all().unwrap(); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).unwrap(); + let sock = tmp.child("sock"); + let _listener = UnixListener::bind(&sock).unwrap(); + fs::set_permissions(&sock, fs::Permissions::from_mode(0o644)).unwrap(); + + let rows: [(&str, PathBuf); 3] = [ + ("directory", dir.to_path_buf()), + ("character device", PathBuf::from("/dev/null")), + ("socket", sock.to_path_buf()), + ]; + + for (name, path) in rows { + let got = open_within_deadline(&path, || {}); + match got { + None => panic!("{name}: KernelImage::open blocked"), + Some(result) => assert!(result.is_err(), "{name} was accepted as a kernel"), + } + } +} + +#[proptest(cases = 24)] +fn staging_round_trips_the_kernel_bytes_on_one_filesystem_and_across_two_as_root( + #[strategy(proptest::collection::vec(any::(), 0..65_536))] bytes: Vec, + #[strategy(1usize..4)] jails: usize, + cross_fs: bool, +) { + if skip_unless_root() { + return Ok(()); + } + let shm = if cross_fs { shm_dir() } else { None }; + let src_dir = shm.unwrap_or_else(|| TempDir::new().unwrap()); + let src = src_dir.child("vmlinux-src"); + write_kernel(&src, &bytes, 0o644); + let kernel = KernelImage::open(&strict(&src)).unwrap(); + let base = TempDir::new().unwrap(); + + for n in 0..jails { + let (root, root_path) = jail_root(base.path(), &format!("vm{n}")); + root.stage_kernel(&kernel).unwrap(); + let staged = root_path.join("vmlinux"); + prop_assert_eq!(fs::read(&staged).unwrap(), bytes.clone(), "jail {}", n); + let mode = fs::metadata(&staged).unwrap().mode(); + prop_assert_eq!( + mode & 0o022, + 0, + "jail {} staged a kernel with mode {:o}", + n, + mode + ); + } +} + +#[test] +fn a_cross_filesystem_stage_is_read_only_for_everyone_as_root() { + if skip_unless_root() { + return; + } + let Some(src_dir) = shm_dir() else { + eprintln!("SKIP: /dev/shm is not a separate filesystem"); + return; + }; + let src = src_dir.child("vmlinux-src"); + write_kernel(&src, b"cross-fs kernel", 0o644); + let kernel = KernelImage::open(&strict(&src)).unwrap(); + let base = TempDir::new().unwrap(); + + let (root, root_path) = jail_root(base.path(), "vm"); + root.stage_kernel(&kernel).unwrap(); + + let meta = fs::metadata(root_path.join("vmlinux")).unwrap(); + assert_ne!(meta.ino(), fs::metadata(&src).unwrap().ino()); + assert_eq!( + meta.mode() & 0o222, + 0, + "copied kernel mode {:o}", + meta.mode() + ); +} + +#[test] +fn replacing_the_kernel_path_after_open_never_stages_the_replacement_as_root() { + if skip_unless_root() { + return; + } + let shm = shm_dir(); + let dirs: Vec<(&str, TempDir)> = [ + ("same fs", Some(TempDir::new().unwrap())), + ("cross fs", shm), + ] + .into_iter() + .filter_map(|(name, dir)| dir.map(|d| (name, d))) + .collect(); + + for (name, src_dir) in dirs { + let src = src_dir.child("vmlinux"); + write_kernel(&src, b"vetted kernel", 0o644); + let kernel = KernelImage::open(&strict(&src)).unwrap(); + + let swap = src_dir.child("swap"); + write_kernel(&swap, b"swapped-in kernel", 0o644); + fs::rename(&swap, &src).unwrap(); + + let base = TempDir::new().unwrap(); + let (root, root_path) = jail_root(base.path(), "vm"); + let staged = root + .stage_kernel(&kernel) + .ok() + .map(|()| fs::read(root_path.join("vmlinux")).unwrap()); + + assert!( + staged.as_deref().is_none_or(|b| b == b"vetted kernel"), + "{name}: staged {:?} after the path was swapped", + staged.map(|b| String::from_utf8_lossy(&b).into_owned()) + ); + } +} diff --git a/native/hyperrs-fs/src/dir.rs b/native/hyperrs-fs/src/dir.rs index 3291f622..e555780e 100644 --- a/native/hyperrs-fs/src/dir.rs +++ b/native/hyperrs-fs/src/dir.rs @@ -6,7 +6,10 @@ use super::file::{ Any, DeviceType, FileType, IsDirectory, OpenFlags, Owner, Ownership, SafeFile, ValidationError, Writability, }; -use super::path::{ComponentWithWhitespace, ComponentWithWhitespaceBuf, StrictComponent}; +use super::path::{ + ComponentWithWhitespace, ComponentWithWhitespaceBuf, IsAbsolute, SafePath, StrictComponent, + StrictComponents, +}; use nix::dir::{Dir, Type}; use nix::errno::Errno; use nix::fcntl::{openat2, AtFlags, OFlag, OpenHow, ResolveFlag}; @@ -17,7 +20,7 @@ use std::ffi::OsStr; use std::io; use std::os::unix::ffi::OsStrExt; use std::os::unix::io::{AsRawFd, FromRawFd, OwnedFd}; -use std::path::Path; +use std::path::{Path, PathBuf}; use thiserror::Error as ThisError; #[derive(Debug, ThisError)] @@ -37,6 +40,15 @@ pub enum DirError { }, } +/// A directory above a path failed its [`SafeFile`] checks. +#[derive(Debug, ThisError)] +#[error("parent {dir}: {source}")] +pub struct ParentError { + pub dir: PathBuf, + #[source] + pub source: ValidationError, +} + impl DirError { /// The underlying errno. pub fn errno(&self) -> Option { @@ -63,6 +75,29 @@ struct Ancestor { pub type SafeDir = SafeFile; impl SafeFile { + /// Open the directory `path` is in, verifying it and every directory above it, from `/` + /// down. Each is opened beneath the one before, so a vetted directory's entries cannot be + /// swapped for the lookups that follow. + pub fn open_parent(path: &SafePath) -> Result { + match path.split_last() { + Some((parent, _)) => Self::open_vetted(parent), + None => Self::open_vetted(path), + } + } + + fn open_vetted(dir: &SafePath) -> Result { + let opened = match dir.split_last() { + None => Self::open(dir, OpenFlags::PATH), + Some((parent, name)) => { + Self::open_vetted(parent)?.openat(name, OpenFlags::PATH, Mode::empty()) + } + }; + opened.map_err(|source| ParentError { + dir: dir.as_path().to_path_buf(), + source, + }) + } + /// Open `name` relative to this directory and verify it as a [`SafeFile`]. As with /// [`SafeFile::open`], `flags` apply only after the checks pass. pub fn openat( diff --git a/native/hyperrs-fs/src/file.rs b/native/hyperrs-fs/src/file.rs index 60ded6cb..d8e333fc 100644 --- a/native/hyperrs-fs/src/file.rs +++ b/native/hyperrs-fs/src/file.rs @@ -58,6 +58,9 @@ pub struct IsDirectory; pub struct RootOwner; /// Require the file not be writable by group or other. pub struct OwnerOnlyWritable; +/// Require the file not be writable by group or other, unless it is sticky: group and other may +/// then add entries but never rename or unlink one they do not own. +pub struct StickyOrOwnerOnlyWritable; /// What's the type of the file? pub trait FileType { @@ -173,6 +176,16 @@ impl Writability for OwnerOnlyWritable { } } +impl Writability for StickyOrOwnerOnlyWritable { + fn check(stat: &FileStat) -> Result<(), ValidationError> { + if stat.st_mode & Mode::S_ISVTX.bits() != 0 { + Ok(()) + } else { + OwnerOnlyWritable::check(stat) + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, BitOr, Into)] pub struct OpenFlags(OFlag); diff --git a/native/hyperrs-fs/tests/writability.rs b/native/hyperrs-fs/tests/writability.rs new file mode 100644 index 00000000..770abd92 --- /dev/null +++ b/native/hyperrs-fs/tests/writability.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MIT + +use assert_fs::prelude::*; +use assert_fs::TempDir; +use hyperrs_fs::file::{ + Any, IsDirectory, OpenFlags, SafeFile, StickyOrOwnerOnlyWritable, ValidationError, +}; +use hyperrs_fs::path::{IsAbsolute, SafePathBuf, StrictComponents}; +use rstest::rstest; +use std::assert_matches; +use std::fs; +use std::os::unix::fs::PermissionsExt; + +#[rstest] +#[case::owner_only(0o755, true)] +#[case::owner_only_private(0o700, true)] +#[case::sticky_world_writable(0o1777, true)] +#[case::sticky_group_writable(0o1775, true)] +#[case::group_writable(0o775, false)] +#[case::other_writable(0o757, false)] +#[case::world_writable(0o777, false)] +fn a_directory_is_accepted_exactly_when_sticky_or_writable_only_by_its_owner( + #[case] mode: u32, + #[case] accepted: bool, +) { + let tmp = TempDir::new().unwrap(); + let dir = tmp.child("d"); + dir.create_dir_all().unwrap(); + fs::set_permissions(&dir, fs::Permissions::from_mode(mode)).unwrap(); + let path: SafePathBuf = + dir.path().to_path_buf().try_into().unwrap(); + + let got = SafeFile::::open(&path, OpenFlags::PATH); + + if accepted { + assert!(got.is_ok(), "{mode:o}: {got:?}"); + } else { + assert_matches!(got, Err(ValidationError::GroupOrOtherWritable)); + } +} diff --git a/native/hyperrs-linux/src/dm/thin.rs b/native/hyperrs-linux/src/dm/thin.rs index 148b6c7a..472b5c56 100644 --- a/native/hyperrs-linux/src/dm/thin.rs +++ b/native/hyperrs-linux/src/dm/thin.rs @@ -499,7 +499,7 @@ mod tests { ); bin.write_str(&script).unwrap(); fs::set_permissions(&bin, fs::Permissions::from_mode(0o755)).unwrap(); - let dmsetup = Dmsetup::new(RootSafeBin::open(&bin).unwrap()); + let dmsetup = Dmsetup::new(RootSafeBin::open(bin.path().try_into().unwrap()).unwrap()); Self { _dir: dir, log, diff --git a/native/hyperrs-linux/src/net/nft/mod.rs b/native/hyperrs-linux/src/net/nft/mod.rs index 75bf552b..1929530f 100644 --- a/native/hyperrs-linux/src/net/nft/mod.rs +++ b/native/hyperrs-linux/src/net/nft/mod.rs @@ -195,13 +195,13 @@ mod tests { }; use crate::net::netns::NetNs; use crate::net::{IpNet, Ipv4Net, Ipv6Net}; + use hyperrs_fs::path::SafePath; use hyperrs_process::binary::RootSafeBin; use hyperrs_process::privilege::Unprivileged; use nix::unistd::Uid; use pretty_assertions::assert_eq; use rstest::rstest; use serde_json::{json, Value}; - use std::path::Path; use std::process::Command; const NFT: &str = "/usr/sbin/nft"; @@ -235,7 +235,7 @@ mod tests { } fn nft() -> Nft { - Nft::new(RootSafeBin::open(Path::new(NFT)).unwrap()) + Nft::new(RootSafeBin::open(SafePath::from_static::()).unwrap()) } fn v4(net: &str) -> Ipv4Net { diff --git a/native/hyperrs-linux/tests/dm_kernel_argv_as_root.rs b/native/hyperrs-linux/tests/dm_kernel_argv_as_root.rs index 4fdeccce..6e06c794 100644 --- a/native/hyperrs-linux/tests/dm_kernel_argv_as_root.rs +++ b/native/hyperrs-linux/tests/dm_kernel_argv_as_root.rs @@ -7,6 +7,8 @@ #![feature(adt_const_params, unsized_const_params)] #![allow(incomplete_features)] +use hyperrs_fs::path::SafePath; + use assert_fs::TempDir; use hyperrs_fs::file::{Any, IsRegularFile, OpenFlags, SafeFile}; use hyperrs_fs::path::{IsAbsolute, SafePathBuf, StrictComponents}; @@ -80,7 +82,7 @@ fn a_linear_table_is_backed_by_the_path_it_names_as_root( let origin: DevPath = PathBuf::from(&named) .try_into() .expect("the type accepts it"); - let dm = Dmsetup::new(RootSafeBin::open(Path::new(DMSETUP)).unwrap()); + let dm = Dmsetup::new(RootSafeBin::open(SafePath::from_static::()).unwrap()); let name: DmName = format!("hyperrs-argv-{}", std::process::id()) .parse() .unwrap(); diff --git a/native/hyperrs-linux/tests/dm_live_as_root.rs b/native/hyperrs-linux/tests/dm_live_as_root.rs index f0113faa..485f2a0b 100644 --- a/native/hyperrs-linux/tests/dm_live_as_root.rs +++ b/native/hyperrs-linux/tests/dm_live_as_root.rs @@ -17,6 +17,8 @@ #![feature(adt_const_params, unsized_const_params)] #![allow(incomplete_features)] +use hyperrs_fs::path::SafePath; + use assert_fs::TempDir; use hyperrs_fs::dev::DeviceNumber; use hyperrs_fs::file::{Any, IsBlockDevice, IsRegularFile, OpenFlags, SafeFile}; @@ -100,7 +102,7 @@ impl FakeDmsetup { ), ); Self { - dmsetup: Dmsetup::new(RootSafeBin::open(&bin).unwrap()), + dmsetup: Dmsetup::new(RootSafeBin::open(bin.as_path().try_into().unwrap()).unwrap()), _dir: dir, base, } @@ -346,7 +348,9 @@ impl Drop for DmGuard { } fn live_dmsetup() -> Option { - RootSafeBin::open(Path::new(DMSETUP)).ok().map(Dmsetup::new) + RootSafeBin::open(SafePath::from_static::()) + .ok() + .map(Dmsetup::new) } #[test] @@ -419,7 +423,7 @@ impl FakeThinDump { ), ); Self { - dump: ThinDump::new(RootSafeBin::open(&bin).unwrap()), + dump: ThinDump::new(RootSafeBin::open(bin.as_path().try_into().unwrap()).unwrap()), _dir: dir, base, } @@ -509,7 +513,7 @@ fn a_failing_thin_dump_is_an_error_not_an_empty_dump_as_root() { "thin_dump", "printf ''\nprintf ' no metadata snap \\n' >&2\nexit 1\n", ); - let dump = ThinDump::new(RootSafeBin::open(&bin).unwrap()); + let dump = ThinDump::new(RootSafeBin::open(bin.as_path().try_into().unwrap()).unwrap()); let (_lo, meta) = metadata_device(&base, b"0123456789abcdef"); let err = dump @@ -593,7 +597,7 @@ fn read_block(dev: &Path, block: u64) -> Vec { fn mapped_blocks(pool: &LivePool, root: &Root<'_>, id: ThinId) -> BTreeSet { let thin_pool = pool.dm.thin_pool(root, &pool.pool).unwrap(); - let dump = ThinDump::new(RootSafeBin::open(Path::new(THIN_DUMP)).unwrap()); + let dump = ThinDump::new(RootSafeBin::open(SafePath::from_static::()).unwrap()); thin_pool.reserve_metadata_snap().unwrap(); let dumped = dump.dump(root, &pool.meta_dev, id); thin_pool.release_metadata_snap().unwrap(); @@ -614,7 +618,7 @@ fn a_real_pools_ids_snapshots_and_dumped_mappings_obey_their_laws_as_root() { require_root!(); let mut token = Unprivileged::take().unwrap(); let root = token.elevate().unwrap(); - if RootSafeBin::<"thin_dump">::open(Path::new(THIN_DUMP)).is_err() { + if RootSafeBin::<"thin_dump">::open(SafePath::from_static::()).is_err() { eprintln!("SKIP: no root-owned {THIN_DUMP}"); return; } @@ -685,7 +689,7 @@ fn a_thin_over_an_external_origin_maps_only_what_diverged_as_root() { require_root!(); let mut token = Unprivileged::take().unwrap(); let root = token.elevate().unwrap(); - if RootSafeBin::<"thin_dump">::open(Path::new(THIN_DUMP)).is_err() { + if RootSafeBin::<"thin_dump">::open(SafePath::from_static::()).is_err() { eprintln!("SKIP: no root-owned {THIN_DUMP}"); return; } diff --git a/native/hyperrs-process/src/binary.rs b/native/hyperrs-process/src/binary.rs index e3e59f68..65494e17 100644 --- a/native/hyperrs-process/src/binary.rs +++ b/native/hyperrs-process/src/binary.rs @@ -1,6 +1,10 @@ //! Utilities for interacting with binaries. -use hyperrs_fs::file::{self, IsRegularFile, OpenFlags, OwnerOnlyWritable, RootOwner, SafeFile}; -use hyperrs_fs::path::{self, IsAbsolute, SafePath, StrictComponent, StrictComponents}; +use hyperrs_fs::dir::ParentError; +use hyperrs_fs::file::{ + self, IsDirectory, IsRegularFile, OpenFlags, OwnerOnlyWritable, RootOwner, SafeFile, + StickyOrOwnerOnlyWritable, +}; +use hyperrs_fs::path::{IsAbsolute, SafePath, StrictComponent, StrictComponents}; use nix::errno::Errno; use nix::fcntl::{fcntl, FcntlArg, FdFlag}; use std::os::unix::io::{AsFd, AsRawFd, BorrowedFd}; @@ -10,29 +14,27 @@ use std::process::Command; use thiserror::Error as ThisError; #[derive(Debug, ThisError)] -pub enum RootSafeBinError { - #[error("binary path {path}: {source}")] - Path { - path: PathBuf, - #[source] - source: path::ValidationError, - }, - #[error("binary basename must be `{expected}`: {got}")] - Name { - expected: &'static str, - got: PathBuf, - }, - #[error("binary {path}: {source}")] - File { - path: PathBuf, - #[source] - source: file::ValidationError, - }, - #[error("binary {path}: clearing FD_CLOEXEC: {errno}")] - Inherit { path: PathBuf, errno: Errno }, +#[error("binary {path}: {kind}")] +pub struct RootSafeBinError { + pub path: PathBuf, + #[source] + pub kind: RootSafeBinErrorKind, +} + +#[derive(Debug, ThisError)] +pub enum RootSafeBinErrorKind { + #[error("basename must be `{0}`")] + Name(&'static str), + #[error(transparent)] + File(#[from] file::ValidationError), + #[error("clearing FD_CLOEXEC: {0}")] + Inherit(#[from] Errno), + #[error(transparent)] + Parent(#[from] ParentError), } type Executable = SafeFile; +type RootOnlyDir = SafeFile; /// A binary that's safe for the root user to use. #[derive(Debug)] @@ -44,32 +46,19 @@ pub struct RootSafeBinPathBuf(PathBuf); impl RootSafeBin { /// Validate that the given path is a safe binary, owned by root, non-symlink, not writable by /// everyone and matches `NAME`. - pub fn open(bin: &Path) -> Result { - let safe: &SafePath = - bin.try_into().map_err(|source| RootSafeBinError::Path { - path: bin.to_path_buf(), - source, - })?; - - if safe.file_name() != Some(const { StrictComponent::from_static(NAME) }) { - return Err(RootSafeBinError::Name { - expected: NAME, - got: bin.to_path_buf(), - }); - } - - let file = - Executable::open(safe, OpenFlags::PATH).map_err(|source| RootSafeBinError::File { - path: bin.to_path_buf(), - source, - })?; - fcntl(file.as_raw_fd(), FcntlArg::F_SETFD(FdFlag::empty())).map_err(|errno| { - RootSafeBinError::Inherit { - path: bin.to_path_buf(), - errno, - } - })?; + pub fn open(bin: &SafePath) -> Result { + Self::vet(bin).map_err(|kind| RootSafeBinError { + path: bin.as_path().to_path_buf(), + kind, + }) + } + fn vet(bin: &SafePath) -> Result { + if bin.file_name() != Some(const { StrictComponent::from_static(NAME) }) { + return Err(RootSafeBinErrorKind::Name(NAME)); + } + let file = Executable::open(bin, OpenFlags::PATH)?; + fcntl(file.as_raw_fd(), FcntlArg::F_SETFD(FdFlag::empty()))?; Ok(Self(file)) } @@ -91,9 +80,19 @@ impl AsFd for RootSafeBin { } impl RootSafeBinPathBuf { - pub fn open(bin: &Path) -> Result { - RootSafeBin::::open(bin)?; - Ok(Self(bin.to_path_buf())) + pub fn open(bin: &SafePath) -> Result { + Self::vet(bin).map_err(|kind| RootSafeBinError { + path: bin.as_path().to_path_buf(), + kind, + }) + } + + fn vet(bin: &SafePath) -> Result { + // The binary is handed on by path, so no one but root may be able to rewrite any + // directory above it. + RootOnlyDir::open_parent(bin)?; + RootSafeBin::::vet(bin)?; + Ok(Self(bin.as_path().to_path_buf())) } pub fn as_path(&self) -> &Path { @@ -109,13 +108,17 @@ mod tests { //! root and not group/other-writable. These assert the refusal axes; the symlink //! axis is root-independent, the owner axis is asserted both ways. - use super::RootSafeBin; + use super::{RootSafeBin, RootSafeBinError, RootSafeBinErrorKind, RootSafeBinPathBuf}; use assert_fs::prelude::*; use assert_fs::TempDir; + use hyperrs_fs::dir::ParentError; + use hyperrs_fs::file::ValidationError::{self, GroupOrOtherWritable, NotRootOwned}; + use hyperrs_fs::path::SafePath; use rstest::{fixture, rstest}; use std::fs; + use std::mem::discriminant; use std::os::unix::fs::PermissionsExt; - use std::path::{Path, PathBuf}; + use std::path::PathBuf; fn is_root() -> bool { nix::unistd::geteuid().is_root() @@ -133,12 +136,19 @@ mod tests { f.to_path_buf() } - #[rstest] - #[case::bare_name("dmsetup")] - #[case::dot_relative("./dmsetup")] - #[case::wrong_basename("/usr/bin/env")] - fn rejects_relative_path_or_wrong_basename(#[case] path: &str) { - assert!(RootSafeBin::<"dmsetup">::open(Path::new(path)).is_err()); + #[test] + fn rejects_wrong_basename() { + let got = RootSafeBin::<"dmsetup">::open(SafePath::from_static::<"/usr/bin/env">()); + assert!( + matches!( + got, + Err(RootSafeBinError { + kind: RootSafeBinErrorKind::Name("dmsetup"), + .. + }) + ), + "{got:?}" + ); } #[rstest] @@ -148,12 +158,13 @@ mod tests { let link = dir.child("dmsetup"); link.symlink_to_file(&target).unwrap(); // Symlink is checked before ownership, so this holds regardless of who runs it. - assert!(RootSafeBin::<"dmsetup">::open(&link).is_err()); + assert!(RootSafeBin::<"dmsetup">::open(link.path().try_into().unwrap()).is_err()); } #[rstest] fn owner_axis_root_owned_accepted_else_rejected(dir: TempDir) { - let got = RootSafeBin::<"dmsetup">::open(&dmsetup_with_mode(&dir, 0o755)); + let bin = dmsetup_with_mode(&dir, 0o755); + let got = RootSafeBin::<"dmsetup">::open(bin.as_path().try_into().unwrap()); if is_root() { // root-owned, 0755, absolute, correct basename, not a symlink → valid. assert!(got.is_ok(), "root-owned valid bin rejected: {got:?}"); @@ -167,9 +178,75 @@ mod tests { fn rejects_group_or_other_writable(dir: TempDir) { if is_root() { // root-owned but group/other-writable → Writable rejection. - assert!(RootSafeBin::<"dmsetup">::open(&dmsetup_with_mode(&dir, 0o757)).is_err()); + let bin = dmsetup_with_mode(&dir, 0o757); + assert!(RootSafeBin::<"dmsetup">::open(bin.as_path().try_into().unwrap()).is_err()); } else { eprintln!("SKIP rejects_group_or_other_writable: needs root to own a 0757 file"); } } + + // `/` is a root-owned 0755 directory and `/tmp` a root-owned sticky world-writable one, so + // both must be passed and the walk must stop at the first directory the caller owns. + #[test] + fn path_binary_is_refused_at_the_first_parent_owned_by_someone_other_than_root() { + let tmp = TempDir::new_in("/tmp").unwrap(); + let bin = tmp.child("dmsetup"); + bin.write_str("#!/bin/true\n").unwrap(); + fs::set_permissions(&bin, fs::Permissions::from_mode(0o755)).unwrap(); + if is_root() { + std::os::unix::fs::chown(tmp.path(), Some(65_534), None).unwrap(); + } + + let got = RootSafeBinPathBuf::<"dmsetup">::open(bin.path().try_into().unwrap()); + + assert!( + matches!(got, Err(RootSafeBinError { kind: RootSafeBinErrorKind::Parent(ParentError { ref dir, source: NotRootOwned }), .. }) if dir == tmp.path()), + "{got:?}" + ); + } + + // `RootSafeBinPathBuf` hands the jailer a path, and the release flow leaves the install + // directory owned by the node user, who could rename their own file over the vetted binary + // after it was checked. Any directory on the path that someone other than root can rewrite + // is refused, at any depth; a sticky one is not, since its root-owned entries stay put. + #[rstest] + #[case::root_0755(0o755, 0, None)] + #[case::root_sticky_world_writable(0o1777, 0, None)] + #[case::owned_by_another_user(0o755, 65_534, Some(NotRootOwned))] + #[case::group_writable(0o775, 0, Some(GroupOrOtherWritable))] + #[case::other_writable(0o757, 0, Some(GroupOrOtherWritable))] + fn path_binary_is_accepted_exactly_when_only_root_can_rewrite_its_parents_as_root( + dir: TempDir, + #[case] mode: u32, + #[case] uid: u32, + #[case] refusal: Option, + #[values(0, 1)] depth: usize, + ) { + if !is_root() { + eprintln!("SKIP path_binary_is_accepted_exactly_when_only_root_can_rewrite_its_parents_as_root"); + return; + } + let outer = dir.child("outer"); + let inner = outer.child("bin"); + inner.create_dir_all().unwrap(); + fs::set_permissions(&outer, fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(&inner, fs::Permissions::from_mode(0o755)).unwrap(); + let bin = inner.child("dmsetup"); + bin.write_str("#!/bin/true\n").unwrap(); + fs::set_permissions(&bin, fs::Permissions::from_mode(0o755)).unwrap(); + let loose = [&outer, &inner][depth]; + std::os::unix::fs::chown(loose, Some(uid), None).unwrap(); + fs::set_permissions(loose, fs::Permissions::from_mode(mode)).unwrap(); + + let got = RootSafeBinPathBuf::<"dmsetup">::open(bin.path().try_into().unwrap()); + + match refusal { + None => assert!(got.is_ok(), "{got:?}"), + Some(want) => assert!( + matches!(got, Err(RootSafeBinError { kind: RootSafeBinErrorKind::Parent(ParentError { ref dir, source }), .. }) + if dir == loose.path() && discriminant(&source) == discriminant(&want)), + "{got:?}" + ), + } + } } diff --git a/native/suidhelper/src/config.rs b/native/suidhelper/src/config.rs index 57e22cfc..0012c5ee 100644 --- a/native/suidhelper/src/config.rs +++ b/native/suidhelper/src/config.rs @@ -23,7 +23,7 @@ use serde::{Deserialize, Deserializer}; use serde_with::{serde_as, DisplayFromStr}; use smart_default::SmartDefault; use std::os::unix::io::OwnedFd; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::LazyLock; use thiserror::Error; @@ -72,8 +72,9 @@ fn config_path() -> PathBuf { /// Hyper's /etc/hyper/config.toml file format. #[derive(Debug, Clone, Deserialize, SmartDefault)] pub struct Config { - #[default(default_work_dir())] - #[serde(deserialize_with = "safe_path")] + // Must match the Elixir node's `@dev_work_dir`, which it uses when the same config file is + // absent, so both sides agree (see `Hyper.Node.check_helper_base`). + #[default(SafePath::from_static::<"/srv/hyper">().to_owned())] work_dir: SafePathBuf, #[serde(default)] tools: Tools, @@ -180,14 +181,6 @@ fn uid_gid_range<'de, D: Deserializer<'de>>(deserializer: D) -> Result::deserialize(deserializer)?).map_err(serde::de::Error::custom) } -fn safe_path<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - PathBuf::deserialize(deserializer)? - .try_into() - .map_err(serde::de::Error::custom) -} - fn strict_component<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result { @@ -208,25 +201,14 @@ fn strict_component<'de, D: Deserializer<'de>>( #[derive(Debug, Clone, Deserialize, SmartDefault)] #[serde(default)] pub struct Tools { - #[default("/usr/sbin/dmsetup".into())] - dmsetup: PathBuf, - #[default("/usr/sbin/thin_dump".into())] - thin_dump: PathBuf, - #[default("/usr/sbin/nft".into())] - nft: PathBuf, - firecracker: Option, - jailer: Option, -} - -fn required(path: &Option) -> Result<&Path, BinError> { - path.as_deref().ok_or(BinError::Unconfigured(NAME)) -} - -// The default data root. Must match the Elixir node's `@dev_work_dir`, which it -// uses when the same config file is absent, so both sides agree (see -// `Hyper.Node.check_helper_base`). -fn default_work_dir() -> SafePathBuf { - SafePath::from_static::<"/srv/hyper">().to_owned() + #[default(SafePath::from_static::<"/usr/sbin/dmsetup">().to_owned())] + dmsetup: SafePathBuf, + #[default(SafePath::from_static::<"/usr/sbin/thin_dump">().to_owned())] + thin_dump: SafePathBuf, + #[default(SafePath::from_static::<"/usr/sbin/nft">().to_owned())] + nft: SafePathBuf, + firecracker: Option>, + jailer: Option>, } const JAILS: &StrictComponent = StrictComponent::from_static("jails"); @@ -298,18 +280,20 @@ impl Config { /// Errors [`BinError::Unconfigured`] when absent from config — an operator /// must set `[tools] firecracker` before any VM can be launched. pub fn firecracker(&self) -> Result, BinError> { - Ok(RootSafeBinPathBuf::open(required::<"firecracker">( - &self.tools.firecracker, - )?)?) + let bin = self.tools.firecracker.as_deref(); + Ok(RootSafeBinPathBuf::open( + bin.ok_or(BinError::Unconfigured("firecracker"))?, + )?) } /// The Firecracker jailer binary, validated as root-owned and correctly named. /// Errors [`BinError::Unconfigured`] when absent from config — an operator /// must set `[tools] jailer` before any VM can be launched. pub fn jailer(&self) -> Result, BinError> { - Ok(RootSafeBin::open(required::<"jailer">( - &self.tools.jailer, - )?)?) + let bin = self.tools.jailer.as_deref(); + Ok(RootSafeBin::open( + bin.ok_or(BinError::Unconfigured("jailer"))?, + )?) } /// The jailer `--parent-cgroup` value, from `[jails] cgroup`. Defaults to diff --git a/native/suidhelper/src/net/vm.rs b/native/suidhelper/src/net/vm.rs index 0ecd6c0a..1b3bfb6b 100644 --- a/native/suidhelper/src/net/vm.rs +++ b/native/suidhelper/src/net/vm.rs @@ -491,6 +491,7 @@ impl VmNet<'_> { #[cfg(test)] mod tests { use super::{Blocked, Firewall, VmNet, TAP, UPLINK}; + use hyperrs_fs::path::SafePath; use hyperrs_linux::net::link::DuplicateAddrDetection; use hyperrs_linux::net::netns::NetNs; use hyperrs_linux::net::nft::Nft; @@ -505,7 +506,6 @@ mod tests { use socket2::{Domain, Protocol, Socket, Type}; use std::io::Read; use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, TcpListener}; - use std::path::Path; use std::time::Duration; const OPEN: u16 = 7071; @@ -621,7 +621,8 @@ mod tests { } vm.get().enable_forwarding().unwrap(); - let nft = Nft::new(RootSafeBin::open(Path::new("/usr/sbin/nft")).unwrap()); + let nft = + Nft::new(RootSafeBin::open(SafePath::from_static::<"/usr/sbin/nft">()).unwrap()); let mut token = Unprivileged::take().unwrap(); let root = token.elevate().unwrap(); let firewall: Firewall = serde_json::from_value(json!({ diff --git a/native/suidhelper/tests/e2e/config.rs b/native/suidhelper/tests/e2e/config.rs index 510c399c..1f56def9 100644 --- a/native/suidhelper/tests/e2e/config.rs +++ b/native/suidhelper/tests/e2e/config.rs @@ -221,7 +221,13 @@ fn jailer_basename_mismatch_rejected() { let err = config .jailer() .expect_err("wrong-basename jailer path must be rejected"); - assert_matches!(err, BinError::Bin(binary::RootSafeBinError::Name { .. })); + assert_matches!( + err, + BinError::Bin(binary::RootSafeBinError { + kind: binary::RootSafeBinErrorKind::Name("jailer"), + .. + }) + ); } #[rstest] diff --git a/test/hyper/firecracker/api/transport_cap_properties_test.exs b/test/hyper/firecracker/api/transport_cap_properties_test.exs new file mode 100644 index 00000000..897ba46e --- /dev/null +++ b/test/hyper/firecracker/api/transport_cap_properties_test.exs @@ -0,0 +1,324 @@ +defmodule Hyper.Firecracker.Api.TransportCapPropertiesTest do + @moduledoc """ + The 1 MiB reply cap of `Hyper.Firecracker.Api.Transport`, held against a + daemon that controls every byte on the wire (a compromised, jailed + Firecracker), not just a well-behaved `content-length` responder. + + Laws: + + * size oracle: whatever the framing (`content-length`, chunked with any + chunk split, close-delimited) and whatever the status, a reply body of + `n` bytes is classified by the ordinary result contract when + `n <= 1 MiB` and is `{:error, {:body_too_large, 1 MiB}}` when + `n > 1 MiB`; + * the cap bounds work, not just the result: a lying `content-length` or an + endless stream is refused as soon as the cap is crossed, without waiting + for bytes that will never come or reading bytes that never end; + * a refused reply never poisons the socket: the next request on the same + daemon sees its own reply, not the remainder of the refused one; + * a reply to a typed operation is the schema struct or an error, never a + raw non-object passed off as `{:ok, _}`; + * a single call is bounded in wall time below `Client.run/2`'s call + timeout, so a drip-feeding daemon cannot wedge the per-VM client. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + alias Hyper.Firecracker.Api.{InstanceInfo, Transport} + + @cap Unit.Information.as_bytes(Unit.Information.mib(1)) + @client_call_timeout :timer.seconds(35) + + defmodule RawDaemon do + @moduledoc false + + @spec start((:gen_tcp.socket(), non_neg_integer() -> :close | :keep)) :: Path.t() + def start(respond) do + path = Path.join(System.tmp_dir!(), "fc-raw-#{System.unique_integer([:positive])}.sock") + + {:ok, listen} = + :gen_tcp.listen(0, [ + :binary, + ifaddr: {:local, String.to_charlist(path)}, + packet: :http_bin, + active: false, + send_timeout: 5_000 + ]) + + counter = :counters.new(1, []) + server = spawn(fn -> accept_loop(listen, respond, counter) end) + + ExUnit.Callbacks.on_exit(fn -> + Process.exit(server, :kill) + :gen_tcp.close(listen) + File.rm(path) + end) + + path + end + + defp accept_loop(listen, respond, counter) do + case :gen_tcp.accept(listen) do + {:ok, sock} -> + spawn(fn -> serve(sock, respond, counter) end) + accept_loop(listen, respond, counter) + + {:error, _} -> + :ok + end + end + + defp serve(sock, respond, counter) do + :ok = :inet.setopts(sock, packet: :http_bin) + + with :ok <- read_request(sock) do + :counters.add(counter, 1, 1) + + case respond.(sock, :counters.get(counter, 1)) do + :keep -> serve(sock, respond, counter) + :close -> :gen_tcp.close(sock) + end + end + end + + defp read_request(sock) do + case :gen_tcp.recv(sock, 0) do + {:ok, {:http_request, _, _, _}} -> read_headers(sock, 0) + _ -> :closed + end + end + + defp read_headers(sock, length) do + case :gen_tcp.recv(sock, 0) do + {:ok, {:http_header, _, :"Content-Length", _, value}} -> + read_headers(sock, String.to_integer(value)) + + {:ok, {:http_header, _, _, _, _}} -> + read_headers(sock, length) + + {:ok, :http_eoh} -> + :ok = :inet.setopts(sock, packet: :raw) + if length > 0, do: {:ok, _} = :gen_tcp.recv(sock, length) + :ok + + _ -> + :closed + end + end + end + + setup do + {:ok, _} = Application.ensure_all_started(:req) + :ok + end + + defp request(path, extra \\ []) do + Map.merge(%{method: :get, url: "/", opts: [socket_path: path]}, Map.new(extra)) + end + + defp json_of_size(n) do + shell = ~s({"x":""}) + ~s({"x":") <> String.duplicate("a", n - byte_size(shell)) <> ~s("}) + end + + defp split(body, sizes) do + Enum.reduce_while(Stream.cycle(sizes), {body, []}, fn + _size, {"", acc} -> + {:halt, {"", acc}} + + size, {rest, acc} -> + take = min(size, byte_size(rest)) + <> = rest + {:cont, {tail, [chunk | acc]}} + end) + |> elem(1) + |> Enum.reverse() + end + + defp send_reply(sock, status, body, :content_length, _sizes) do + :gen_tcp.send(sock, [ + "HTTP/1.1 #{status} X\r\ncontent-type: application/json\r\n", + "content-length: #{byte_size(body)}\r\nconnection: close\r\n\r\n", + body + ]) + end + + defp send_reply(sock, status, body, :chunked, sizes) do + _ = + :gen_tcp.send( + sock, + "HTTP/1.1 #{status} X\r\ncontent-type: application/json\r\n" <> + "transfer-encoding: chunked\r\nconnection: close\r\n\r\n" + ) + + for chunk <- split(body, sizes) do + :gen_tcp.send(sock, [Integer.to_string(byte_size(chunk), 16), "\r\n", chunk, "\r\n"]) + end + + :gen_tcp.send(sock, "0\r\n\r\n") + end + + defp send_reply(sock, status, body, :close_delimited, sizes) do + _ = + :gen_tcp.send( + sock, + "HTTP/1.1 #{status} X\r\ncontent-type: application/json\r\nconnection: close\r\n\r\n" + ) + + for chunk <- split(body, sizes), do: :gen_tcp.send(sock, chunk) + end + + defp size_around_cap do + one_of([ + integer((@cap - 64)..(@cap + 64)), + integer(16..4096), + integer((@cap + 65)..(3 * @cap)) + ]) + end + + property "a reply is refused exactly when its body exceeds 1 MiB, under every framing and status" do + check all( + size <- size_around_cap(), + framing <- member_of([:content_length, :chunked, :close_delimited]), + status <- member_of([200, 400, 500]), + sizes <- list_of(integer(1..200_000), min_length: 1, max_length: 6), + max_runs: 60 + ) do + body = json_of_size(size) + + path = + RawDaemon.start(fn sock, _n -> + _ = send_reply(sock, status, body, framing, sizes) + :close + end) + + got = Transport.request(request(path)) + + expected = + cond do + size > @cap -> {:error, {:body_too_large, @cap}} + status == 200 -> {:ok, %{"x" => String.duplicate("a", size - 8)}} + true -> {:error, {:api, status, nil}} + end + + assert got == expected, "#{size} bytes, #{framing}, #{status}" + end + end + + test "a lying content-length is refused once the cap is crossed, not after the advertised length" do + path = + RawDaemon.start(fn sock, _n -> + _ = + :gen_tcp.send(sock, [ + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\n", + "content-length: #{1024 * @cap}\r\n\r\n", + String.duplicate("a", 2 * @cap) + ]) + + Process.sleep(:infinity) + end) + + {micros, got} = :timer.tc(fn -> Transport.request(request(path)) end) + + assert got == {:error, {:body_too_large, @cap}} + assert micros < 5_000_000, "took #{div(micros, 1000)} ms" + end + + test "an endless chunked stream is refused rather than read forever" do + path = + RawDaemon.start(fn sock, _n -> + _ = :gen_tcp.send(sock, "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n") + chunk = String.duplicate("a", 65_536) + frame = [Integer.to_string(byte_size(chunk), 16), "\r\n", chunk, "\r\n"] + + Stream.repeatedly(fn -> :gen_tcp.send(sock, frame) end) + |> Enum.find(&match?({:error, _}, &1)) + + :close + end) + + task = Task.async(fn -> Transport.request(request(path)) end) + + assert Task.yield(task, 10_000) == {:ok, {:error, {:body_too_large, @cap}}} + end + + test "after a refused reply on a kept-alive connection the next request reads its own reply" do + path = + RawDaemon.start(fn sock, n -> + body = if n == 1, do: json_of_size(2 * @cap), else: ~s({"n":#{n}}) + + _ = + :gen_tcp.send(sock, [ + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\n", + "content-length: #{byte_size(body)}\r\n\r\n", + body + ]) + + :keep + end) + + assert Transport.request(request(path)) == {:error, {:body_too_large, @cap}} + assert {:ok, %{"n" => n}} = Transport.request(request(path)) + assert is_integer(n) + end + + test "a typed reply that is not a JSON object is never {:ok, non-struct}" do + rows = [ + {"empty body", "", "application/json"}, + {"JSON array", "[1,2,3]", "application/json"}, + {"JSON string", ~s("Running"), "application/json"}, + {"JSON null", "null", "application/json"}, + {"plain text", "Running", "text/plain"} + ] + + leaks = + for {name, body, type} <- rows, + got = typed_reply(body, type), + not (match?({:ok, %InstanceInfo{}}, got) or match?({:error, _}, got)), + do: {name, got} + + assert leaks == [] + end + + defp typed_reply(body, type) do + path = + RawDaemon.start(fn sock, _n -> + _ = + :gen_tcp.send(sock, [ + "HTTP/1.1 200 OK\r\ncontent-type: #{type}\r\n", + "content-length: #{byte_size(body)}\r\nconnection: close\r\n\r\n", + body + ]) + + :close + end) + + Transport.request(request(path, response: [{200, {InstanceInfo, :t}}])) + end + + @tag timeout: 120_000 + test "a drip-fed reply is cut off before Client.run/2's call timeout" do + path = + RawDaemon.start(fn sock, _n -> + _ = + :gen_tcp.send( + sock, + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 1000\r\n\r\n" + ) + + Stream.repeatedly(fn -> + Process.sleep(2_000) + :gen_tcp.send(sock, " ") + end) + |> Enum.find(&match?({:error, _}, &1)) + + :close + end) + + task = Task.async(fn -> Transport.request(request(path)) end) + got = Task.yield(task, @client_call_timeout) || Task.shutdown(task, :brutal_kill) + + assert match?({:ok, {:error, _}}, got), + "Transport.request still running after #{@client_call_timeout} ms: #{inspect(got)}" + end +end diff --git a/test/hyper/firecracker/api/transport_hostile_headers_test.exs b/test/hyper/firecracker/api/transport_hostile_headers_test.exs new file mode 100644 index 00000000..164977a7 --- /dev/null +++ b/test/hyper/firecracker/api/transport_hostile_headers_test.exs @@ -0,0 +1,78 @@ +defmodule Hyper.Firecracker.Api.TransportHostileHeadersTest do + @moduledoc """ + The reply cap of `Hyper.Firecracker.Api.Transport` must bound everything a + compromised, jailed Firecracker can make the node buffer, not only the body. + + Law: a reply whose header block alone exceeds the 1 MiB body cap — whether + as many headers or as one endless header line — is never accepted as a + successful call; it is an error, and the call returns in bounded time. + """ + use ExUnit.Case, async: true + + alias Hyper.Firecracker.Api.Transport + + @cap Unit.Information.as_bytes(Unit.Information.mib(1)) + @header_bytes 16 * @cap + + setup do + {:ok, _} = Application.ensure_all_started(:req) + :ok + end + + defp daemon(header_block) do + path = Path.join(System.tmp_dir!(), "fc-hdr-#{System.unique_integer([:positive])}.sock") + + {:ok, listen} = + :gen_tcp.listen(0, [:binary, ifaddr: {:local, String.to_charlist(path)}, active: false]) + + server = + spawn_link(fn -> + {:ok, sock} = :gen_tcp.accept(listen) + {:ok, _request} = :gen_tcp.recv(sock, 0) + + _ = + :gen_tcp.send(sock, [ + "HTTP/1.1 200 OK\r\n", + header_block, + "content-type: application/json\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}" + ]) + + :gen_tcp.close(sock) + end) + + on_exit(fn -> + Process.exit(server, :kill) + :gen_tcp.close(listen) + File.rm(path) + end) + + path + end + + defp many_headers do + line = "x-pad: " <> String.duplicate("a", 1016) <> "\r\n" + List.duplicate(line, div(@header_bytes, byte_size(line))) + end + + defp one_long_header, do: ["x-pad: ", String.duplicate("a", @header_bytes), "\r\n"] + + test "a header block past the cap is not a successful call, however it is shaped" do + for {name, block} <- [one_long_header: one_long_header(), many_headers: many_headers()] do + path = daemon(block) + + task = + Task.async(fn -> + Transport.request(%{method: :get, url: "/", opts: [socket_path: path]}) + end) + + result = + case Task.yield(task, :timer.seconds(20)) || Task.shutdown(task, :brutal_kill) do + {:ok, result} -> result + nil -> :timed_out + end + + refute match?({:ok, _}, result) or result in [:ok, :timed_out], + "#{name}: #{div(@header_bytes, @cap)} MiB of headers -> #{inspect(result, limit: 5)}" + end + end +end diff --git a/test/hyper/node/vmlinux_test.exs b/test/hyper/node/vmlinux_test.exs new file mode 100644 index 00000000..924b82d5 --- /dev/null +++ b/test/hyper/node/vmlinux_test.exs @@ -0,0 +1,84 @@ +defmodule Hyper.Node.VmlinuxTest do + @moduledoc """ + Boot-time verdict of `Hyper.Node.Vmlinux.test_system/0` on an + operator-configured kernel path. + + Law: the node's boot check and the setuid helper agree. `test_system/0` + exists so "a misconfigured node aborts at boot instead of failing the first + VM launch", so it must refuse exactly the kernels the helper's + `KernelImage::open` refuses (absent, non-regular, group/other-writable, + owned by neither root nor the node user) and accept exactly the ones it + stages. The helper resolves symlinks before vetting the target + (`kernel_symlink_stages_its_target`), so a symlink is judged by what it + points at. + """ + use ExUnit.Case, async: false + + alias Hyper.Cfg.VmLinux + alias Hyper.Node.Vmlinux + + setup do + dir = Path.join(System.tmp_dir!(), "vmlinux-boot-#{System.unique_integer([:positive])}") + File.mkdir_p!(dir) + Hyper.Cfg.Toml.put_cache(%{}) + + on_exit(fn -> + Application.delete_env(:hyper, VmLinux) + Hyper.Cfg.Toml.reload() + File.rm_rf!(dir) + end) + + {:ok, dir: dir} + end + + defp verdict(path) do + Application.put_env(:hyper, VmLinux, amd64: path, aarch64: path) + Vmlinux.test_system() + end + + defp kernel(dir, name, mode) do + path = Path.join(dir, name) + File.write!(path, "kernel") + File.chmod!(path, mode) + path + end + + defp link(dir, name, target) do + path = Path.join(dir, name) + File.ln_s!(target, path) + path + end + + test "refuses exactly what the helper refuses", %{dir: dir} do + trusted = kernel(dir, "trusted", 0o644) + loose = kernel(dir, "loose", 0o664) + missing = Path.join(dir, "missing") + subdir = Path.join(dir, "subdir") + File.mkdir_p!(subdir) + + rows = [ + {"node-owned 0644", trusted, :ok}, + {"node-owned 0444", kernel(dir, "readonly", 0o444), :ok}, + {"root-owned 0644", "/etc/passwd", :ok}, + {"group-writable", loose, {:error, {:vmlinux_untrusted, loose}}}, + {"absent", missing, {:error, {:vmlinux_missing, missing}}}, + {"directory", subdir, {:error, {:vmlinux_missing, subdir}}} + ] + + for {name, path, expected} <- rows do + assert verdict(path) == expected, name + end + end + + test "a symlink is judged by its target, as the helper judges it", %{dir: dir} do + trusted = kernel(dir, "vmlinux-6.1.1", 0o644) + loose = kernel(dir, "vmlinux-loose", 0o666) + to_trusted = link(dir, "vmlinux", trusted) + to_loose = link(dir, "vmlinux-l", loose) + to_root = link(dir, "vmlinux-root", "/etc/passwd") + + got = for path <- [to_trusted, to_root, to_loose], do: verdict(path) + + assert got == [:ok, :ok, {:error, {:vmlinux_untrusted, to_loose}}] + end +end diff --git a/test/redist/targz_symlink_chain_test.exs b/test/redist/targz_symlink_chain_test.exs new file mode 100644 index 00000000..c9f9a963 --- /dev/null +++ b/test/redist/targz_symlink_chain_test.exs @@ -0,0 +1,57 @@ +defmodule Redist.TargzSymlinkChainTest do + @moduledoc """ + `Redist.Targz.install/3` against an archive whose symlinks are each + individually "inside" `dest_dir` but chain to escape it (`s -> .`, + `t -> s/..`), followed by an entry written through the chain. + + Law: such an archive is refused with an error and nothing lands outside + `dest_dir` — and the refusal comes in bounded time, because the install + runs in the caller with no timeout of its own. + """ + use ExUnit.Case, async: true + + alias Redist.Support.HttpServer + alias Redist.Targz + + setup do + {:ok, _} = Application.ensure_all_started(:req) + tmp = Path.join(System.tmp_dir!(), "targz-chain-#{System.unique_integer([:positive])}") + File.mkdir_p!(tmp) + on_exit(fn -> File.rm_rf!(tmp) end) + {:ok, server: HttpServer.start(), tmp: tmp} + end + + defp chained_archive(tmp) do + src = Path.join(tmp, "src") + File.mkdir_p!(src) + File.ln_s!(".", Path.join(src, "s")) + File.ln_s!("s/..", Path.join(src, "t")) + + archive = Path.join(tmp, "chain.tar.gz") + {:ok, tar} = :erl_tar.open(String.to_charlist(archive), [:write, :compressed]) + :ok = :erl_tar.add(tar, String.to_charlist(Path.join(src, "s")), ~c"s", []) + :ok = :erl_tar.add(tar, String.to_charlist(Path.join(src, "t")), ~c"t", []) + :ok = :erl_tar.add(tar, "pwned", ~c"t/escape.txt", []) + :ok = :erl_tar.close(tar) + File.read!(archive) + end + + test "a symlink chain out of dest_dir is refused promptly and writes nothing outside", %{ + server: server, + tmp: tmp + } do + bytes = chained_archive(tmp) + url = HttpServer.put_file(server, "chain.tar.gz", bytes) + sha = :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) + root = Path.join(tmp, "root") + dest = Path.join(root, "dest") + + task = Task.async(fn -> Targz.install(url, sha, dest) end) + result = Task.yield(task, :timer.seconds(10)) || Task.shutdown(task, :brutal_kill) + + refute File.exists?(Path.join(root, "escape.txt")) + + assert match?({:ok, {:error, _}}, result), + "install/3 -> #{inspect(result)} (nil: still running after 10s)" + end +end