Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions native/.cargo/mutants.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ exclude_re = [
"guest-agent/src/net\\.rs:.*GuestNet::(from_proc|configure)",
# Equivalent: the VM range tag and a NetId never share bits with a normalised /64.
"replace \\| with \\^ in NodePrefix::(vm_range|guest)$",
# Equivalent: the operands never share a bit (no OpenFlags carries O_EXCL without O_CREAT,
# nor O_DIRECTORY or O_CLOEXEC; the RESOLVE_* flags are distinct bits).
"hyperrs-fs/src/(dir|file)\\.rs:.*replace \\| with \\^ in SafeFile<.*>::(openat|open_at|open_raw)$",
# Equivalent: the dst-nat chain runs after conntrack and before routing at -100 or 100.
"suidhelper/src/net/mod\\.rs:\\d+:\\d+: delete -$",
]
185 changes: 185 additions & 0 deletions native/hyperrs-firecracker/tests/chroot_remove_names.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
// SPDX-License-Identifier: MIT
//! Teardown contract of [`JailLayout::remove`]: it removes the per-VM chroot whatever the jailed
//! Firecracker left in it, and is idempotent.
//!
//! Laws:
//! - the jail root is owned by the jailed uid, so Firecracker can create any legal file name in
//! it; `remove` must still delete the whole `<base>/firecracker/<id>` tree for every such name
//! (whitespace and non-UTF-8 bytes included), or a VM's jail leaks past teardown;
//! - removing an already-removed jail is `Ok`, and sibling jails are never touched;
//! - `remove` succeeds on a tree deeper than the soft fd limit, which a jail process can build
//! holding one fd at a time, and never follows a link planted in the jail out of the tree.

use assert_fs::TempDir;
use hyperrs_firecracker::jailer::chroot::JailLayout;
use hyperrs_firecracker::VmId;
use hyperrs_fs::path::{IsAbsolute, SafePathBuf, StrictComponents};
use nix::fcntl::{openat, OFlag};
use nix::sys::stat::{mkdirat, Mode};
use nix::unistd::mkfifo;
use std::ffi::OsStr;
use std::fs;
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::symlink;
use std::os::unix::net::UnixListener;
use std::path::Path;

type Strict = SafePathBuf<IsAbsolute, StrictComponents>;

fn base(tmp: &TempDir) -> Strict {
fs::canonicalize(tmp.path()).unwrap().try_into().unwrap()
}

#[test]
fn remove_deletes_the_jail_whatever_names_the_guest_side_left_in_it() {
let names: [&[u8]; 9] = [
b"firecracker.log",
b" ",
b"-rf",
&[b'n'; 255],
b"\xff\xfe",
b"two words",
b"line\nbreak",
b"tab\there",
b"trailing ",
];
let leaked: Vec<String> = names
.iter()
.filter_map(|raw| {
let tmp = TempDir::new().unwrap();
let base = base(&tmp);
let id: VmId = "vm-a".parse().unwrap();
let sibling: VmId = "vm-b".parse().unwrap();
let jail = JailLayout::new(&base, &id);
let other = JailLayout::new(&base, &sibling);
jail.create_root().unwrap();
other.create_root().unwrap();
fs::write(jail.root().as_path().join(OsStr::from_bytes(raw)), b"x").unwrap();

let got = jail.remove();
let gone = !jail.chroot_dir().as_path().exists();
assert!(other.root().as_path().is_dir());
(got.is_err() || !gone).then(|| format!("{:?}: {got:?}", String::from_utf8_lossy(raw)))
})
.collect();
assert!(
leaked.is_empty(),
"jail leaked past teardown for {leaked:#?}"
);
}

#[test]
fn remove_is_idempotent_and_spares_siblings() {
let tmp = TempDir::new().unwrap();
let base = base(&tmp);
let id: VmId = "vm-a".parse().unwrap();
let sibling: VmId = "vm-b".parse().unwrap();
let jail = JailLayout::new(&base, &id);
let other = JailLayout::new(&base, &sibling);
jail.create_root().unwrap();
other.create_root().unwrap();
fs::write(other.root().as_path().join("keep"), b"keep").unwrap();

jail.remove().unwrap();
jail.remove().unwrap();

assert!(!jail.chroot_dir().as_path().exists());
assert_eq!(
fs::read(other.root().as_path().join("keep")).unwrap(),
b"keep"
);
}

struct NofileLimit(nix::libc::rlimit);

impl NofileLimit {
fn lower_to(soft: u64) -> Self {
let mut old = nix::libc::rlimit {
rlim_cur: 0,
rlim_max: 0,
};
// SAFETY: getrlimit/setrlimit only read/write the rlimit structs passed in.
unsafe {
assert_eq!(nix::libc::getrlimit(nix::libc::RLIMIT_NOFILE, &mut old), 0);
let new = nix::libc::rlimit {
rlim_cur: soft.min(old.rlim_max),
rlim_max: old.rlim_max,
};
assert_eq!(nix::libc::setrlimit(nix::libc::RLIMIT_NOFILE, &new), 0);
}
Self(old)
}
}

impl Drop for NofileLimit {
fn drop(&mut self) {
// SAFETY: restores the struct getrlimit returned.
unsafe { nix::libc::setrlimit(nix::libc::RLIMIT_NOFILE, &self.0) };
}
}

/// Builds `depth` nested directories below `top` holding only one fd at a time, which is how a
/// jail process can nest past any per-process fd limit.
fn nest(top: &Path, depth: usize) {
let mut cur: OwnedFd = fs::File::open(top).unwrap().into();
for _ in 0..depth {
mkdirat(Some(cur.as_raw_fd()), "d", Mode::from_bits_truncate(0o755)).unwrap();
let next = openat(
Some(cur.as_raw_fd()),
"d",
OFlag::O_RDONLY | OFlag::O_DIRECTORY,
Mode::empty(),
)
.unwrap();
// SAFETY: openat just returned this fd and nothing else owns it.
cur = unsafe { OwnedFd::from_raw_fd(next) };
}
}

#[test]
fn remove_clears_a_tree_deeper_than_the_default_fd_limit() {
let tmp = TempDir::new().unwrap();
let base = base(&tmp);
let id: VmId = "vm-a".parse().unwrap();
let jail = JailLayout::new(&base, &id);
jail.create_root().unwrap();
nest(jail.root().as_path(), 1500);

let got = {
let _limit = NofileLimit::lower_to(1024);
jail.remove()
};

assert!(got.is_ok(), "teardown failed on a 1500-deep tree: {got:?}");
assert!(!jail.chroot_dir().as_path().exists());
}

#[test]
fn remove_never_follows_links_planted_in_the_jail() {
let tmp = TempDir::new().unwrap();
let base = base(&tmp);
let id: VmId = "vm-a".parse().unwrap();
let jail = JailLayout::new(&base, &id);
jail.create_root().unwrap();
let root = jail.root().as_path().to_path_buf();
let outside = tmp.path().join("outside");
fs::create_dir(&outside).unwrap();
fs::write(outside.join("keep.txt"), b"keep").unwrap();
let outside_file = tmp.path().join("outside-file");
fs::write(&outside_file, b"keep").unwrap();

symlink(&outside, root.join("to-dir")).unwrap();
symlink(&outside_file, root.join("to-file")).unwrap();
fs::create_dir(root.join("nested")).unwrap();
symlink(tmp.path(), root.join("nested/up")).unwrap();
fs::hard_link(&outside_file, root.join("hard")).unwrap();
mkfifo(&root.join("fifo"), Mode::from_bits_truncate(0o600)).unwrap();
let _sock = UnixListener::bind(root.join("sock")).unwrap();

jail.remove().unwrap();

assert!(!jail.chroot_dir().as_path().exists());
assert_eq!(fs::read(outside.join("keep.txt")).unwrap(), b"keep");
assert_eq!(fs::read(&outside_file).unwrap(), b"keep");
}
120 changes: 101 additions & 19 deletions native/hyperrs-fs/src/dir.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use super::file::{
Any, DeviceType, FileType, IsDirectory, OpenFlags, Owner, Ownership, SafeFile, ValidationError,
Writability,
};
use super::path::{StrictComponent, StrictComponentBuf};
use super::path::{ComponentWithWhitespace, ComponentWithWhitespaceBuf, StrictComponent};
use nix::dir::{Dir, Type};
use nix::errno::Errno;
use nix::fcntl::{openat2, AtFlags, OFlag, OpenHow, ResolveFlag};
Expand All @@ -28,6 +28,8 @@ pub enum DirError {
Sys(#[from] Errno),
#[error(transparent)]
Io(#[from] io::Error),
#[error("directory moved while it was being removed")]
Moved,
#[error("existing device node is {found}, expected {expected}")]
DeviceMismatch {
expected: DeviceNumber,
Expand All @@ -41,29 +43,41 @@ impl DirError {
match self {
DirError::Open(ValidationError::Open(errno) | ValidationError::Fstat(errno))
| DirError::Sys(errno) => Some(*errno),
DirError::Open(_) | DirError::Io(_) | DirError::DeviceMismatch { .. } => None,
DirError::Open(_)
| DirError::Io(_)
| DirError::Moved
| DirError::DeviceMismatch { .. } => None,
}
}
}

type FileId = (libc::dev_t, libc::ino_t);

struct Ancestor {
id: FileId,
name: ComponentWithWhitespaceBuf,
subdirs: Vec<ComponentWithWhitespaceBuf>,
}

/// See [`SafeFile<IsDirectory, R, O>`].
pub type SafeDir = SafeFile<IsDirectory, Any, Any>;

impl<R: Ownership, O: Writability> SafeFile<IsDirectory, R, O> {
/// Open `name` relative to this directory and verify it as a [`SafeFile<T, R2, O2>`].
/// Open `name` relative to this directory and verify it as a [`SafeFile<T, R2, O2>`]. As with
/// [`SafeFile::open`], `flags` apply only after the checks pass.
pub fn openat<T: FileType, R2: Ownership, O2: Writability>(
&self,
name: impl AsRef<Path>,
flags: OpenFlags,
mode: Mode,
) -> Result<SafeFile<T, R2, O2>, ValidationError> {
let how = OpenHow::new()
.flags(OFlag::from(flags) | T::OPEN_FLAGS | OFlag::O_CLOEXEC)
.mode(mode)
.resolve(ResolveFlag::RESOLVE_NO_SYMLINKS | ResolveFlag::RESOLVE_BENEATH);
let raw = openat2(self.as_raw_fd(), name.as_ref(), how).map_err(ValidationError::Open)?;
// SAFETY: openat2 just handed us this fd; nobody else owns it.
SafeFile::try_from(unsafe { OwnedFd::from_raw_fd(raw) })
SafeFile::open_at(
self.as_raw_fd(),
name.as_ref(),
flags,
mode,
ResolveFlag::RESOLVE_NO_SYMLINKS | ResolveFlag::RESOLVE_BENEATH,
)
}

/// Create file `name` in this directory, failing if it exists.
Expand Down Expand Up @@ -154,7 +168,7 @@ impl<R: Ownership, O: Writability> SafeFile<IsDirectory, R, O> {
///
/// Note that this must remain private as this function is a footgun. It's very dangerous and
/// risky to use this function as it causes an array of TOCTOU bugs.
fn statat(&self, name: &StrictComponent) -> nix::Result<FileStat> {
fn statat(&self, name: &ComponentWithWhitespace) -> nix::Result<FileStat> {
let how = OpenHow::new()
.flags(OFlag::O_PATH | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC)
.resolve(ResolveFlag::RESOLVE_NO_SYMLINKS | ResolveFlag::RESOLVE_BENEATH);
Expand Down Expand Up @@ -183,22 +197,77 @@ impl<R: Ownership, O: Writability> SafeFile<IsDirectory, R, O> {
}

/// Recursively, without following symlinks, remove child `name` and everything under it.
/// Holds a bounded number of fds whatever the depth. Fails with [`DirError::Moved`] if a
/// directory is moved out of the tree mid-walk.
pub fn rmdirat_recursive(&self, name: &StrictComponent) -> Result<(), DirError> {
let child: SafeDir = self.openat(name, OpenFlags::RDONLY, Mode::empty())?;
let entries = child.entries()?.collect::<nix::Result<Vec<_>>>()?;
for (entry, ty) in entries {
match ty {
Type::Directory => child.rmdirat_recursive(&entry)?,
_ => child.unlinkat(&entry)?,
// Walk with no fd held per level, so depth is bounded by memory rather than
// RLIMIT_NOFILE; climbing through `..` is confined by matching the recorded parent inode.
let mut dir: SafeDir = self.openat(name, OpenFlags::RDONLY, Mode::empty())?;
let mut subdirs = dir.unlink_non_dirs()?;
let mut ancestors: Vec<Ancestor> = Vec::new();
loop {
if let Some(sub) = subdirs.pop() {
let child: SafeDir = dir.openat(&sub, OpenFlags::RDONLY, Mode::empty())?;
ancestors.push(Ancestor {
id: dir.id()?,
name: sub,
subdirs: std::mem::replace(&mut subdirs, child.unlink_non_dirs()?),
});
dir = child;
} else if let Some(ancestor) = ancestors.pop() {
let parent = dir.parent()?;
if parent.id()? != ancestor.id {
return Err(DirError::Moved);
}
unlinkat(
Some(parent.as_raw_fd()),
ancestor.name.as_path(),
UnlinkatFlags::RemoveDir,
)?;
dir = parent;
subdirs = ancestor.subdirs;
} else {
break;
}
}
drop(dir);
Ok(self.rmdirat(name)?)
}

fn unlink_non_dirs(&self) -> Result<Vec<ComponentWithWhitespaceBuf>, DirError> {
let mut subdirs = Vec::new();
for entry in self.entries()? {
match entry? {
(name, Type::Directory) => subdirs.push(name),
(name, _) => unlinkat(
Some(self.as_raw_fd()),
name.as_path(),
UnlinkatFlags::NoRemoveDir,
)?,
}
}
Ok(subdirs)
}

fn parent(&self) -> Result<SafeDir, ValidationError> {
SafeFile::open_at(
self.as_raw_fd(),
Path::new(".."),
OpenFlags::RDONLY,
Mode::empty(),
ResolveFlag::RESOLVE_NO_SYMLINKS,
)
}

fn id(&self) -> nix::Result<FileId> {
fstat(self.as_raw_fd()).map(|stat| (stat.st_dev, stat.st_ino))
}

/// Get an iterator through all the entries in this directory.
pub fn entries(
&self,
) -> Result<impl Iterator<Item = nix::Result<(StrictComponentBuf, Type)>> + '_, DirError> {
) -> Result<impl Iterator<Item = nix::Result<(ComponentWithWhitespaceBuf, Type)>> + '_, DirError>
{
let fresh: SafeDir = self.openat(Path::new("."), OpenFlags::RDONLY, Mode::empty())?;

Ok(Dir::from(OwnedFd::from(fresh))?
Expand All @@ -214,7 +283,7 @@ impl<R: Ownership, O: Writability> SafeFile<IsDirectory, R, O> {
return None;
}

let name: &StrictComponent = match OsStr::from_bytes(bytes).try_into() {
let name: &ComponentWithWhitespace = match OsStr::from_bytes(bytes).try_into() {
Ok(name) => name,
Err(_) => return Some(Err(Errno::EINVAL)),
};
Expand Down Expand Up @@ -378,4 +447,17 @@ mod tests {
]);
assert_eq!(got, want);
}

#[rstest]
fn id_names_one_directory_however_it_is_reached(tmp: TempDir) {
tmp.child("a/child").create_dir_all().unwrap();
tmp.child("b").create_dir_all().unwrap();
let root = open(tmp.path());
let a: SafeDir = root.openat("a", OpenFlags::RDONLY, Mode::empty()).unwrap();
let b: SafeDir = root.openat("b", OpenFlags::RDONLY, Mode::empty()).unwrap();
let child: SafeDir = a.openat("child", OpenFlags::RDONLY, Mode::empty()).unwrap();

assert_eq!(child.parent().unwrap().id().unwrap(), a.id().unwrap());
assert_ne!(a.id().unwrap(), b.id().unwrap());
}
}
Loading
Loading