From 5a2f2832476c47c8424d59d1931c9e79d66c42b3 Mon Sep 17 00:00:00 2001 From: ilovecrayons Date: Fri, 13 Nov 2026 02:32:12 +0000 Subject: [PATCH 1/4] feat(runtime): run sessions in Cloud Hypervisor VMs behind a setting Before, every session ran in a Docker container. With the new server setting session-runtime=vm, the server runs each session in its own Cloud Hypervisor VM through internal/runtime/vm, while environment terminals, the assistant, the agent updater, the cleanup helper, the browser companion and image operations stay on Docker until work item 9. The default, docker, changes nothing. Each VM's monitor runs in its own systemd unit in aether.slice, so a VM survives server restarts and updates; the server finds it again from /vm//vm.json. Mounts reach the guest over one virtio-fs share composed from host bind mounts inside the unit's private mount namespace; host sockets are forwarded over vsock. Session preparation helpers and the dev-exec control parser move out of the Docker files unchanged so both engines share them. The OOM watcher asks the runtime for kills when it can report guest ones itself. --- cmd/aether-server/main.go | 12 + docs/environments.md | 40 ++ internal/runtime/docker.go | 124 +---- internal/runtime/docker_integration_test.go | 2 +- internal/runtime/docker_managed_exec.go | 27 +- internal/runtime/docker_sudo.go | 41 +- internal/runtime/docker_test.go | 18 +- internal/runtime/managed_exec.go | 39 ++ internal/runtime/runtime.go | 22 + internal/runtime/session.go | 149 ++++++ internal/runtime/vm/chapi.go | 98 ++++ internal/runtime/vm/engine.go | 448 ++++++++++++++++++ internal/runtime/vm/engine_test.go | 438 ++++++++++++++++++ internal/runtime/vm/exec.go | 233 ++++++++++ internal/runtime/vm/fakes_test.go | 489 ++++++++++++++++++++ internal/runtime/vm/helper.go | 54 +++ internal/runtime/vm/kvm_integration_test.go | 216 +++++++++ internal/runtime/vm/lifecycle.go | 312 +++++++++++++ internal/runtime/vm/managed.go | 259 +++++++++++ internal/runtime/vm/monitor.go | 30 ++ internal/runtime/vm/monitor_linux.go | 148 ++++++ internal/runtime/vm/monitor_other.go | 9 + internal/runtime/vm/network.go | 150 ++++++ internal/runtime/vm/network_test.go | 84 ++++ internal/runtime/vm/record.go | 149 ++++++ internal/runtime/vm/save.go | 161 +++++++ internal/runtime/vm/share.go | 120 +++++ internal/runtime/vm/share_test.go | 90 ++++ internal/runtime/vm/stats.go | 114 +++++ internal/runtime/vm/unit.go | 81 ++++ internal/runtime/vm/vm.go | 270 +++++++++++ internal/runtime/vm/vm_test.go | 78 ++++ internal/scheduler/oomwatch.go | 6 + internal/scheduler/oomwatch_test.go | 43 ++ internal/server/server.go | 23 + internal/server/vmruntime.go | 31 ++ 36 files changed, 4422 insertions(+), 186 deletions(-) create mode 100644 internal/runtime/session.go create mode 100644 internal/runtime/vm/chapi.go create mode 100644 internal/runtime/vm/engine.go create mode 100644 internal/runtime/vm/engine_test.go create mode 100644 internal/runtime/vm/exec.go create mode 100644 internal/runtime/vm/fakes_test.go create mode 100644 internal/runtime/vm/helper.go create mode 100644 internal/runtime/vm/kvm_integration_test.go create mode 100644 internal/runtime/vm/lifecycle.go create mode 100644 internal/runtime/vm/managed.go create mode 100644 internal/runtime/vm/monitor.go create mode 100644 internal/runtime/vm/monitor_linux.go create mode 100644 internal/runtime/vm/monitor_other.go create mode 100644 internal/runtime/vm/network.go create mode 100644 internal/runtime/vm/network_test.go create mode 100644 internal/runtime/vm/record.go create mode 100644 internal/runtime/vm/save.go create mode 100644 internal/runtime/vm/share.go create mode 100644 internal/runtime/vm/share_test.go create mode 100644 internal/runtime/vm/stats.go create mode 100644 internal/runtime/vm/unit.go create mode 100644 internal/runtime/vm/vm.go create mode 100644 internal/runtime/vm/vm_test.go create mode 100644 internal/server/vmruntime.go diff --git a/cmd/aether-server/main.go b/cmd/aether-server/main.go index b8c660ac7..1767fdda9 100644 --- a/cmd/aether-server/main.go +++ b/cmd/aether-server/main.go @@ -26,6 +26,7 @@ import ( "github.com/3xDevOps/Aether/internal/guest/helper" "github.com/3xDevOps/Aether/internal/harness" "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/runtime/vm" "github.com/3xDevOps/Aether/internal/scheduler" "github.com/3xDevOps/Aether/internal/server" "github.com/3xDevOps/Aether/internal/serversetup" @@ -50,6 +51,9 @@ func main() { if len(os.Args) > 1 && os.Args[1] == "guest" { os.Exit(helper.Main()) } + if len(os.Args) > 1 && os.Args[1] == vm.MonitorCommand { + exitOn(vm.Monitor(os.Args[2:])) + } if filepath.Base(os.Args[0]) == "aether-internal" { os.Exit(coordcli.Main(os.Args[1:])) } @@ -251,6 +255,8 @@ type serveOptions struct { runMemory *runCountValue runPids *runCountValue containerRuntime *string + sessionRuntime *string + vmKernel *string } // serveFlags declares the server options on fs. It is the single definition @@ -312,6 +318,10 @@ func serveFlags(fs *flag.FlagSet) *serveOptions { o.containerRuntime = fs.String("container-runtime", server.DefaultContainerRuntime, "OCI runtime for run and member environment containers: auto uses sysbox-runc when Docker has it registered "+ "(Docker inside sessions without --privileged), otherwise the daemon default; any other value is used as given, e.g. runc to opt out") + o.sessionRuntime = fs.String("session-runtime", server.SessionRuntimeDocker, + "where sessions run: docker, or vm for one Cloud Hypervisor VM per session (needs /dev/kvm, cloud-hypervisor and virtiofsd; "+ + "every other container stays on Docker)") + o.vmKernel = fs.String("vm-kernel", server.DefaultVMKernel, "the guest kernel session VMs boot with -session-runtime vm") return o } @@ -393,6 +403,8 @@ func serve(args []string) error { RunPidsLimit: int64(*o.runPids), ContainerRuntime: *o.containerRuntime, + SessionRuntime: *o.sessionRuntime, + VMKernel: *o.vmKernel, SysboxInstaller: true, }) if err != nil { diff --git a/docs/environments.md b/docs/environments.md index daa4be68f..5441b4e33 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -314,6 +314,46 @@ pressure](failure-handling.md#recover-from-disk-pressure) and [Recover from memory pressure](failure-handling.md#recover-from-memory-pressure) have the thresholds and what to do. +## Sessions in virtual machines + +`--session-runtime` (config key `session-runtime`) defaults to `docker`. With +`vm`, each new session runs in its own [Cloud +Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor) virtual +machine instead of a container. Environment terminals, the assistant, the +agent updater, the development browser and **Save environment** stay on +Docker. +This is work in progress; [the plan](plans/2026-11-13-microvm-sessions.md) +lists what is still to come. + +```sh +sudo aether-server config set session-runtime vm +sudo systemctl restart aether-server +``` + +The host needs `/dev/kvm`, `cloud-hypervisor`, `/usr/libexec/virtiofsd`, +`qemu-img`, `mkfs.ext4`, `iptables` and systemd, besides Docker, and the +release's guest kernel `aether-vmlinux-x86_64` at `/usr/local/lib/aether/vmlinux` +(`--vm-kernel`, config key `vm-kernel`, names another path). + +What changes: + +- Each VM runs in the systemd unit `aether-vm-.service` in + `aether.slice`, so it keeps running through a server restart or update. + Its disks and logs are in `/vm//`: `monitor.log` holds the + VM monitor's output and `console.log` the guest kernel's. +- The VM gets the CPU limit rounded up to whole vCPUs, held to the exact + limit by the unit's `CPUQuota`, and the memory limit as its memory. The + process limit applies inside the guest. +- The checkout, the home and the other mounts reach the VM over one virtio-fs + share; read-only mounts stay read-only. Each VM has its own address in + `10.200.0.0/16` and reaches the internet through the host. +- Starting a stopped session builds a fresh root disk from its image and runs + the setup script again: packages installed outside the mounts are gone. + Docker images inside the session stay on its data disk. +- Not yet supported, refused at launch with an error naming the work item: + the Isolated tier and egress control, and the development browser beside a + session. + ## After a session closes Reporting success or failure leaves an interactive session open for follow-up, and diff --git a/internal/runtime/docker.go b/internal/runtime/docker.go index e62508455..038be5300 100644 --- a/internal/runtime/docker.go +++ b/internal/runtime/docker.go @@ -15,7 +15,6 @@ import ( "math" "path/filepath" "slices" - "strconv" "strings" "sync" "time" @@ -55,13 +54,6 @@ const ( setupSentinelPrefix = "/tmp/.aether-setup-" ) -// execOutputLimit covers stdout and stderr together. -const execOutputLimit = 1 << 20 - -// setupOutputLimit is how much of a setup script's output is kept, from the -// end, where the failure is. -const setupOutputLimit = 16 << 10 - // hijackWriteTimeout bounds one stdin write. Cancellation interrupts a write // only through its deadline; it never closes the shared stdin stream. const hijackWriteTimeout = 5 * time.Second @@ -290,7 +282,7 @@ func (d *Docker) containerConfig(spec Spec, containerRuntime string) (*container cfg := &container.Config{ Image: spec.Image, User: spec.User, - Env: dockerEnv(trustCheckouts(spec.Env, spec.WorktreeMountPath, spec.ExtraWorktrees)), + Env: EnvList(TrustCheckouts(spec.Env, spec.WorktreeMountPath, spec.ExtraWorktrees)), WorkingDir: spec.WorkingDir, Entrypoint: spec.Command, Labels: labels, @@ -313,7 +305,7 @@ func (d *Docker) containerConfig(spec Spec, containerRuntime string) (*container nestedDockerd := spec.NestedDocker && containerRuntime == runtimeSysbox // Gated so the agent starts only once Start has checked every subpath mount. subpaths := slices.ContainsFunc(spec.Mounts, func(m Mount) bool { return m.Subpath != "" }) - if spec.SetupScript != "" || nestedDockerd || subpaths || unprivileged(spec.User) { + if spec.SetupScript != "" || nestedDockerd || subpaths || Unprivileged(spec.User) { sentinel := newSetupSentinel() cfg.Entrypoint = gateEntrypoint(sentinel) cfg.Cmd = spec.Command @@ -486,47 +478,6 @@ func gateEntrypoint(sentinel string) []string { return []string{"/bin/sh", "-c", script, "aether-gate"} } -func dockerEnv(env map[string]string) []string { - if len(env) == 0 { - return nil - } - out := make([]string, 0, len(env)) - for _, k := range slices.Sorted(maps.Keys(env)) { - out = append(out, k+"="+env[k]) - } - return out -} - -func trustCheckouts(env map[string]string, checkout string, extras []Mount) map[string]string { - env = trustCheckout(env, checkout) - for _, m := range extras { - env = trustCheckout(env, m.ContainerPath) - } - return env -} - -// Execs inherit the container env, so this also covers every git the server -// runs inside the container. -func trustCheckout(env map[string]string, checkout string) map[string]string { - if checkout == "" { - return env - } - n := 0 - if count, ok := env["GIT_CONFIG_COUNT"]; ok { - parsed, err := strconv.Atoi(count) - if err != nil || parsed < 0 { - return env - } - n = parsed - } - out := make(map[string]string, len(env)+3) - maps.Copy(out, env) - out["GIT_CONFIG_KEY_"+strconv.Itoa(n)] = "safe.directory" - out["GIT_CONFIG_VALUE_"+strconv.Itoa(n)] = checkout - out["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1) - return out -} - func nanoCPUs(cores float64) int64 { return int64(math.Round(cores * 1e9)) } @@ -564,7 +515,7 @@ func (d *Docker) Start(ctx context.Context, id ID) error { return nil } secrets := secretEnvValues(labels[labelSecretEnv], info.Container.Config.Env) - if err := d.prepare(ctx, id, labels, sentinel, info.Container.Config.WorkingDir, info.Container.Config.User, envValue(info.Container.Config.Env, "HOME"), secrets); err != nil { + if err := d.prepare(ctx, id, labels, sentinel, info.Container.Config.WorkingDir, info.Container.Config.User, EnvValue(info.Container.Config.Env, "HOME"), secrets); err != nil { // Never kill a run that was already live before this Start call: a // redundant Start must not take down a healthy container. if !wasRunning { @@ -580,7 +531,7 @@ func (d *Docker) Start(ctx context.Context, id ID) error { } func (d *Docker) prepare(ctx context.Context, id ID, labels map[string]string, sentinel, workDir, user, home string, secrets []string) error { - if unprivileged(user) { + if Unprivileged(user) { if err := d.prepareUser(ctx, id, user, home, labels[labelSudo] != ""); err != nil { return err } @@ -641,7 +592,7 @@ func (d *Docker) startDockerd(ctx context.Context, id ID, user string) error { ctx, cancel := context.WithTimeout(ctx, dockerdStartTimeout) defer cancel() code, output, cut, err := d.execCombined(ctx, id, []string{"/bin/sh", "-c", dockerdScript, "aether-dockerd", user}, "", "0:0") - output = setupOutput(output, cut, nil) + output = SetupOutput(output, cut, nil) if err != nil { slog.Error("runtime: dockerd start failed", "container", id, "output", output, "error", err) return fmt.Errorf("runtime: start dockerd: %w", err) @@ -669,7 +620,7 @@ func (d *Docker) runSetup(ctx context.Context, id ID, script, sentinel, workDir d.settingUp.Store(id, struct{}{}) code, output, cut, err := d.execCombined(ctx, id, []string{"/bin/sh", "-ec", "exec 2>&1; " + script}, workDir, "") d.settingUp.Delete(id) - output = setupOutput(output, cut, secrets) + output = SetupOutput(output, cut, secrets) if err != nil { slog.Error("runtime: setup script execution failed", "container", id, "working_dir", workDir, "script", script, "output", output, "error", err) return fmt.Errorf("runtime: setup script: %w", err) @@ -695,51 +646,6 @@ func (d *Docker) releaseGate(ctx context.Context, id ID, sentinel string) error return nil } -func secretEnvValues(label string, env []string) []string { - var names []string - if json.Unmarshal([]byte(label), &names) != nil { - return nil - } - var values []string - for _, pair := range env { - name, value, _ := strings.Cut(pair, "=") - if value != "" && slices.Contains(names, name) { - values = append(values, value) - } - } - return values -} - -const secretMask = "***" - -// setupOutput masks exact secret values, longest first so a secret holding -// another is masked whole, then keeps the last setupOutputLimit bytes. cut -// says output already lost its beginning, which may have split a secret: the -// span that could hold its remainder is dropped. That span is measured after -// masking, where a shorter secret inside the remainder can have grown it to -// len(secretMask) times its length. -func setupOutput(output string, cut bool, secrets []string) string { - if len(secrets) > 0 { - slices.SortFunc(secrets, func(a, b string) int { return len(b) - len(a) }) - pairs := make([]string, 0, 2*len(secrets)) - for _, secret := range secrets { - pairs = append(pairs, secret, secretMask) - } - output = strings.NewReplacer(pairs...).Replace(output) - if cut { - output = output[min(len(secretMask)*len(secrets[0]), len(output)):] - } - } - if cut || len(output) > setupOutputLimit { - output = output[max(0, len(output)-setupOutputLimit):] - if i := strings.IndexByte(output, '\n'); i >= 0 { - output = output[i+1:] - } - output = "[earlier output omitted]\n" + output - } - return strings.ToValidUTF8(output, "\uFFFD") -} - // tailWriter keeps the last limit bytes written to it. The buffer grows to // twice that before it is compacted, so a script printing line by line does // not cost a copy of the whole buffer per line. @@ -764,9 +670,9 @@ func (w *tailWriter) tail() (kept string, cut bool) { } // execCombined is Exec for the setup script, which may print without limit: -// one transcript, of which the last execOutputLimit bytes are kept. +// one transcript, of which the last ExecOutputLimit bytes are kept. func (d *Docker) execCombined(ctx context.Context, id ID, cmd []string, workDir, user string) (code int, output string, cut bool, err error) { - kept := &tailWriter{limit: execOutputLimit} + kept := &tailWriter{limit: ExecOutputLimit} code, err = d.exec(ctx, id, cmd, workDir, user, nil, func(r io.Reader) error { _, copyErr := stdcopy.StdCopy(kept, kept, r) return copyErr @@ -786,10 +692,6 @@ func (d *Docker) Exec(ctx context.Context, id ID, cmd []string, workDir string) // exec writes a non-nil stdin to the command's input, which stays open until // exec returns. -// rootPath is sudo's secure_path. A root exec never searches the -// container's PATH, which puts the agent-writable ~/.local/bin first. -const rootPath = "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - func (d *Docker) exec(ctx context.Context, id ID, cmd []string, workDir, user string, stdin []byte, read func(io.Reader) error) (int, error) { opts := client.ExecCreateOptions{ Cmd: cmd, @@ -800,7 +702,7 @@ func (d *Docker) exec(ctx context.Context, id ID, cmd []string, workDir, user st AttachStderr: true, } if uid, _, _ := strings.Cut(user, ":"); uid == "0" || uid == "root" { - opts.Env = []string{rootPath} + opts.Env = []string{RootPath} } created, err := d.cli.ExecCreate(ctx, string(id), opts) if err != nil { @@ -850,16 +752,16 @@ func (d *Docker) exec(ctx context.Context, id ID, cmd []string, workDir, user st } } -// readExecOutput refuses output past execOutputLimit: the container chooses +// readExecOutput refuses output past ExecOutputLimit: the container chooses // what it prints and the caller buffers it all. func readExecOutput(r io.Reader) (stdout, stderr string, err error) { // One byte past the cap, so an output of exactly that size is not // mistaken for a truncated one. - counted := &countingReader{r: io.LimitReader(r, execOutputLimit+1)} + counted := &countingReader{r: io.LimitReader(r, ExecOutputLimit+1)} var out, errOut bytes.Buffer _, copyErr := stdcopy.StdCopy(&out, &errOut, counted) - if counted.read > execOutputLimit { - return out.String(), errOut.String(), fmt.Errorf("output passed the %d byte limit", execOutputLimit) + if counted.read > ExecOutputLimit { + return out.String(), errOut.String(), fmt.Errorf("output passed the %d byte limit", ExecOutputLimit) } return out.String(), errOut.String(), copyErr } diff --git a/internal/runtime/docker_integration_test.go b/internal/runtime/docker_integration_test.go index e607b8ea7..e099610f3 100644 --- a/internal/runtime/docker_integration_test.go +++ b/internal/runtime/docker_integration_test.go @@ -343,7 +343,7 @@ func TestDockerSetupFailure(t *testing.T) { } // TestDockerSetupOutputPastTheBuffer verifies a setup script may print more -// than execOutputLimit: a passing one still starts the container, and a +// than ExecOutputLimit: a passing one still starts the container, and a // failing one reports the end of what it printed. func TestDockerSetupOutputPastTheBuffer(t *testing.T) { t.Parallel() diff --git a/internal/runtime/docker_managed_exec.go b/internal/runtime/docker_managed_exec.go index fd09809ef..f2abbb088 100644 --- a/internal/runtime/docker_managed_exec.go +++ b/internal/runtime/docker_managed_exec.go @@ -4,13 +4,11 @@ import ( "context" "crypto/rand" "encoding/hex" - "encoding/json" "errors" "fmt" "io" "path" "slices" - "strconv" "strings" "sync" "time" @@ -199,36 +197,15 @@ func (e *dockerManagedExec) forgotten(ctx context.Context, cause error) (devexec } func (e *dockerManagedExec) control(ctx context.Context, action string, grace time.Duration) (devexec.State, error) { - args := []string{coordtransport.CLIPath, devexec.Command, "control", e.identity.CreationKey, e.identity.ExecID, e.identity.ClaimToken, action, strconv.FormatInt(grace.Milliseconds(), 10)} var stdout, stderr string - code, err := e.docker.exec(ctx, e.identity.ContainerID, args, "", e.identity.User, nil, func(r io.Reader) (copyErr error) { + code, err := e.docker.exec(ctx, e.identity.ContainerID, ExecControlArgv(e.identity, action, grace), "", e.identity.User, nil, func(r io.Reader) (copyErr error) { stdout, stderr, copyErr = readExecOutput(r) return copyErr }) if err != nil { return devexec.State{}, err } - var state devexec.State - decodeErr := json.Unmarshal([]byte(stdout), &state) - if decodeErr == nil && state.ExecID == e.identity.ExecID && state.CreationKey == e.identity.CreationKey && state.ClaimToken == e.identity.ClaimToken { - if state.Error != "" { - if state.Exited && state.ExitCode != nil && (*state.ExitCode == 126 || *state.ExitCode == 127) { - if action == "start" { - return state, &ExecExitError{Code: *state.ExitCode} - } - return state, nil - } - return state, errors.New(state.Error) - } - if code == 0 { - return state, nil - } - } - failed := fmt.Errorf("%w: helper %s failed (status %d): %s", ErrExecUnavailable, action, code, strings.TrimSpace(stderr)) - if strings.Contains(stderr, devexec.ErrNoSupervisor.Error()) { - return state, fmt.Errorf("%w: %w", devexec.ErrNoSupervisor, failed) - } - return state, failed + return ExecControlState(e.identity, action, code, stdout, stderr) } func (e *dockerManagedExec) Status(ctx context.Context) (ExecState, error) { diff --git a/internal/runtime/docker_sudo.go b/internal/runtime/docker_sudo.go index 324d88020..8e5d7fde7 100644 --- a/internal/runtime/docker_sudo.go +++ b/internal/runtime/docker_sudo.go @@ -14,42 +14,12 @@ import ( "github.com/3xDevOps/Aether/internal/devexec" ) -// userScript gives uid $1 a passwd entry whose home is $2: git refuses to -// commit and ssh and ssh-keygen refuse to run for a uid with none, and ssh -// reads ~/.ssh from the passwd home, never $HOME. With $3 set it also lets -// the uid sudo without a password; the sudoers file is checked before it -// goes live, since sudo refuses to run at all with a broken one. -// -// The home at /root is the member's, shared by every container they run, -// and systemd's postinst runs systemd-tmpfiles, whose provision.conf hands -// /root to root: installing any package that pulls in systemd would take -// the home from all of them. The /etc entry masks it. -const userScript = `uid=${1%%:*} gid=${1#*:} -if grep -q "^[^:]*:[^:]*:$uid:" /etc/passwd; then - awk -F: -v OFS=: -v u="$uid" -v h="$2" '$3 == u { $6 = h } { print }' /etc/passwd > /etc/passwd.aether && cat /etc/passwd.aether > /etc/passwd && rm -f /etc/passwd.aether || exit 1 -else - echo "aether:x:$uid:$gid::$2:/bin/sh" >> /etc/passwd || exit 1 -fi -grep -q "^[^:]*:[^:]*:$gid:" /etc/group || echo "aether:x:$gid:" >> /etc/group || exit 1 -mkdir -p /etc/tmpfiles.d && ln -sfn /dev/null /etc/tmpfiles.d/provision.conf || exit 1 -[ -n "$3" ] || exit 0 -mkdir -p /etc/sudoers.d -printf '#%s ALL=(ALL:ALL) NOPASSWD: ALL\n' "$uid" > /etc/sudoers.d/.aether.new -chmod 0440 /etc/sudoers.d/.aether.new -if command -v visudo >/dev/null 2>&1 && ! visudo -cqf /etc/sudoers.d/.aether.new; then rm -f /etc/sudoers.d/.aether.new; exit 1; fi -mv -f /etc/sudoers.d/.aether.new /etc/sudoers.d/aether` - -func unprivileged(user string) bool { - uid, _, _ := strings.Cut(user, ":") - return uid != "" && uid != "0" -} - func (d *Docker) prepareUser(ctx context.Context, id ID, user, home string, sudo bool) error { allow := "" if sudo { allow = "sudo" } - code, output, _, err := d.execCombined(ctx, id, []string{"/bin/sh", "-c", userScript, "aether-user", user, cmp.Or(home, "/"), allow}, "", "0:0") + code, output, _, err := d.execCombined(ctx, id, []string{"/bin/sh", "-c", UserScript, "aether-user", user, cmp.Or(home, "/"), allow}, "", "0:0") if err != nil { return fmt.Errorf("runtime: prepare user %s: %w", user, err) } @@ -80,12 +50,3 @@ func (d *Docker) ExecAttached(ctx context.Context, id ID, spec AttachedExec) (in return err }) } - -func envValue(env []string, name string) string { - for _, pair := range env { - if key, value, _ := strings.Cut(pair, "="); key == name { - return value - } - } - return "" -} diff --git a/internal/runtime/docker_test.go b/internal/runtime/docker_test.go index b367ad447..29afb20ce 100644 --- a/internal/runtime/docker_test.go +++ b/internal/runtime/docker_test.go @@ -314,7 +314,7 @@ func TestSetupOutputMasksSecretEnv(t *testing.T) { cfg, _ := d.containerConfig(spec, "") secrets := secretEnvValues(cfg.Labels[labelSecretEnv], cfg.Env) - got := setupOutput("+ curl -H abc-and-more\nmode plain, token abc\n", false, secrets) + got := SetupOutput("+ curl -H abc-and-more\nmode plain, token abc\n", false, secrets) if want := "+ curl -H ***\nmode plain, token ***\n"; got != want { t.Errorf("setupOutput = %q, want %q", got, want) } @@ -327,7 +327,7 @@ func TestSetupOutputMasksSecretEnv(t *testing.T) { func TestSetupOutputKeepsTheEnd(t *testing.T) { long := strings.Repeat("noise\n", setupOutputLimit) + "npm error missing script\n" - got := setupOutput(long, false, nil) + got := SetupOutput(long, false, nil) if len(got) > setupOutputLimit+64 || !strings.HasPrefix(got, "[earlier output omitted]\nnoise\n") || !strings.HasSuffix(got, "npm error missing script\n") { t.Errorf("setupOutput kept %d bytes, starting %q", len(got), got[:min(len(got), 40)]) } @@ -354,12 +354,12 @@ func TestSetupOutputKeepsTheEnd(t *testing.T) { if !cut || !strings.HasPrefix(text, "et-value\n") { t.Fatalf("tail = %q, cut %v, want it to start inside the secret", text, cut) } - if got := setupOutput(text, cut, []string{"secret-value"}); strings.Contains(got, "value") || !strings.HasSuffix(got, "last line\n") { + if got := SetupOutput(text, cut, []string{"secret-value"}); strings.Contains(got, "value") || !strings.HasSuffix(got, "last line\n") { t.Errorf("setupOutput after a cut inside the secret = %q", got) } // Nor when a shorter secret, masked inside that second half, makes it longer. grown := "qqqqqq\nZ9\n" + strings.Repeat("padding line\n", 4) + "last line\n" - if got := setupOutput(grown, true, []string{"q", "qqqqqqqq\nZ9"}); strings.Contains(got, "Z9") || !strings.HasSuffix(got, "last line\n") { + if got := SetupOutput(grown, true, []string{"q", "qqqqqqqq\nZ9"}); strings.Contains(got, "Z9") || !strings.HasSuffix(got, "last line\n") { t.Errorf("setupOutput after a cut inside a secret that holds another = %q", got) } } @@ -397,7 +397,7 @@ func TestContainerConfigNoMountNoLimits(t *testing.T) { func TestTrustCheckoutKeepsWorkspaceGitConfig(t *testing.T) { env := map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "core.editor", "GIT_CONFIG_VALUE_0": "vi"} - got := dockerEnv(trustCheckout(env, "/workspace")) + got := EnvList(trustCheckout(env, "/workspace")) want := []string{"GIT_CONFIG_COUNT=2", "GIT_CONFIG_KEY_0=core.editor", "GIT_CONFIG_KEY_1=safe.directory", "GIT_CONFIG_VALUE_0=vi", "GIT_CONFIG_VALUE_1=/workspace"} if !slices.Equal(got, want) { t.Errorf("env = %v, want %v", got, want) @@ -408,7 +408,7 @@ func TestTrustCheckoutKeepsWorkspaceGitConfig(t *testing.T) { } func TestDockerEnvSorted(t *testing.T) { - got := dockerEnv(map[string]string{"ZED": "1", "ALPHA": "2", "MID": "3"}) + got := EnvList(map[string]string{"ZED": "1", "ALPHA": "2", "MID": "3"}) want := []string{"ALPHA=2", "MID=3", "ZED=1"} if !slices.Equal(got, want) { t.Errorf("dockerEnv = %v, want %v", got, want) @@ -694,7 +694,7 @@ func TestReadExecOutput(t *testing.T) { var flood bytes.Buffer chunk := bytes.Repeat([]byte("y"), 64<<10) - for flood.Len() <= execOutputLimit+1 { + for flood.Len() <= ExecOutputLimit+1 { if writeErr := writeExecFrame(&flood, stdcopy.Stdout, chunk); writeErr != nil { t.Fatal(writeErr) } @@ -703,8 +703,8 @@ func TestReadExecOutput(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "1048576 byte limit") { t.Fatalf("readExecOutput on a flood = %v, want the cap named", err) } - if len(stdout) > execOutputLimit { - t.Errorf("buffered %d bytes, want no more than %d", len(stdout), execOutputLimit) + if len(stdout) > ExecOutputLimit { + t.Errorf("buffered %d bytes, want no more than %d", len(stdout), ExecOutputLimit) } } diff --git a/internal/runtime/managed_exec.go b/internal/runtime/managed_exec.go index e6ce40030..8543a1efc 100644 --- a/internal/runtime/managed_exec.go +++ b/internal/runtime/managed_exec.go @@ -2,8 +2,15 @@ package runtime import ( "context" + "encoding/json" "errors" + "fmt" + "strconv" + "strings" "time" + + "github.com/3xDevOps/Aether/internal/coordtransport" + "github.com/3xDevOps/Aether/internal/devexec" ) // ErrExecUnavailable means execution identity or its terminal cannot be proved. @@ -83,3 +90,35 @@ type ManagedExec interface { Resize(context.Context, uint, uint) error Detach() error } + +// ExecControlArgv is the staged helper's control command for one action on +// the execution identity names. +func ExecControlArgv(identity ExecIdentity, action string, grace time.Duration) []string { + return []string{coordtransport.CLIPath, devexec.Command, "control", identity.CreationKey, identity.ExecID, identity.ClaimToken, action, strconv.FormatInt(grace.Milliseconds(), 10)} +} + +// ExecControlState reads the staged helper's reply to a control command that +// exited with code and printed stdout and stderr. +func ExecControlState(identity ExecIdentity, action string, code int, stdout, stderr string) (devexec.State, error) { + var state devexec.State + decodeErr := json.Unmarshal([]byte(stdout), &state) + if decodeErr == nil && state.ExecID == identity.ExecID && state.CreationKey == identity.CreationKey && state.ClaimToken == identity.ClaimToken { + if state.Error != "" { + if state.Exited && state.ExitCode != nil && (*state.ExitCode == 126 || *state.ExitCode == 127) { + if action == "start" { + return state, &ExecExitError{Code: *state.ExitCode} + } + return state, nil + } + return state, errors.New(state.Error) + } + if code == 0 { + return state, nil + } + } + failed := fmt.Errorf("%w: helper %s failed (status %d): %s", ErrExecUnavailable, action, code, strings.TrimSpace(stderr)) + if strings.Contains(stderr, devexec.ErrNoSupervisor.Error()) { + return state, fmt.Errorf("%w: %w", devexec.ErrNoSupervisor, failed) + } + return state, failed +} diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 911304faa..8c8d6c533 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -52,6 +52,28 @@ type InitPIDRuntime interface { InitPID(context.Context, ID) (int, error) } +// MemoryEventsRuntime optionally reports out-of-memory kills inside a +// container whose own kernel does the killing, where its host cgroup cannot +// see them. ok is false for a container its host cgroup accounts for. +type MemoryEventsRuntime interface { + MemoryEvents(context.Context, ID) (events MemoryEvents, ok bool, err error) +} + +// MemoryEvents counts kills since the container started; LimitBytes is 0 +// when unknown. +type MemoryEvents struct { + OOMKills uint64 + LimitBytes uint64 +} + +// SaveRuntime optionally writes a running container's memory and device +// state to disk and ends it, then later restores it with every process +// where it was. A container that was saved and not restored is stopped. +type SaveRuntime interface { + Save(context.Context, ID) error + Restore(context.Context, ID) error +} + // Mount is one additional mount into a run container. Host paths are // validated against Aether-owned roots by the mount validator (see // mounts.go) before a spec reaches any runtime; the runtime itself only diff --git a/internal/runtime/session.go b/internal/runtime/session.go new file mode 100644 index 000000000..8f60140be --- /dev/null +++ b/internal/runtime/session.go @@ -0,0 +1,149 @@ +package runtime + +import ( + "encoding/json" + "maps" + "slices" + "strconv" + "strings" +) + +// ExecOutputLimit covers stdout and stderr together. +const ExecOutputLimit = 1 << 20 + +// setupOutputLimit is how much of a setup script's output is kept, from the +// end, where the failure is. +const setupOutputLimit = 16 << 10 + +// RootPath is sudo's secure_path. A root exec never searches the +// container's PATH, which puts the agent-writable ~/.local/bin first. +const RootPath = "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +// UserScript gives uid $1 a passwd entry whose home is $2: git refuses to +// commit and ssh and ssh-keygen refuse to run for a uid with none, and ssh +// reads ~/.ssh from the passwd home, never $HOME. With $3 set it also lets +// the uid sudo without a password; the sudoers file is checked before it +// goes live, since sudo refuses to run at all with a broken one. +// +// The home at /root is the member's, shared by every container they run, +// and systemd's postinst runs systemd-tmpfiles, whose provision.conf hands +// /root to root: installing any package that pulls in systemd would take +// the home from all of them. The /etc entry masks it. +const UserScript = `uid=${1%%:*} gid=${1#*:} +if grep -q "^[^:]*:[^:]*:$uid:" /etc/passwd; then + awk -F: -v OFS=: -v u="$uid" -v h="$2" '$3 == u { $6 = h } { print }' /etc/passwd > /etc/passwd.aether && cat /etc/passwd.aether > /etc/passwd && rm -f /etc/passwd.aether || exit 1 +else + echo "aether:x:$uid:$gid::$2:/bin/sh" >> /etc/passwd || exit 1 +fi +grep -q "^[^:]*:[^:]*:$gid:" /etc/group || echo "aether:x:$gid:" >> /etc/group || exit 1 +mkdir -p /etc/tmpfiles.d && ln -sfn /dev/null /etc/tmpfiles.d/provision.conf || exit 1 +[ -n "$3" ] || exit 0 +mkdir -p /etc/sudoers.d +printf '#%s ALL=(ALL:ALL) NOPASSWD: ALL\n' "$uid" > /etc/sudoers.d/.aether.new +chmod 0440 /etc/sudoers.d/.aether.new +if command -v visudo >/dev/null 2>&1 && ! visudo -cqf /etc/sudoers.d/.aether.new; then rm -f /etc/sudoers.d/.aether.new; exit 1; fi +mv -f /etc/sudoers.d/.aether.new /etc/sudoers.d/aether` + +func Unprivileged(user string) bool { + uid, _, _ := strings.Cut(user, ":") + return uid != "" && uid != "0" +} + +func EnvList(env map[string]string) []string { + if len(env) == 0 { + return nil + } + out := make([]string, 0, len(env)) + for _, k := range slices.Sorted(maps.Keys(env)) { + out = append(out, k+"="+env[k]) + } + return out +} + +func EnvValue(env []string, name string) string { + for _, pair := range env { + if key, value, _ := strings.Cut(pair, "="); key == name { + return value + } + } + return "" +} + +func TrustCheckouts(env map[string]string, checkout string, extras []Mount) map[string]string { + env = trustCheckout(env, checkout) + for _, m := range extras { + env = trustCheckout(env, m.ContainerPath) + } + return env +} + +// Execs inherit the container env, so this also covers every git the server +// runs inside the container. +func trustCheckout(env map[string]string, checkout string) map[string]string { + if checkout == "" { + return env + } + n := 0 + if count, ok := env["GIT_CONFIG_COUNT"]; ok { + parsed, err := strconv.Atoi(count) + if err != nil || parsed < 0 { + return env + } + n = parsed + } + out := make(map[string]string, len(env)+3) + maps.Copy(out, env) + out["GIT_CONFIG_KEY_"+strconv.Itoa(n)] = "safe.directory" + out["GIT_CONFIG_VALUE_"+strconv.Itoa(n)] = checkout + out["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1) + return out +} + +func secretEnvValues(label string, env []string) []string { + var names []string + if json.Unmarshal([]byte(label), &names) != nil { + return nil + } + return SecretValues(names, env) +} + +func SecretValues(names, env []string) []string { + var values []string + for _, pair := range env { + name, value, _ := strings.Cut(pair, "=") + if value != "" && slices.Contains(names, name) { + values = append(values, value) + } + } + return values +} + +const secretMask = "***" + +// SetupOutput masks exact secret values, longest first so a secret holding +// another is masked whole, then keeps the last setupOutputLimit bytes. cut +// says output already lost its beginning, which may have split a secret: the +// span that could hold its remainder is dropped. That span is measured after +// masking, where a shorter secret inside the remainder can have grown it to +// len(secretMask) times its length. +func SetupOutput(output string, cut bool, secrets []string) string { + if len(secrets) > 0 { + slices.SortFunc(secrets, func(a, b string) int { return len(b) - len(a) }) + pairs := make([]string, 0, 2*len(secrets)) + for _, secret := range secrets { + pairs = append(pairs, secret, secretMask) + } + output = strings.NewReplacer(pairs...).Replace(output) + if cut { + output = output[min(len(secretMask)*len(secrets[0]), len(output)):] + } + } + if cut || len(output) > setupOutputLimit { + output = output[max(0, len(output)-setupOutputLimit):] + if i := strings.IndexByte(output, '\n'); i >= 0 { + output = output[i+1:] + } + output = "[earlier output omitted]\n" + output + } + return strings.ToValidUTF8(output, "�") +} diff --git a/internal/runtime/vm/chapi.go b/internal/runtime/vm/chapi.go new file mode 100644 index 000000000..cbfa609fe --- /dev/null +++ b/internal/runtime/vm/chapi.go @@ -0,0 +1,98 @@ +package vm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" +) + +type chAPI struct { + client *http.Client +} + +func newCHAPI(socket string) chAPI { + return chAPI{client: &http.Client{Transport: &http.Transport{ + DisableKeepAlives: true, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socket) + }, + }}} +} + +// state is vm.info's state: Created, Running, Paused or Shutdown. +func (c chAPI) state(ctx context.Context) (string, error) { + var info struct { + State string `json:"state"` + } + if err := c.call(ctx, http.MethodGet, "vm.info", nil, &info); err != nil { + return "", err + } + return info.State, nil +} + +func (c chAPI) pause(ctx context.Context) error { + return c.call(ctx, http.MethodPut, "vm.pause", nil, nil) +} + +func (c chAPI) resume(ctx context.Context) error { + return c.call(ctx, http.MethodPut, "vm.resume", nil, nil) +} + +func (c chAPI) snapshot(ctx context.Context, dir string) error { + return c.call(ctx, http.MethodPut, "vm.snapshot", map[string]string{"destination_url": "file://" + dir}, nil) +} + +// shutdownVMM ends the monitor process; the guest gets no warning. +func (c chAPI) shutdownVMM(ctx context.Context) error { + return c.call(ctx, http.MethodPut, "vmm.shutdown", nil, nil) +} + +// counters maps each device ID to its counters, such as a disk's read_bytes +// or a net device's rx_bytes. +func (c chAPI) counters(ctx context.Context) (map[string]map[string]uint64, error) { + var out map[string]map[string]uint64 + return out, c.call(ctx, http.MethodGet, "vm.counters", nil, &out) +} + +func (c chAPI) call(ctx context.Context, method, endpoint string, body, out any) error { + var payload io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("cloud-hypervisor %s: %w", endpoint, err) + } + payload = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, "http://localhost/api/v1/"+endpoint, payload) + if err != nil { + return fmt.Errorf("cloud-hypervisor %s: %w", endpoint, err) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("cloud-hypervisor %s: %w", endpoint, err) + } + defer func() { _ = resp.Body.Close() }() + data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return fmt.Errorf("cloud-hypervisor %s: %w", endpoint, err) + } + if resp.StatusCode/100 != 2 { + return fmt.Errorf("cloud-hypervisor %s: %s: %s", endpoint, resp.Status, strings.TrimSpace(string(data))) + } + if out == nil || len(data) == 0 { + return nil + } + if err := json.Unmarshal(data, out); err != nil { + return fmt.Errorf("cloud-hypervisor %s: decode reply: %w", endpoint, err) + } + return nil +} diff --git a/internal/runtime/vm/engine.go b/internal/runtime/vm/engine.go new file mode 100644 index 000000000..e9e8c2679 --- /dev/null +++ b/internal/runtime/vm/engine.go @@ -0,0 +1,448 @@ +package vm + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + goruntime "runtime" + "strings" + "sync" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +type Images interface { + Kernel() (string, error) + HelperDisk(ctx context.Context, binary, path string) (vmimage.Disk, error) + BaseDisk(ctx context.Context, ref string) (vmimage.Base, error) + NewRootDisk(ctx context.Context, base vmimage.Base, path string) (vmimage.Disk, error) + NewDataDisk(ctx context.Context, path string, size int64) (vmimage.Disk, error) +} + +const ( + // dataDiskBytes is sparse: it costs only what the guest writes. + dataDiskBytes = 128 << 30 + minMemoryMiB = 256 + bootTimeout = 90 * time.Second + agentOOMScore = 500 + // cmdlineBase boots the read-only helper disk, whose /init is the helper. + // Page reporting order 0 lets freed guest page cache back to the host; + // at the default order it stalls with most of it still held. + cmdlineBase = "root=/dev/vda ro rootfstype=ext4 init=/init console=ttyS0 quiet page_reporting.page_reporting_order=0" +) + +type engine struct { + cfg Config + root string + units units + dial func(vsockPath string) (helper, error) + run func(ctx context.Context, name string, args ...string) error + + mu sync.Mutex + locks map[string]*sync.Mutex + helpers map[string]helper + booting map[string]bool + fresh map[string]bool + + netMu sync.Mutex + reserved map[int]bool + natReady bool +} + +func newEngine(cfg Config, u units, dial func(string) (helper, error)) *engine { + return &engine{ + cfg: cfg, root: filepath.Join(cfg.DataDir, "vm"), units: u, dial: dial, run: runCommand, + locks: map[string]*sync.Mutex{}, helpers: map[string]helper{}, booting: map[string]bool{}, + fresh: map[string]bool{}, reserved: map[int]bool{}, + } +} + +func runCommand(ctx context.Context, name string, args ...string) error { + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if err != nil { + return fmt.Errorf("vm runtime: %s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) + } + return nil +} + +func (e *engine) lock(key string) func() { + e.mu.Lock() + l := e.locks[key] + if l == nil { + l = &sync.Mutex{} + e.locks[key] = l + } + e.mu.Unlock() + l.Lock() + return l.Unlock +} + +// helper returns the key's client, dialing once per VM lifetime; the client +// redials by itself across a save and restore. +func (e *engine) helper(key string) (helper, error) { + e.mu.Lock() + defer e.mu.Unlock() + if h := e.helpers[key]; h != nil { + return h, nil + } + h, err := e.dial(e.path(key, vsockSocket)) + if err != nil { + return nil, fmt.Errorf("vm runtime: connect to the helper of VM %s: %w", key, err) + } + e.helpers[key] = h + return h, nil +} + +func (e *engine) forgetHelper(key string) { + e.mu.Lock() + h := e.helpers[key] + delete(e.helpers, key) + e.mu.Unlock() + if h != nil { + _ = h.Close() + } +} + +func (e *engine) setFlag(m map[string]bool, key string, on bool) { + e.mu.Lock() + defer e.mu.Unlock() + if on { + m[key] = true + } else { + delete(m, key) + } +} + +func (e *engine) flag(m map[string]bool, key string) bool { + e.mu.Lock() + defer e.mu.Unlock() + return m[key] +} + +func (e *engine) create(ctx context.Context, spec runtime.Spec) (_ runtime.ID, err error) { + if err = spec.Validate(); err != nil { + return "", err + } + key := spec.CreationKey + if err = checkKey(key); err != nil { + return "", err + } + if spec.Egress { + return "", errors.New("vm runtime: egress control (the Isolated tier) is not supported in session VMs until work item 5") + } + for _, m := range spec.Mounts { + if m.Overlay != "" { + return "", fmt.Errorf("vm runtime: the overlay mount at %s (the Isolated tier) is not supported in session VMs until work item 7", m.ContainerPath) + } + } + memMiB, err := memoryMiB(spec.MemoryLimitBytes) + if err != nil { + return "", err + } + unlock := e.lock(key) + defer unlock() + if _, err = os.Stat(e.dir(key)); err == nil { + return "", fmt.Errorf("vm runtime: a VM named %s already exists", key) + } + base, err := e.cfg.Images.BaseDisk(ctx, spec.Image) + if err != nil { + return "", fmt.Errorf("vm runtime: base disk for %s: %w", spec.Image, err) + } + index, err := e.allocateNet() + if err != nil { + return "", err + } + defer e.releaseNet(index) + if err = os.MkdirAll(e.dir(key), 0o700); err != nil { + return "", fmt.Errorf("vm runtime: %w", err) + } + defer func() { + if err != nil { + _ = os.RemoveAll(e.dir(key)) + } + }() + root, err := e.cfg.Images.NewRootDisk(ctx, base, e.path(key, rootDisk)) + if err != nil { + return "", fmt.Errorf("vm runtime: root disk: %w", err) + } + data, err := e.cfg.Images.NewDataDisk(ctx, e.path(key, dataDisk), dataDiskBytes) + if err != nil { + return "", fmt.Errorf("vm runtime: data disk: %w", err) + } + cpus, limit := vcpus(spec.CPULimit) + rec := &record{ + Key: key, Fingerprint: spec.Fingerprint, Image: spec.Image, Base: base, Root: root, Data: data, + User: spec.User, + Env: overrideEnv(base.Config.Env, runtime.EnvList(runtime.TrustCheckouts(spec.Env, spec.WorktreeMountPath, spec.ExtraWorktrees))), + WorkingDir: cmp.Or(spec.WorkingDir, base.Config.WorkingDir, "/"), Command: spec.Command, TTY: spec.TTY, + SetupScript: spec.SetupScript, SecretEnv: spec.SecretEnv, NestedDocker: spec.NestedDocker, Sudo: spec.Sudo, + CPUs: cpus, CPULimit: limit, MemoryMiB: memMiB, PidsLimit: spec.PidsLimit, Mounts: specMounts(spec), Net: index, + } + if err = e.store(rec); err != nil { + return "", err + } + return runtime.ID(idPrefix + key), nil +} + +// vcpus rounds a fractional CPU limit up to whole vCPUs and keeps the limit +// itself as the unit's CPU quota; no limit means every host CPU. +func vcpus(limit float64) (int, float64) { + host := goruntime.NumCPU() + if limit <= 0 { + return host, 0 + } + return min(host, max(1, int(math.Ceil(limit)))), limit +} + +func memoryMiB(limit int64) (int64, error) { + if limit <= 0 { + total, err := hostMemoryBytes() + if err != nil { + return 0, err + } + return total >> 20, nil + } + mib := (limit + (1<<20 - 1)) >> 20 + if mib < minMemoryMiB { + return 0, fmt.Errorf("vm runtime: a memory limit of %d bytes is below the %d MiB a session VM needs to boot", limit, minMemoryMiB) + } + return mib, nil +} + +func hostMemoryBytes() (int64, error) { + var kb int64 + data, err := os.ReadFile("/proc/meminfo") + if err != nil { + return 0, fmt.Errorf("vm runtime: host memory: %w", err) + } + for line := range strings.SplitSeq(string(data), "\n") { + if rest, ok := strings.CutPrefix(line, "MemTotal:"); ok { + if _, err := fmt.Sscanf(strings.TrimSpace(rest), "%d kB", &kb); err != nil { + return 0, fmt.Errorf("vm runtime: parse MemTotal: %w", err) + } + return kb << 10, nil + } + } + return 0, errors.New("vm runtime: /proc/meminfo has no MemTotal") +} + +// start cold-starts a stopped VM; on a running one it only makes sure Main +// was started, as a redundant docker start does. +func (e *engine) start(ctx context.Context, key string) (err error) { + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return err + } + st, err := e.units.State(ctx, unitName(key)) + if err != nil { + return err + } + if st.Active { + return e.boot(ctx, rec) + } + if rec.Saved { + if err = os.RemoveAll(e.path(key, snapshotDir)); err != nil { + return fmt.Errorf("vm runtime: discard saved state: %w", err) + } + rec.Saved = false + } + if rec.Booted { + if rec.Root, err = e.cfg.Images.NewRootDisk(ctx, rec.Base, e.path(key, rootDisk)); err != nil { + return fmt.Errorf("vm runtime: root disk: %w", err) + } + } + l, err := e.bootLaunch(ctx, rec) + if err != nil { + return err + } + if err = e.ensureNAT(ctx); err != nil { + return err + } + rec.Booted, rec.Exited, rec.ExitCode = true, false, 0 + if err = e.store(rec); err != nil { + return err + } + if err = e.launch(ctx, rec, l); err != nil { + return err + } + defer func() { + if err != nil { + e.kill(context.WithoutCancel(ctx), key) + } + }() + if err = e.boot(ctx, rec); err != nil { + return err + } + e.setFlag(e.fresh, key, true) + return nil +} + +func (e *engine) bootLaunch(ctx context.Context, rec *record) (launch, error) { + p, err := planMounts(rec.Mounts, rec.NestedDocker, e.path(rec.Key, vsockSocket)) + if err != nil { + return launch{}, err + } + if err = linkSockets(e.dir(rec.Key), p.links); err != nil { + return launch{}, err + } + kernel, err := e.cfg.Images.Kernel() + if err != nil { + return launch{}, fmt.Errorf("vm runtime: guest kernel: %w", err) + } + helperDisk, err := e.cfg.Images.HelperDisk(ctx, e.cfg.Helper, e.path(rec.Key, helperImage)) + if err != nil { + return launch{}, fmt.Errorf("vm runtime: helper disk: %w", err) + } + cmdline, err := guest.Config{ + Hostname: rec.Key, Network: guestNetwork(rec.Net, hostDNS()), Mounts: p.mounts, Sockets: p.sockets, + }.KernelArgs(cmdlineBase) + if err != nil { + return launch{}, fmt.Errorf("vm runtime: kernel command line: %w", err) + } + args := []string{ + "--api-socket", "path=" + e.path(rec.Key, apiSocket), + "--vsock", "cid=3,socket=" + e.path(rec.Key, vsockSocket), + "--serial", "file=" + e.path(rec.Key, consoleLog), "--console", "off", + "--kernel", kernel, "--cmdline", cmdline, + "--disk", diskArg(helperDisk, "helper") + ",readonly=on", diskArg(rec.Root, "root"), diskArg(rec.Data, "data"), + "--cpus", fmt.Sprintf("boot=%d", rec.CPUs), + "--memory", fmt.Sprintf("size=%dM,shared=on", rec.MemoryMiB), + "--balloon", "size=0,free_page_reporting=on", + "--net", "id=net0," + chNetArg(rec.Net), + } + l := launch{CloudHypervisor: e.cfg.CloudHypervisor, Args: args, Shares: p.shares} + if len(p.shares) > 0 { + l.Args = append(l.Args, "--fs", "tag="+guest.ShareTag+",socket="+e.path(rec.Key, fsSocket)) + l.Virtiofsd = e.cfg.Virtiofsd + } + return l, nil +} + +func diskArg(d vmimage.Disk, id string) string { + arg := fmt.Sprintf("path=%s,image_type=%s,id=%s,serial=%s", d.Path, d.Format, id, id) + if d.Format == vmimage.FormatQcow2 { + arg += ",backing_files=on" + } + return arg +} + +func (e *engine) launch(ctx context.Context, rec *record, l launch) error { + data, err := json.MarshalIndent(l, "", " ") + if err != nil { + return fmt.Errorf("vm runtime: encode launch: %w", err) + } + if err := writeFileAtomic(e.path(rec.Key, launchFile), data); err != nil { + return err + } + logPath := e.path(rec.Key, monitorLog) + properties := []string{ + "Type=exec", "PrivateMounts=yes", "KillMode=control-group", "TimeoutStopSec=15", + fmt.Sprintf("OOMScoreAdjust=%d", agentOOMScore), + "StandardOutput=append:" + logPath, "StandardError=append:" + logPath, + } + if rec.CPULimit > 0 { + properties = append(properties, fmt.Sprintf("CPUQuota=%d%%", int(math.Ceil(rec.CPULimit*100)))) + } + argv := append(append([]string{}, e.cfg.Monitor...), e.dir(rec.Key)) + if err := e.units.Start(ctx, unitName(rec.Key), argv, properties); err != nil { + return err + } + return nil +} + +func (e *engine) boot(ctx context.Context, rec *record) error { + h, err := e.helper(rec.Key) + if err != nil { + return err + } + if err = e.waitHelper(ctx, rec.Key, h); err != nil { + return err + } + if runtime.Unprivileged(rec.User) { + allow := "" + if rec.Sudo { + allow = "sudo" + } + home := cmp.Or(runtime.EnvValue(rec.Env, "HOME"), "/") + code, out, prepErr := e.runAsRoot(ctx, h, []string{"/bin/sh", "-c", runtime.UserScript, "aether-user", rec.User, home, allow}) + if prepErr != nil { + return fmt.Errorf("vm runtime: prepare user %s: %w", rec.User, prepErr) + } + if code != 0 { + return fmt.Errorf("vm runtime: prepare user %s exited %d: %s", rec.User, code, strings.TrimSpace(out)) + } + } + var tty *guest.Size + if rec.TTY { + tty = &guest.Size{Cols: 80, Rows: 24} + } + e.setFlag(e.booting, rec.Key, true) + err = h.Boot(ctx, guest.BootSpec{ + Main: guest.ProcessSpec{ID: guest.MainID, Argv: rec.Command, Env: rec.Env, User: rec.User, Dir: rec.WorkingDir, TTY: tty}, + SetupScript: rec.SetupScript, NestedDocker: rec.NestedDocker, PidsLimit: rec.PidsLimit, + }) + e.setFlag(e.booting, rec.Key, false) + var setup *guest.SetupError + if errors.As(err, &setup) { + return &runtime.SetupError{ExitCode: setup.ExitCode, Output: runtime.SetupOutput(setup.Output, setup.Cut, runtime.SecretValues(rec.SecretEnv, rec.Env))} + } + if err != nil { + return fmt.Errorf("vm runtime: boot VM %s: %w", rec.Key, err) + } + return nil +} + +func (e *engine) waitHelper(ctx context.Context, key string, h helper) error { + ctx, cancel := context.WithTimeout(ctx, bootTimeout) + defer cancel() + answered := make(chan error, 1) + var info guest.Info + go func() { + var err error + info, err = h.Ping(ctx) + answered <- err + }() + tick := time.NewTicker(250 * time.Millisecond) + defer tick.Stop() + for { + select { + case err := <-answered: + if err != nil { + return fmt.Errorf("vm runtime: helper of VM %s did not answer: %w%s", key, err, e.logTail(key)) + } + if info.BootError != "" { + return fmt.Errorf("vm runtime: VM %s failed to boot: %s", key, info.BootError) + } + return nil + case <-tick.C: + if st, err := e.units.State(ctx, unitName(key)); err == nil && !st.Active { + cancel() + return fmt.Errorf("vm runtime: VM %s stopped while booting%s", key, e.logTail(key)) + } + } + } +} + +func (e *engine) logTail(key string) string { + var out strings.Builder + for _, name := range []string{monitorLog, consoleLog} { + data, err := os.ReadFile(e.path(key, name)) + if err != nil || len(data) == 0 { + continue + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + fmt.Fprintf(&out, "\n%s:\n%s", name, strings.Join(lines[max(0, len(lines)-15):], "\n")) + } + return out.String() +} diff --git a/internal/runtime/vm/engine_test.go b/internal/runtime/vm/engine_test.go new file mode 100644 index 000000000..6e1d2f890 --- /dev/null +++ b/internal/runtime/vm/engine_test.go @@ -0,0 +1,438 @@ +package vm + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "testing" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +func (h *harness) createStarted(key string, spec runtime.Spec) runtime.ID { + h.t.Helper() + id, err := h.engine.create(h.t.Context(), spec) + if err != nil { + h.t.Fatalf("create: %v", err) + } + if err := h.engine.start(h.t.Context(), key); err != nil { + h.t.Fatalf("start: %v", err) + } + return id +} + +func TestCreateRecordsWhatAContainerWouldReport(t *testing.T) { + h := newHarness(t) + spec := sessionSpec("run-a") + spec.WorktreeHostPath, spec.WorktreeMountPath = t.TempDir(), "/workspace" + spec.User = "1000:1000" + id, err := h.engine.create(t.Context(), spec) + if err != nil || id != "vm-run-a" { + t.Fatalf("create = %q, %v", id, err) + } + info, err := h.engine.inspect(t.Context(), "run-a") + if err != nil { + t.Fatal(err) + } + if info.Image != "aether/standard:1" || info.User != "1000:1000" || info.Fingerprint != "fp-1" || info.Exited { + t.Fatalf("inspect = %+v", info) + } + for _, want := range []string{"HOME=/root", "PATH=/usr/bin:/bin", "GIT_CONFIG_KEY_0=safe.directory", "GIT_CONFIG_VALUE_0=/workspace"} { + if !slices.Contains(info.Env, want) { + t.Errorf("env %v lacks %s", info.Env, want) + } + } + if !slices.Equal(info.MountTargets, []string{"/workspace"}) { + t.Errorf("mount targets = %v", info.MountTargets) + } + rec, _ := h.engine.load("run-a") + if rec.CPUs != 2 || rec.CPULimit != 1.5 || rec.MemoryMiB != 2048 || rec.WorkingDir != "/srv" { + t.Errorf("record = %+v", rec) + } + if _, err := h.engine.create(t.Context(), spec); err == nil || !strings.Contains(err.Error(), "already exists") { + t.Errorf("second create = %v", err) + } + if found, err := h.engine.find("run-a"); err != nil || found != id { + t.Errorf("find = %q, %v", found, err) + } + if _, err := h.engine.find("run-b"); !errors.Is(err, runtime.ErrNotFound) { + t.Errorf("find missing = %v", err) + } +} + +func TestCreateRefusesWhatLaterItemsAdd(t *testing.T) { + h := newHarness(t) + egress := sessionSpec("run-e") + egress.Egress = true + if _, err := h.engine.create(t.Context(), egress); err == nil || !strings.Contains(err.Error(), "work item 5") { + t.Errorf("egress create = %v", err) + } + overlay := sessionSpec("run-o") + overlay.Mounts = []runtime.Mount{{HostPath: "/h", ContainerPath: "/root", Overlay: "/scratch"}} + if _, err := h.engine.create(t.Context(), overlay); err == nil || !strings.Contains(err.Error(), "work item 7") { + t.Errorf("overlay create = %v", err) + } + missing := sessionSpec("run-m") + missing.Image = "missing:1" + if _, err := h.engine.create(t.Context(), missing); err == nil || !strings.Contains(err.Error(), "manifest unknown") { + t.Errorf("missing image create = %v", err) + } + if _, err := os.Stat(h.engine.dir("run-m")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("failed create left %s: %v", h.engine.dir("run-m"), err) + } +} + +func TestStartBootsTheVMInItsUnit(t *testing.T) { + h := newHarness(t) + home := t.TempDir() + spec := sessionSpec("run-a") + spec.User, spec.Sudo, spec.NestedDocker = "1000:1000", true, true + spec.SetupScript = "make deps" + spec.Mounts = []runtime.Mount{{HostPath: home, ContainerPath: "/root"}} + h.createStarted("run-a", spec) + + call := h.units.last() + if call.name != "aether-vm-run-a.service" || call.argv[1] != MonitorCommand || call.argv[2] != h.engine.dir("run-a") { + t.Fatalf("unit start = %+v", call) + } + for _, want := range []string{"PrivateMounts=yes", "CPUQuota=150%", "OOMScoreAdjust=500"} { + if !slices.Contains(call.properties, want) { + t.Errorf("properties %v lack %s", call.properties, want) + } + } + args := strings.Join(call.launch.Args, " ") + for _, want := range []string{ + "--cpus boot=2", "--memory size=2048M,shared=on", "--balloon size=0,free_page_reporting=on", + "--kernel /k/vmlinux", "page_reporting.page_reporting_order=0", "--fs tag=aether,socket=", + "helper.img,image_type=raw,id=helper,serial=helper,readonly=on", "root.img,image_type=qcow2,id=root,serial=root,backing_files=on", + "--net id=net0,tap=aethervm0,ip=10.200.0.1,mask=255.255.255.252", + } { + if !strings.Contains(args, want) { + t.Errorf("launch args lack %q:\n%s", want, args) + } + } + if len(call.launch.Shares) != 1 || call.launch.Shares[0].Source != home || call.launch.Virtiofsd != "virtiofsd" { + t.Errorf("shares = %+v", call.launch) + } + g := h.guest("run-a") + if len(g.boots) != 1 { + t.Fatalf("boots = %d", len(g.boots)) + } + boot := g.boots[0] + if boot.SetupScript != "make deps" || !boot.NestedDocker || boot.PidsLimit != 4096 || boot.Main.ID != guest.MainID || + boot.Main.User != "1000:1000" || boot.Main.Dir != "/srv" || boot.Main.TTY == nil || !slices.Equal(boot.Main.Argv, spec.Command) { + t.Errorf("boot = %+v", boot) + } + var prepared bool + for _, p := range g.spawned() { + if p.User == "0:0" && slices.Contains(p.Argv, runtime.UserScript) { + prepared = slices.Equal(p.Argv[len(p.Argv)-3:], []string{"1000:1000", "/root", "sudo"}) + } + } + if !prepared { + t.Errorf("no root user prep among %+v", g.spawned()) + } + if !h.engine.flag(h.engine.fresh, "run-a") || h.engine.settingUp("run-a") { + t.Errorf("fresh/settingUp after start") + } + if err := h.engine.start(t.Context(), "run-a"); err != nil || len(h.units.started) != 1 || len(g.boots) != 2 { + t.Errorf("redundant start: %v, %d unit starts, %d boots", err, len(h.units.started), len(g.boots)) + } +} + +func TestStartReportsSetupFailureMaskedAndKillsTheVM(t *testing.T) { + h := newHarness(t) + spec := sessionSpec("run-a") + spec.SecretEnv = []string{"TOKEN"} + if _, err := h.engine.create(t.Context(), spec); err != nil { + t.Fatal(err) + } + h.guest("run-a").bootErr = &guest.SetupError{ExitCode: 3, Output: "using s3cret\nfailed\n"} + err := h.engine.start(t.Context(), "run-a") + var setup *runtime.SetupError + if !errors.As(err, &setup) || setup.ExitCode != 3 || strings.Contains(setup.Output, "s3cret") || !strings.Contains(setup.Output, "using ***") { + t.Fatalf("start = %v", err) + } + if h.units.vmm("run-a") != nil { + t.Errorf("a failed start left the VM running") + } +} + +func TestStartFailsWithTheLogWhenTheVMDiesWhileBooting(t *testing.T) { + h := newHarness(t) + if _, err := h.engine.create(t.Context(), sessionSpec("run-a")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(h.engine.path("run-a", monitorLog), []byte("cloud-hypervisor: Error booting VM: KvmNotAvailable\n"), 0o600); err != nil { + t.Fatal(err) + } + h.units.failNext = true + h.guest("run-a").pingErr = errors.New("connection refused") + err := h.engine.start(t.Context(), "run-a") + if err == nil || !strings.Contains(err.Error(), "stopped while booting") || !strings.Contains(err.Error(), "KvmNotAvailable") { + t.Fatalf("start = %v", err) + } +} + +func TestStopEndsMainRecordsItsExitAndPowersOff(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + if err := h.engine.pause(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + vmm := h.units.vmm("run-a") + if err := h.engine.stop(t.Context(), "run-a", time.Second); err != nil { + t.Fatal(err) + } + if !slices.Contains(vmm.called(), "vm.resume") { + t.Errorf("stop did not resume the paused VM first: %v", vmm.called()) + } + if h.units.vmm("run-a") != nil { + t.Errorf("VM still running after stop") + } + info, err := h.engine.inspect(t.Context(), "run-a") + if err != nil || !info.Exited || info.ExitCode != 128+int(syscall.SIGTERM) { + t.Fatalf("inspect = %+v, %v", info, err) + } + if st, err := h.engine.wait(t.Context(), "run-a"); err != nil || st.Code != info.ExitCode { + t.Errorf("wait = %+v, %v", st, err) + } + if err := h.engine.stop(t.Context(), "run-a", 0); err != nil { + t.Errorf("stop of a stopped VM = %v", err) + } + if err := h.engine.start(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if h.images.roots() != 2 { + t.Errorf("cold start used %d root disks, want a fresh one", h.images.roots()) + } + if info, _ := h.engine.inspect(t.Context(), "run-a"); info.Exited { + t.Errorf("restarted VM still reports exited") + } +} + +func TestStopKillsMainThatIgnoresTerm(t *testing.T) { + h := newHarness(t) + h.guest("run-a").ignoreTE = true + h.createStarted("run-a", sessionSpec("run-a")) + if err := h.engine.stop(t.Context(), "run-a", 50*time.Millisecond); err != nil { + t.Fatal(err) + } + if info, _ := h.engine.inspect(t.Context(), "run-a"); info.ExitCode != 128+int(syscall.SIGKILL) { + t.Errorf("exit code = %d, want SIGKILL's", info.ExitCode) + } +} + +func TestWaitReportsMainsExitAndALostVM(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + main := h.guest("run-a").procs[guest.MainID] + go func() { + time.Sleep(50 * time.Millisecond) + main.exit(7) + }() + st, err := h.engine.wait(t.Context(), "run-a") + if err != nil || st.Code != 7 { + t.Fatalf("wait = %+v, %v", st, err) + } + if h.units.vmm("run-a") != nil { + t.Errorf("VM still running after Main exited") + } + + h.createStarted("run-b", sessionSpec("run-b")) + _ = h.units.Stop(t.Context(), unitName("run-b")) + if st, err := h.engine.wait(t.Context(), "run-b"); err != nil || st.Code != lostExitCode { + t.Fatalf("wait on a lost VM = %+v, %v", st, err) + } +} + +func TestAnotherEngineFindsAndDrivesARunningVM(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + after := h.newEngine(filepath.Dir(h.engine.root)) + id, err := after.find("run-a") + if err != nil || id != "vm-run-a" { + t.Fatalf("find after restart = %q, %v", id, err) + } + info, err := after.inspect(t.Context(), "run-a") + if err != nil || info.Exited { + t.Fatalf("inspect after restart = %+v, %v", info, err) + } + if pid, pidErr := after.initPID(t.Context(), "run-a"); pidErr != nil || pid != 4242 { + t.Errorf("initPID = %d, %v", pid, pidErr) + } + if ip, ipErr := after.containerIP(t.Context(), "run-a"); ipErr != nil || ip != "10.200.0.2" { + t.Errorf("containerIP = %q, %v", ip, ipErr) + } + h.guest("run-a").output = map[string]string{"echo": "hi\n"} + code, out, _, err := after.exec(t.Context(), "run-a", []string{"echo", "hi"}, "") + if err != nil || code != 0 || out != "hi\n" { + t.Errorf("exec after restart = %d %q %v", code, out, err) + } + if err := after.stop(t.Context(), "run-a", time.Second); err != nil { + t.Fatal(err) + } + if err := after.destroy(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(after.dir("run-a")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("destroy left %s", after.dir("run-a")) + } + if err := after.destroy(t.Context(), "run-a"); err != nil { + t.Errorf("second destroy = %v", err) + } +} + +func TestExecRunsAsTheVMsUserWithItsEnvironment(t *testing.T) { + h := newHarness(t) + spec := sessionSpec("run-a") + spec.User = "1000:1000" + h.createStarted("run-a", spec) + g := h.guest("run-a") + g.output = map[string]string{"id": "uid=1000\n", "nope": "start-error:"} + if code, out, _, err := h.engine.exec(t.Context(), "run-a", []string{"id"}, "/workspace"); err != nil || code != 0 || out != "uid=1000\n" { + t.Fatalf("exec = %d %q %v", code, out, err) + } + var ran guest.ProcessSpec + for _, p := range g.spawned() { + if p.Argv[0] == "id" { + ran = p + } + } + if ran.User != "1000:1000" || ran.Dir != "/workspace" || !slices.Contains(ran.Env, "TOKEN=s3cret") || !slices.Contains(ran.Env, "PATH=/usr/bin:/bin") { + t.Errorf("exec spawned %+v", ran) + } + if code, _, _, err := h.engine.exec(t.Context(), "run-a", []string{"nope"}, ""); err != nil || code != 127 { + t.Errorf("exec of a missing command = %d, %v", code, err) + } + var exit *runtime.ExecExitError + if _, err := h.engine.execTTY(t.Context(), "run-a", []string{"nope"}, "", 80, 24); !errors.As(err, &exit) || exit.Code != 127 { + t.Errorf("execTTY of a missing command = %v", err) + } + g.output["big"] = strings.Repeat("x", runtime.ExecOutputLimit+1) + if _, _, _, err := h.engine.exec(t.Context(), "run-a", []string{"big"}, ""); err == nil || !strings.Contains(err.Error(), "byte limit") { + t.Errorf("oversized exec = %v", err) + } + if env := execEnv(&record{Env: []string{"PATH=/home/agent/.local/bin:/usr/bin"}}, "0:0", nil); !slices.Equal(env, []string{runtime.RootPath}) { + t.Errorf("root exec env = %v", env) + } +} + +func TestManagedExecGoesThroughDevExec(t *testing.T) { + h := newHarness(t) + spec := sessionSpec("run-a") + spec.User = "1000:1000" + id := h.createStarted("run-a", spec) + m, err := h.engine.startManagedExec(t.Context(), "run-a", runtime.ExecSpec{Argv: []string{"bash"}, User: "0", CreationKey: "terminal-1", Cols: 80, Rows: 24}, true) + if err != nil { + t.Fatal(err) + } + identity := m.Identity() + if identity.ContainerID != id || identity.CreationKey != "terminal-1" || identity.User != "0" { + t.Errorf("identity = %+v", identity) + } + supervisor := h.guest("run-a").procs[identity.ExecID] + if supervisor == nil || !slices.Equal(supervisor.spec.Argv[1:4], []string{"dev-exec", "run", "terminal-1"}) || + !slices.Contains(supervisor.spec.Env, "AETHER_DEVEXEC_CONTROL_GROUP=1000") { + t.Fatalf("supervisor = %+v", supervisor) + } + if st, stErr := m.Status(t.Context()); stErr != nil || !st.Running || !st.Attached { + t.Errorf("status = %+v, %v", st, stErr) + } + recovered, err := h.newEngine(filepath.Dir(h.engine.root)).recoverExec(t.Context(), "run-a", identity) + if err != nil || recovered.Attachment() != nil { + t.Fatalf("recover = %v, attachment %v", err, recovered) + } + st, err := recovered.Stop(t.Context(), time.Second) + if err != nil || st.Code != 143 { + t.Errorf("stop = %+v, %v", st, err) + } + if _, err := h.engine.recoverExec(t.Context(), "run-a", runtime.ExecIdentity{ExecID: "gone", CreationKey: "k", ClaimToken: "c"}); !errors.Is(err, runtime.ErrNotFound) { + t.Errorf("recover of an unknown exec = %v", err) + } +} + +func TestSaveRefusesSharesAndRestoresAVMWithout(t *testing.T) { + h := newHarness(t) + shared := sessionSpec("run-s") + shared.WorktreeHostPath, shared.WorktreeMountPath = t.TempDir(), "/workspace" + h.createStarted("run-s", shared) + if err := h.engine.save(t.Context(), "run-s"); err == nil || !strings.Contains(err.Error(), "/workspace") { + t.Fatalf("save with a share = %v", err) + } + if h.units.vmm("run-s") == nil { + t.Error("a refused save stopped the VM") + } + + h.createStarted("run-a", sessionSpec("run-a")) + vmm := h.units.vmm("run-a") + if err := h.engine.save(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if !slices.Equal(vmm.called(), []string{"vm.pause", "vm.snapshot", "vmm.shutdown"}) { + t.Errorf("save calls = %v", vmm.called()) + } + if _, err := os.Stat(h.engine.path("run-a", snapshotDir+"/state.json")); err != nil { + t.Errorf("no saved state: %v", err) + } + if info, _ := h.engine.inspect(t.Context(), "run-a"); info.Exited { + t.Errorf("a saved VM reports exited") + } + waited := make(chan struct{}) + go func() { + _, _ = h.engine.wait(context.WithoutCancel(t.Context()), "run-a") + close(waited) + }() + if err := h.engine.restore(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + restored := h.units.vmm("run-a") + if !restored.restored || !slices.Contains(restored.called(), "vm.resume") || h.guest("run-a").clock.IsZero() { + t.Errorf("restore: vmm %+v calls %v", restored, restored.called()) + } + if _, err := os.Stat(h.engine.path("run-a", snapshotDir)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("snapshot kept after restore: %v", err) + } + select { + case <-waited: + t.Fatal("wait returned across a save and restore") + case <-time.After(300 * time.Millisecond): + } + h.guest("run-a").procs[guest.MainID].exit(0) + <-waited + if err := h.engine.restore(t.Context(), "run-a"); err == nil || !strings.Contains(err.Error(), "no saved state") { + t.Errorf("restore without state = %v", err) + } +} + +func TestStatsAndMemoryEventsComeFromTheHelper(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + h.guest("run-a").stats = guest.Stats{CPUUsageNanos: 5, MemoryBytes: 300 << 20, MemoryLimitBytes: 2 << 30, Pids: 12, PidsLimit: 4096, OOMKills: 2} + stats, err := h.engine.stats(t.Context()) + if err != nil || len(stats) != 1 { + t.Fatalf("stats = %+v, %v", stats, err) + } + st := stats[0] + if st.ID != "vm-run-a" || st.CreationKey != "run-a" || st.CPULimitCores != 1.5 || st.MemoryBytes != 300<<20 || st.Pids != 12 || + st.NetRxBytes != 10 || st.NetTxBytes != 20 || st.DiskReadBytes != 4 || st.DiskWriteBytes != 6 { + t.Errorf("stat = %+v", st) + } + events, err := h.engine.memoryEvents(t.Context(), "run-a") + if err != nil || events.OOMKills != 2 || events.LimitBytes != 2<<30 { + t.Errorf("memory events = %+v, %v", events, err) + } + if err := h.engine.stop(t.Context(), "run-a", time.Second); err != nil { + t.Fatal(err) + } + if _, err := h.engine.memoryEvents(t.Context(), "run-a"); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("memory events of a stopped VM = %v, want not-exist", err) + } +} diff --git a/internal/runtime/vm/exec.go b/internal/runtime/vm/exec.go new file mode 100644 index 000000000..98bbe8650 --- /dev/null +++ b/internal/runtime/vm/exec.go @@ -0,0 +1,233 @@ +package vm + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "strconv" + "strings" + "sync" + "time" + + "github.com/3xDevOps/Aether/internal/coordtransport" + "github.com/3xDevOps/Aether/internal/devexec" + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +const stdinCloseTimeout = 5 * time.Second + +func processID(kind string) string { + var b [8]byte + _, _ = rand.Read(b[:]) + return kind + "-" + hex.EncodeToString(b[:]) +} + +// A root exec gets sudo's secure_path instead of the VM's PATH, as with +// Docker. +func execEnv(rec *record, user string, extra []string) []string { + if uid, _, _ := strings.Cut(user, ":"); uid == "0" || uid == "root" { + extra = append(append([]string{}, extra...), runtime.RootPath) + } + return overrideEnv(rec.Env, extra) +} + +func overrideEnv(base, extra []string) []string { + merged := make(map[string]string, len(base)+len(extra)) + for _, list := range [][]string{base, extra} { + for _, pair := range list { + if k, v, ok := strings.Cut(pair, "="); ok { + merged[k] = v + } + } + } + return runtime.EnvList(merged) +} + +func (e *engine) settingUp(key string) bool { return e.flag(e.booting, key) } + +// attach replays Main's output from its start only to the first attach after +// this process booted the VM; later ones, like docker attach, see output from +// now on. +func (e *engine) attach(ctx context.Context, key string) (runtime.Attachment, error) { + if _, err := e.requireRunning(ctx, key, "attach to"); err != nil { + return nil, err + } + h, err := e.helper(key) + if err != nil { + return nil, err + } + e.mu.Lock() + fromStart := e.fresh[key] + delete(e.fresh, key) + e.mu.Unlock() + return h.Process(guest.MainID).Attach(fromStart), nil +} + +func (e *engine) spawn(ctx context.Context, key string, spec guest.ProcessSpec) (process, runtime.Attachment, error) { + rec, err := e.requireRunning(ctx, key, "exec in") + if err != nil { + return nil, nil, err + } + h, err := e.helper(key) + if err != nil { + return nil, nil, err + } + if spec.User == "" { + spec.User = rec.User + } + spec.Env = execEnv(rec, spec.User, spec.Env) + if spec.Dir == "" { + spec.Dir = rec.WorkingDir + } + p, err := h.Spawn(ctx, spec) + if err != nil { + return nil, nil, err + } + return p, p.Attach(true), nil +} + +func (e *engine) exec(ctx context.Context, key string, argv []string, workDir string) (int, string, string, error) { + p, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: processID("exec"), Argv: argv, Dir: workDir, Lossless: true}) + var start *guest.StartError + if errors.As(err, &start) { + return start.Code, "", err.Error(), nil + } + if err != nil { + return 0, "", "", fmt.Errorf("vm runtime: exec: %w", err) + } + defer func() { _ = att.Close() }() + if err = p.CloseStdin(ctx); err != nil { + return 0, "", "", fmt.Errorf("vm runtime: exec input: %w", err) + } + stdout, stderr, err := readLimited(att) + if err != nil { + return 0, stdout, stderr, fmt.Errorf("vm runtime: exec output: %w", err) + } + code, err := p.Wait(ctx) + if err != nil { + return 0, stdout, stderr, fmt.Errorf("vm runtime: exec wait: %w", err) + } + return code, stdout, stderr, nil +} + +// readLimited reads both streams to their end and refuses output past +// runtime.ExecOutputLimit, which the two share. +func readLimited(att runtime.Attachment) (string, string, error) { + var mu sync.Mutex + total := 0 + var out, errOut bytes.Buffer + read := func(r io.Reader, into *bytes.Buffer) error { + buf := make([]byte, 32<<10) + for { + n, err := r.Read(buf) + mu.Lock() + total += n + over := total > runtime.ExecOutputLimit + if !over { + into.Write(buf[:n]) + } + mu.Unlock() + if over { + return fmt.Errorf("output passed the %d byte limit", runtime.ExecOutputLimit) + } + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + } + } + errc := make(chan error, 1) + go func() { errc <- read(att.Stderr(), &errOut) }() + outErr := read(att.Stdout(), &out) + if outErr != nil { + _ = att.Close() + } + errErr := <-errc + return out.String(), errOut.String(), errors.Join(outErr, errErr) +} + +func (e *engine) execTTY(ctx context.Context, key string, argv []string, workDir string, cols, rows uint) (runtime.Attachment, error) { + size := guest.Size{Cols: 80, Rows: 24} + if cols != 0 && rows != 0 { + size = guest.Size{Cols: uint16(min(cols, 65535)), Rows: uint16(min(rows, 65535))} + } + _, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: processID("tty"), Argv: argv, Dir: workDir, TTY: &size}) + var start *guest.StartError + if errors.As(err, &start) && (start.Code == 126 || start.Code == 127) { + return nil, &runtime.ExecExitError{Code: start.Code} + } + if err != nil { + return nil, fmt.Errorf("vm runtime: exec: %w", err) + } + return att, nil +} + +// execAttached keeps the staged helper's input open while it runs: closing +// it, when ctx ends or the output cannot be delivered, kills the command. +func (e *engine) execAttached(ctx context.Context, key string, spec runtime.AttachedExec) (int, error) { + argv := append([]string{coordtransport.CLIPath, devexec.Command, devexec.Attached, strconv.Itoa(len(spec.Stdin))}, spec.Argv...) + p, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: processID("attached"), Argv: argv, User: spec.User, Dir: spec.WorkingDir, Lossless: true}) + if err != nil { + return 0, fmt.Errorf("vm runtime: exec: %w", err) + } + defer func() { _ = att.Close() }() + closeStdin := sync.OnceFunc(func() { + closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stdinCloseTimeout) + defer cancel() + _ = p.CloseStdin(closeCtx) + }) + stop := context.AfterFunc(ctx, closeStdin) + defer stop() + if len(spec.Stdin) > 0 { + if _, err = att.Stdin().Write(spec.Stdin); err != nil { + closeStdin() + return 0, fmt.Errorf("vm runtime: exec input: %w", err) + } + } + copied := make(chan error, 1) + go func() { + _, copyErr := io.Copy(spec.Stderr, att.Stderr()) + copied <- copyErr + }() + _, outErr := io.Copy(spec.Stdout, att.Stdout()) + if outErr != nil { + closeStdin() + } + errErr := <-copied + if err = ctx.Err(); err != nil { + return 0, err + } + if err = errors.Join(outErr, errErr); err != nil { + return 0, fmt.Errorf("vm runtime: exec output: %w", err) + } + code, err := p.Wait(ctx) + if err != nil { + return 0, fmt.Errorf("vm runtime: exec wait: %w", err) + } + return code, nil +} + +func (e *engine) runAsRoot(ctx context.Context, h helper, argv []string) (int, string, error) { + p, err := h.Spawn(ctx, guest.ProcessSpec{ID: processID("prepare"), Argv: argv, User: "0:0", Env: []string{runtime.RootPath}, Dir: "/", Lossless: true}) + if err != nil { + return 0, "", err + } + att := p.Attach(true) + defer func() { _ = att.Close() }() + if err = p.CloseStdin(ctx); err != nil { + return 0, "", err + } + stdout, stderr, err := readLimited(att) + if err != nil { + return 0, "", err + } + code, err := p.Wait(ctx) + return code, stdout + stderr, err +} diff --git a/internal/runtime/vm/fakes_test.go b/internal/runtime/vm/fakes_test.go new file mode 100644 index 000000000..da40e6b20 --- /dev/null +++ b/internal/runtime/vm/fakes_test.go @@ -0,0 +1,489 @@ +package vm + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/3xDevOps/Aether/internal/devexec" + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +// harness wires an engine to fakes: systemd units that serve a fake Cloud +// Hypervisor API on the VM's real API socket, one fake helper per VM, and a +// fake image store. +type harness struct { + t *testing.T + engine *engine + units *fakeUnits + images *fakeImages + mu sync.Mutex + guests map[string]*fakeHelper +} + +func newHarness(t *testing.T) *harness { + t.Helper() + dir, err := os.MkdirTemp("", "vmt") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + h := &harness{t: t, images: &fakeImages{}, guests: map[string]*fakeHelper{}} + h.units = &fakeUnits{t: t, active: map[string]*fakeVMM{}} + h.engine = h.newEngine(dir) + t.Cleanup(h.units.stopAll) + return h +} + +// newEngine is a second engine process over the same data directory. +func (h *harness) newEngine(dir string) *engine { + cfg := Config{DataDir: dir, Images: h.images, Monitor: []string{"/srv/aether-server", MonitorCommand}, Helper: "/srv/aether-server", CloudHypervisor: "cloud-hypervisor", Virtiofsd: "virtiofsd"} + e := newEngine(cfg, h.units, func(vsock string) (helper, error) { + return h.guest(filepath.Base(filepath.Dir(vsock))), nil + }) + e.run = func(context.Context, string, ...string) error { return nil } + e.natReady = true + return e +} + +func (h *harness) guest(key string) *fakeHelper { + h.mu.Lock() + defer h.mu.Unlock() + g := h.guests[key] + if g == nil { + g = &fakeHelper{key: key, units: h.units, procs: map[string]*fakeProc{}} + h.guests[key] = g + } + return g +} + +func sessionSpec(key string) runtime.Spec { + return runtime.Spec{ + Image: "aether/standard:1", Command: []string{"/bin/sh", "-c", "agent"}, TTY: true, + Env: map[string]string{"AETHER_RUN_ID": key, "HOME": "/root", "TOKEN": "s3cret"}, CreationKey: key, + Fingerprint: "fp-1", CPULimit: 1.5, MemoryLimitBytes: 2 << 30, PidsLimit: 4096, + } +} + +type fakeImages struct { + mu sync.Mutex + rootDisks int +} + +func (f *fakeImages) Kernel() (string, error) { return "/k/vmlinux", nil } + +func (f *fakeImages) HelperDisk(_ context.Context, _, path string) (vmimage.Disk, error) { + return vmimage.Disk{Path: path, Format: "raw"}, os.WriteFile(path, nil, 0o600) +} + +func (f *fakeImages) BaseDisk(_ context.Context, ref string) (vmimage.Base, error) { + if ref == "missing:1" { + return vmimage.Base{}, errors.New("manifest unknown") + } + return vmimage.Base{ID: "sha256:b", Path: "/images/b.raw", Config: vmimage.ImageConfig{ + Env: []string{"PATH=/usr/bin:/bin", "HOME=/image-home"}, WorkingDir: "/srv", + }}, nil +} + +func (f *fakeImages) NewRootDisk(_ context.Context, _ vmimage.Base, path string) (vmimage.Disk, error) { + f.mu.Lock() + f.rootDisks++ + f.mu.Unlock() + return vmimage.Disk{Path: path, Format: "qcow2"}, os.WriteFile(path, nil, 0o600) +} + +func (f *fakeImages) NewDataDisk(_ context.Context, path string, _ int64) (vmimage.Disk, error) { + return vmimage.Disk{Path: path, Format: "raw"}, os.WriteFile(path, nil, 0o600) +} + +func (f *fakeImages) roots() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.rootDisks +} + +// fakeUnits starts a fake VMM per unit: it reads the launch the engine +// wrote and serves Cloud Hypervisor's API on the API socket it names. +type fakeUnits struct { + t *testing.T + mu sync.Mutex + active map[string]*fakeVMM + started []startCall + failNext bool +} + +type startCall struct { + name string + argv []string + properties []string + launch launch +} + +func (u *fakeUnits) Start(_ context.Context, name string, argv, properties []string) error { + data, err := os.ReadFile(filepath.Join(argv[len(argv)-1], launchFile)) + if err != nil { + return err + } + var l launch + if err = json.Unmarshal(data, &l); err != nil { + return err + } + u.mu.Lock() + defer u.mu.Unlock() + u.started = append(u.started, startCall{name, argv, properties, l}) + if u.failNext { + u.failNext = false + return nil + } + socket := "" + for i, arg := range l.Args { + if arg == "--api-socket" { + socket = strings.TrimPrefix(l.Args[i+1], "path=") + } + } + vmm, err := startVMM(socket, l.Args, func() { go func() { _ = u.Stop(context.Background(), name) }() }) + if err != nil { + return err + } + u.active[name] = vmm + return nil +} + +func (u *fakeUnits) Stop(_ context.Context, name string) error { + u.mu.Lock() + vmm := u.active[name] + delete(u.active, name) + u.mu.Unlock() + if vmm != nil { + vmm.close() + } + return nil +} + +func (u *fakeUnits) State(_ context.Context, name string) (unitState, error) { + u.mu.Lock() + defer u.mu.Unlock() + if u.active[name] != nil { + return unitState{Active: true, MainPID: 4242}, nil + } + return unitState{}, nil +} + +func (u *fakeUnits) stopAll() { + u.mu.Lock() + names := make([]string, 0, len(u.active)) + for name := range u.active { + names = append(names, name) + } + u.mu.Unlock() + for _, name := range names { + _ = u.Stop(context.Background(), name) + } +} + +func (u *fakeUnits) vmm(key string) *fakeVMM { + u.mu.Lock() + defer u.mu.Unlock() + return u.active[unitName(key)] +} + +func (u *fakeUnits) last() startCall { + u.mu.Lock() + defer u.mu.Unlock() + return u.started[len(u.started)-1] +} + +// fakeVMM is Cloud Hypervisor's REST API with the state transitions the +// engine relies on. +type fakeVMM struct { + mu sync.Mutex + state string + calls []string + restored bool + srv *http.Server +} + +func startVMM(socket string, args []string, exit func()) (*fakeVMM, error) { + _ = os.Remove(socket) + l, err := net.Listen("unix", socket) + if err != nil { + return nil, err + } + v := &fakeVMM{state: "Running"} + for _, arg := range args { + if arg == "--restore" { + v.state, v.restored = "Paused", true + } + } + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/", func(w http.ResponseWriter, r *http.Request) { + endpoint := strings.TrimPrefix(r.URL.Path, "/api/v1/") + body, _ := io.ReadAll(r.Body) + v.mu.Lock() + defer v.mu.Unlock() + v.calls = append(v.calls, endpoint) + switch endpoint { + case "vm.info": + _ = json.NewEncoder(w).Encode(map[string]string{"state": v.state}) + case "vm.pause": + v.state = "Paused" + case "vm.resume": + v.state = "Running" + case "vm.snapshot": + var req struct { + Destination string `json:"destination_url"` + } + _ = json.Unmarshal(body, &req) + dir := strings.TrimPrefix(req.Destination, "file://") + _ = os.WriteFile(filepath.Join(dir, "state.json"), []byte("{}"), 0o600) + case "vm.counters": + _ = json.NewEncoder(w).Encode(map[string]map[string]uint64{ + "net0": {"rx_bytes": 10, "tx_bytes": 20}, "root": {"read_bytes": 1, "write_bytes": 2}, "data": {"read_bytes": 3, "write_bytes": 4}, + }) + case "vmm.shutdown": + exit() + default: + http.Error(w, "unknown endpoint", http.StatusNotFound) + } + }) + v.srv = &http.Server{Handler: mux, ReadHeaderTimeout: time.Second} + go func() { _ = v.srv.Serve(l) }() + return v, nil +} + +func (v *fakeVMM) close() { _ = v.srv.Close() } + +func (v *fakeVMM) called() []string { + v.mu.Lock() + defer v.mu.Unlock() + return append([]string(nil), v.calls...) +} + +// fakeHelper is the in-VM helper. Main exits when it is signalled or told to. +type fakeHelper struct { + key string + units *fakeUnits + mu sync.Mutex + boots []guest.BootSpec + procs map[string]*fakeProc + // bootErr and output program the next Boot and Spawn. + bootErr error + pingErr error + output map[string]string + ignoreTE bool + clock time.Time + stats guest.Stats +} + +func (g *fakeHelper) Ping(ctx context.Context) (guest.Info, error) { + if g.pingErr != nil { + <-ctx.Done() + return guest.Info{}, g.pingErr + } + return guest.Info{BootID: "b1", Protocol: 1}, nil +} + +func (g *fakeHelper) Boot(_ context.Context, spec guest.BootSpec) error { + g.mu.Lock() + defer g.mu.Unlock() + g.boots = append(g.boots, spec) + if g.bootErr != nil { + return g.bootErr + } + if main := g.procs[guest.MainID]; main == nil || main.exited() { + g.procs[guest.MainID] = newFakeProc(spec.Main, "", 0, g.ignoreTE) + } + return nil +} + +func (g *fakeHelper) Spawn(_ context.Context, spec guest.ProcessSpec) (process, error) { + g.mu.Lock() + defer g.mu.Unlock() + if _, ok := g.procs[spec.ID]; ok { + return nil, errors.New("already exists") + } + out := g.output[spec.Argv[0]] + if len(spec.Argv) >= 8 && spec.Argv[1] == devexec.Command && spec.Argv[2] == "control" { + out = g.control(spec.Argv[3], spec.Argv[4], spec.Argv[5], spec.Argv[6]) + } + if strings.HasPrefix(out, "start-error:") { + return nil, &guest.StartError{Code: 127, Err: errors.New("exec: no such file")} + } + p := newFakeProc(spec, out, 0, false) + g.procs[spec.ID] = p + if !strings.Contains(strings.Join(spec.Argv, " "), "dev-exec run") { + p.exit(0) + } + return p, nil +} + +// control answers like dev-exec's control command; it holds g.mu. +func (g *fakeHelper) control(key, execID, claim, action string) string { + state := devexec.State{CreationKey: key, ExecID: execID, ClaimToken: claim} + supervisor := g.procs[execID] + if supervisor == nil { + state.Error = "no such execution" + } else { + if action == "stop" { + supervisor.exit(143) + } + if st, _ := supervisor.Status(context.Background()); st.Exited { + code := st.ExitCode + state.Exited, state.ExitCode = true, &code + } else { + state.Running = true + } + } + data, _ := json.Marshal(state) + return string(data) +} + +func (g *fakeHelper) Process(id string) process { + g.mu.Lock() + defer g.mu.Unlock() + if p := g.procs[id]; p != nil { + return p + } + return missingProc{} +} + +func (g *fakeHelper) spawned() []guest.ProcessSpec { + g.mu.Lock() + defer g.mu.Unlock() + var out []guest.ProcessSpec + for _, p := range g.procs { + out = append(out, p.spec) + } + return out +} + +func (g *fakeHelper) Stats(context.Context) (guest.Stats, error) { return g.stats, nil } + +func (g *fakeHelper) SyncClock(_ context.Context, t time.Time) error { + g.mu.Lock() + g.clock = t + g.mu.Unlock() + return nil +} + +// PowerOff ends the VM the way a guest poweroff ends Cloud Hypervisor. +func (g *fakeHelper) PowerOff(ctx context.Context) error { + return g.units.Stop(ctx, unitName(g.key)) +} + +func (g *fakeHelper) Close() error { return nil } + +type fakeProc struct { + spec guest.ProcessSpec + out string + ignoreTE bool + mu sync.Mutex + signals []syscall.Signal + done chan struct{} + code int + stdin bool +} + +func newFakeProc(spec guest.ProcessSpec, out string, code int, ignoreTerm bool) *fakeProc { + return &fakeProc{spec: spec, out: out, code: code, ignoreTE: ignoreTerm, done: make(chan struct{})} +} + +func (p *fakeProc) exited() bool { + select { + case <-p.done: + return true + default: + return false + } +} + +func (p *fakeProc) exit(code int) { + p.mu.Lock() + defer p.mu.Unlock() + select { + case <-p.done: + default: + p.code = code + close(p.done) + } +} + +func (p *fakeProc) Attach(bool) runtime.Attachment { + return &fakeAttachment{stdout: strings.NewReader(p.out), stderr: strings.NewReader("")} +} + +func (p *fakeProc) CloseStdin(context.Context) error { + p.mu.Lock() + p.stdin = true + p.mu.Unlock() + return nil +} + +func (p *fakeProc) Resize(context.Context, guest.Size) error { return nil } + +func (p *fakeProc) Signal(_ context.Context, sig syscall.Signal) error { + p.mu.Lock() + p.signals = append(p.signals, sig) + ignore := p.ignoreTE && sig == syscall.SIGTERM + p.mu.Unlock() + if !ignore { + p.exit(128 + int(sig)) + } + return nil +} + +func (p *fakeProc) Status(context.Context) (guest.State, error) { + select { + case <-p.done: + return guest.State{Exited: true, ExitCode: p.code}, nil + default: + return guest.State{}, nil + } +} + +func (p *fakeProc) Wait(ctx context.Context) (int, error) { + select { + case <-p.done: + return p.code, nil + case <-ctx.Done(): + return 0, ctx.Err() + } +} + +type missingProc struct{} + +func (missingProc) Attach(bool) runtime.Attachment { return &fakeAttachment{} } +func (missingProc) CloseStdin(context.Context) error { return guest.ErrNotFound } +func (missingProc) Resize(context.Context, guest.Size) error { return guest.ErrNotFound } +func (missingProc) Signal(context.Context, syscall.Signal) error { return guest.ErrNotFound } +func (missingProc) Status(context.Context) (guest.State, error) { + return guest.State{}, guest.ErrNotFound +} +func (missingProc) Wait(context.Context) (int, error) { return 0, guest.ErrNotFound } + +type fakeAttachment struct { + stdout, stderr io.Reader +} + +func (a *fakeAttachment) Stdin() io.WriteCloser { return nopWriteCloser{} } +func (a *fakeAttachment) Stdout() io.Reader { return a.stdout } +func (a *fakeAttachment) Stderr() io.Reader { return a.stderr } +func (a *fakeAttachment) Resize(context.Context, uint, uint) error { return nil } +func (a *fakeAttachment) Close() error { return nil } + +type nopWriteCloser struct{} + +func (nopWriteCloser) Write(p []byte) (int, error) { return len(p), nil } +func (nopWriteCloser) Close() error { return nil } diff --git a/internal/runtime/vm/helper.go b/internal/runtime/vm/helper.go new file mode 100644 index 000000000..d7fc60453 --- /dev/null +++ b/internal/runtime/vm/helper.go @@ -0,0 +1,54 @@ +package vm + +import ( + "context" + "syscall" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +type helper interface { + Ping(context.Context) (guest.Info, error) + Boot(context.Context, guest.BootSpec) error + Spawn(context.Context, guest.ProcessSpec) (process, error) + Process(id string) process + Stats(context.Context) (guest.Stats, error) + SyncClock(context.Context, time.Time) error + PowerOff(context.Context) error + Close() error +} + +type process interface { + Attach(fromStart bool) runtime.Attachment + CloseStdin(context.Context) error + Resize(context.Context, guest.Size) error + Signal(context.Context, syscall.Signal) error + Status(context.Context) (guest.State, error) + Wait(context.Context) (int, error) +} + +func dialHelper(vsockPath string) (helper, error) { + c, err := guest.Dial(vsockPath) + if err != nil { + return nil, err + } + return guestClient{c}, nil +} + +type guestClient struct{ *guest.Client } + +func (c guestClient) Spawn(ctx context.Context, spec guest.ProcessSpec) (process, error) { + p, err := c.Client.Spawn(ctx, spec) + if err != nil { + return nil, err + } + return guestProcess{p}, nil +} + +func (c guestClient) Process(id string) process { return guestProcess{c.Client.Process(id)} } + +type guestProcess struct{ *guest.Process } + +func (p guestProcess) Attach(fromStart bool) runtime.Attachment { return p.Process.Attach(fromStart) } diff --git a/internal/runtime/vm/kvm_integration_test.go b/internal/runtime/vm/kvm_integration_test.go new file mode 100644 index 000000000..2a83f0af9 --- /dev/null +++ b/internal/runtime/vm/kvm_integration_test.go @@ -0,0 +1,216 @@ +//go:build integration + +package vm + +import ( + "bufio" + "context" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +// kvmEngine runs the engine as the server does: real systemd units, the +// server binary as monitor and helper, and the image store. It needs root +// on a KVM host with cloud-hypervisor and virtiofsd, plus: +// +// AETHER_VM_TEST_DIR data directory, on the filesystem the store uses +// AETHER_VM_TEST_KERNEL guest kernel +// AETHER_VM_TEST_SERVER aether-server built from this tree +func kvmEngine(t *testing.T) *engine { + t.Helper() + dir, kernel, server := os.Getenv("AETHER_VM_TEST_DIR"), os.Getenv("AETHER_VM_TEST_KERNEL"), os.Getenv("AETHER_VM_TEST_SERVER") + if dir == "" || kernel == "" || server == "" { + t.Skip("set AETHER_VM_TEST_DIR, AETHER_VM_TEST_KERNEL and AETHER_VM_TEST_SERVER to boot real VMs") + } + if _, err := os.Stat("/dev/kvm"); err != nil || os.Geteuid() != 0 { + t.Skip("needs root and /dev/kvm") + } + images, err := vmimage.Open(filepath.Join(dir, "vm", "images"), kernel, nil) + if err != nil { + t.Fatal(err) + } + return newEngine(Config{DataDir: dir, Images: images, Monitor: []string{server, MonitorCommand}, Helper: server, + CloudHypervisor: "cloud-hypervisor", Virtiofsd: "/usr/libexec/virtiofsd"}, systemdUnits{}, dialHelper) +} + +func vmRun(t *testing.T, e *engine, key, script string) (string, int) { + t.Helper() + code, out, errOut, err := e.exec(t.Context(), key, []string{"/bin/sh", "-c", script}, "") + if err != nil { + t.Fatalf("exec %q: %v", script, err) + } + return out + errOut, code +} + +func readUntil(t *testing.T, att runtime.Attachment, want string) { + t.Helper() + found := make(chan bool, 1) + go func() { + sc := bufio.NewScanner(att.Stdout()) + for sc.Scan() { + if strings.Contains(sc.Text(), want) { + found <- true + return + } + } + found <- false + }() + select { + case ok := <-found: + if !ok { + t.Fatalf("main's output ended before %q", want) + } + case <-time.After(30 * time.Second): + t.Fatalf("no %q from main", want) + } +} + +func TestKVMSessionLifecycle(t *testing.T) { + e := kvmEngine(t) + key := "run-vmtest-a" + t.Cleanup(func() { _ = e.destroy(context.Background(), key) }) + src := filepath.Join(e.root, "..", "src-"+key) + if err := os.MkdirAll(filepath.Join(src, "checkout"), 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(src) }) + if err := os.WriteFile(filepath.Join(src, "checkout", "hello"), []byte("from host\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "ro"), []byte("read only\n"), 0o644); err != nil { + t.Fatal(err) + } + coord := filepath.Join(src, "coord") + if err := os.MkdirAll(coord, 0o755); err != nil { + t.Fatal(err) + } + l, err := net.Listen("unix", filepath.Join(coord, "coord3.sock")) + if err != nil { + t.Fatal(err) + } + defer func() { _ = l.Close() }() + go func() { + for c, acceptErr := l.Accept(); acceptErr == nil; c, acceptErr = l.Accept() { + _, _ = fmt.Fprintln(c, "host answered") + _ = c.Close() + } + }() + spec := runtime.Spec{ + Image: "docker.io/library/ubuntu:24.04", CreationKey: key, TTY: true, + Command: []string{"/bin/sh", "-c", "echo main-started; exec sleep infinity"}, + SetupScript: "echo setup-ran >> /var/tmp/setup-log", + Env: map[string]string{"AETHER_RUN_ID": key, "HOME": "/root"}, + CPULimit: 1.5, MemoryLimitBytes: 1 << 30, PidsLimit: 512, + WorktreeHostPath: filepath.Join(src, "checkout"), WorktreeMountPath: "/workspace", WorkingDir: "/workspace", + Mounts: []runtime.Mount{ + {HostPath: filepath.Join(src, "ro"), ContainerPath: "/opt/ro", ReadOnly: true}, + {HostPath: coord, ContainerPath: "/run/aether", ReadOnly: true}, + }, + } + start := time.Now() + id, err := e.create(t.Context(), spec) + if err != nil { + t.Fatal(err) + } + t.Logf("create (base disk, root and data disks) took %s", time.Since(start)) + start = time.Now() + if err = e.start(t.Context(), key); err != nil { + t.Fatal(err) + } + t.Logf("start (unit, boot, setup, main) took %s", time.Since(start)) + att, err := e.attach(t.Context(), key) + if err != nil { + t.Fatal(err) + } + readUntil(t, att, "main-started") + _ = att.Close() + + for script, want := range map[string]string{ + "cat /workspace/hello": "from host", + "pwd": "/workspace", + "cat /var/tmp/setup-log": "setup-ran", + "echo x >> /opt/ro || echo refused": "refused", + "nproc": "2", + "cat /proc/self/cgroup": "session", + "test -S /run/aether/coord3.sock && echo socket": "socket", + "getent hosts example.com >/dev/null && echo resolved": "resolved", + } { + if out, _ := vmRun(t, e, key, script); !strings.Contains(out, want) { + t.Errorf("%s: %q, want %q", script, out, want) + } + } + if out, code := vmRun(t, e, key, "echo from-guest > /workspace/written"); code != 0 { + t.Fatalf("write to the checkout: %d %s", code, out) + } + if data, _ := os.ReadFile(filepath.Join(src, "checkout", "written")); string(data) != "from-guest\n" { + t.Errorf("host sees %q", data) + } + + after := kvmEngine(t) + if found, err := after.find(key); err != nil || found != id { + t.Fatalf("a new engine finds %q, %v", found, err) + } + if info, err := after.inspect(t.Context(), key); err != nil || info.Exited { + t.Fatalf("a new engine inspects %+v, %v", info, err) + } + if out, _ := vmRun(t, after, key, "cat /var/tmp/setup-log | wc -l"); strings.TrimSpace(out) != "1" { + t.Errorf("setup ran %s times before the engine restart", out) + } + start = time.Now() + if err := after.stop(t.Context(), key, 2*time.Second); err != nil { + t.Fatal(err) + } + t.Logf("stop took %s", time.Since(start)) + if info, _ := after.inspect(t.Context(), key); !info.Exited { + t.Errorf("stopped VM inspects as %+v", info) + } + if err := after.start(t.Context(), key); err != nil { + t.Fatal(err) + } + if out, _ := vmRun(t, after, key, "cat /var/tmp/setup-log | wc -l"); strings.TrimSpace(out) != "1" { + t.Errorf("a cold start kept the old root disk: setup log has %s lines", out) + } +} + +func TestKVMSaveRestoreKeepsProcesses(t *testing.T) { + e := kvmEngine(t) + key := "run-vmtest-b" + t.Cleanup(func() { _ = e.destroy(context.Background(), key) }) + spec := runtime.Spec{ + Image: "docker.io/library/ubuntu:24.04", CreationKey: key, Command: []string{"/bin/sh", "-c", "exec sleep infinity"}, + Env: map[string]string{"AETHER_RUN_ID": key}, CPULimit: 2, MemoryLimitBytes: 1 << 30, + } + if _, err := e.create(t.Context(), spec); err != nil { + t.Fatal(err) + } + if err := e.start(t.Context(), key); err != nil { + t.Fatal(err) + } + before, _ := vmRun(t, e, key, "setsid sleep 4000 /dev/null 2>&1 & sleep 0.5; pgrep -x sleep | sort | tr '\\n' ' '; cat /proc/sys/kernel/random/boot_id") + start := time.Now() + if err := e.save(t.Context(), key); err != nil { + t.Fatal(err) + } + t.Logf("save took %s", time.Since(start)) + if up, _ := e.running(t.Context(), key); up { + t.Fatal("the VM still runs after save") + } + start = time.Now() + if err := e.restore(t.Context(), key); err != nil { + t.Fatal(err) + } + t.Logf("restore took %s", time.Since(start)) + after, _ := vmRun(t, e, key, "pgrep -x sleep | sort | tr '\\n' ' '; cat /proc/sys/kernel/random/boot_id") + if before != after { + t.Errorf("processes or boot changed across save and restore:\n%s\nvs\n%s", before, after) + } + t.Logf("same processes and boot after restore: %s", strings.TrimSpace(after)) +} diff --git a/internal/runtime/vm/lifecycle.go b/internal/runtime/vm/lifecycle.go new file mode 100644 index 000000000..e9fd8caf6 --- /dev/null +++ b/internal/runtime/vm/lifecycle.go @@ -0,0 +1,312 @@ +package vm + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "syscall" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +const ( + defaultStopGrace = 10 * time.Second + killGrace = 5 * time.Second + powerOffTimeout = 15 * time.Second + // lostExitCode reports a VM that ended without its helper reporting + // Main's exit: the host killed it, the guest crashed or the host + // rebooted. + lostExitCode = 255 + // killedExitCode is 128+SIGKILL, what a container reports after a stop + // had to kill. + killedExitCode = 137 + waitPoll = 200 * time.Millisecond +) + +func (e *engine) running(ctx context.Context, key string) (bool, error) { + st, err := e.units.State(ctx, unitName(key)) + return st.Active, err +} + +func (e *engine) requireRunning(ctx context.Context, key, action string) (*record, error) { + rec, err := e.load(key) + if err != nil { + return nil, err + } + up, err := e.running(ctx, key) + if err != nil { + return nil, err + } + if !up { + return nil, fmt.Errorf("vm runtime: %s VM %s: it is not running", action, key) + } + return rec, nil +} + +// stop ends Main as docker stop ends a container's main process. +func (e *engine) stop(ctx context.Context, key string, grace time.Duration) error { + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return err + } + up, err := e.running(ctx, key) + if err != nil || !up { + return err + } + api := newCHAPI(e.path(key, apiSocket)) + if state, stateErr := api.state(ctx); stateErr == nil && state == "Paused" { + if err = api.resume(ctx); err != nil { + return fmt.Errorf("vm runtime: resume before stop: %w", err) + } + } + if grace < 0 { + grace = defaultStopGrace + } + h, err := e.helper(key) + if err != nil { + return err + } + code := e.endMain(ctx, h.Process(guest.MainID), grace) + return e.finish(ctx, rec, code) +} + +func (e *engine) endMain(ctx context.Context, main process, grace time.Duration) int { + if err := main.Signal(ctx, syscall.SIGTERM); errors.Is(err, guest.ErrNotFound) { + return 0 + } + waitFor := func(d time.Duration) (int, bool) { + waitCtx, cancel := context.WithTimeout(ctx, d) + defer cancel() + code, err := main.Wait(waitCtx) + return code, err == nil + } + if code, ok := waitFor(grace); ok { + return code + } + _ = main.Signal(ctx, syscall.SIGKILL) + if code, ok := waitFor(killGrace); ok { + return code + } + return killedExitCode +} + +// finish records Main's exit and powers the VM off; it holds the key's lock. +func (e *engine) finish(ctx context.Context, rec *record, code int) error { + rec.Exited, rec.ExitCode = true, code + if err := e.store(rec); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), powerOffTimeout) + defer cancel() + if h, err := e.helper(rec.Key); err == nil { + _ = h.PowerOff(ctx) + } + e.forgetHelper(rec.Key) + for { + up, err := e.running(ctx, rec.Key) + if err == nil && !up { + return nil + } + select { + case <-ctx.Done(): + return e.units.Stop(context.WithoutCancel(ctx), unitName(rec.Key)) + case <-time.After(waitPoll): + } + } +} + +func (e *engine) kill(ctx context.Context, key string) { + e.forgetHelper(key) + _ = e.units.Stop(ctx, unitName(key)) +} + +// wait blocks until Main exits. On a VM that never booted it waits for the +// first boot, and on a saved VM until it is restored, so a save is not +// mistaken for an exit. +func (e *engine) wait(ctx context.Context, key string) (runtime.ExitStatus, error) { + for { + rec, err := e.load(key) + if err != nil { + return runtime.ExitStatus{}, err + } + if rec.Exited { + return runtime.ExitStatus{Code: rec.ExitCode}, nil + } + if rec.Booted && !rec.Saved && !e.flag(e.booting, key) { + up, err := e.running(ctx, key) + if err != nil { + return runtime.ExitStatus{}, err + } + if !up { + if code, err := e.lost(ctx, key); err != nil || code >= 0 { + return runtime.ExitStatus{Code: code}, err + } + continue + } + if code, ok := e.waitMain(ctx, key); ok { + return runtime.ExitStatus{Code: code}, nil + } + } + select { + case <-ctx.Done(): + return runtime.ExitStatus{}, ctx.Err() + case <-time.After(waitPoll): + } + } +} + +// waitMain waits for Main through the helper and finishes the VM; false +// means the helper went away first and the caller looks again. +func (e *engine) waitMain(ctx context.Context, key string) (int, bool) { + h, err := e.helper(key) + if err != nil { + return 0, false + } + code, err := h.Process(guest.MainID).Wait(ctx) + if err != nil { + return 0, false + } + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return 0, false + } + if rec.Exited { + return rec.ExitCode, true + } + if err := e.finish(ctx, rec, code); err != nil { + return 0, false + } + return code, true +} + +// lost records a booted VM whose unit is gone without an exit, and returns +// -1 when it turns out to have been stopped or saved properly meanwhile. +func (e *engine) lost(ctx context.Context, key string) (int, error) { + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return 0, err + } + if rec.Exited { + return rec.ExitCode, nil + } + if rec.Saved { + return -1, nil + } + up, err := e.running(ctx, key) + if err != nil || up { + return -1, err + } + rec.Exited, rec.ExitCode = true, lostExitCode + e.forgetHelper(key) + return lostExitCode, e.store(rec) +} + +// destroy removes the VM and every file it had; a missing VM is not an +// error. +func (e *engine) destroy(ctx context.Context, key string) error { + if checkKey(key) != nil { + return nil + } + unlock := e.lock(key) + defer unlock() + if _, err := os.Stat(e.dir(key)); errors.Is(err, fs.ErrNotExist) { + return nil + } + e.forgetHelper(key) + if err := e.units.Stop(ctx, unitName(key)); err != nil { + return err + } + if err := os.RemoveAll(e.dir(key)); err != nil { + return fmt.Errorf("vm runtime: remove VM %s: %w", key, err) + } + return nil +} + +// pause stops the vCPUs; the guest keeps its memory, as a frozen container +// does. +func (e *engine) pause(ctx context.Context, key string) error { + if _, err := e.requireRunning(ctx, key, "pause"); err != nil { + return err + } + if err := newCHAPI(e.path(key, apiSocket)).pause(ctx); err != nil { + return fmt.Errorf("vm runtime: pause VM %s: %w", key, err) + } + return nil +} + +func (e *engine) resume(ctx context.Context, key string) error { + if _, err := e.requireRunning(ctx, key, "resume"); err != nil { + return err + } + if err := newCHAPI(e.path(key, apiSocket)).resume(ctx); err != nil { + return fmt.Errorf("vm runtime: resume VM %s: %w", key, err) + } + return nil +} + +// inspect answers from the record and the unit, never from the guest. +func (e *engine) inspect(ctx context.Context, key string) (runtime.ContainerInfo, error) { + rec, err := e.load(key) + if err != nil { + return runtime.ContainerInfo{}, err + } + if rec.Booted && !rec.Exited && !rec.Saved && !e.flag(e.booting, key) { + up, err := e.running(ctx, key) + if err != nil { + return runtime.ContainerInfo{}, err + } + if !up { + if code, err := e.lost(ctx, key); err != nil { + return runtime.ContainerInfo{}, err + } else if code >= 0 { + rec.Exited, rec.ExitCode = true, code + } + } + } + targets := make([]string, len(rec.Mounts)) + for i, m := range rec.Mounts { + targets[i] = m.ContainerPath + } + return runtime.ContainerInfo{ + Image: rec.Image, User: rec.User, Env: append([]string(nil), rec.Env...), MountTargets: targets, + Fingerprint: rec.Fingerprint, Exited: rec.Exited, ExitCode: rec.ExitCode, + }, nil +} + +func (e *engine) find(key string) (runtime.ID, error) { + if _, err := e.load(key); err != nil { + return "", err + } + return runtime.ID(idPrefix + key), nil +} + +// initPID is Cloud Hypervisor's PID, whose cgroup is the VM's unit. +func (e *engine) initPID(ctx context.Context, key string) (int, error) { + if _, err := e.load(key); err != nil { + return 0, err + } + st, err := e.units.State(ctx, unitName(key)) + if err != nil || !st.Active { + return 0, err + } + return st.MainPID, nil +} + +func (e *engine) containerIP(ctx context.Context, key string) (string, error) { + rec, err := e.requireRunning(ctx, key, "address") + if err != nil { + return "", err + } + _, addr := netAddrs(rec.Net) + return addr.String(), nil +} diff --git a/internal/runtime/vm/managed.go b/internal/runtime/vm/managed.go new file mode 100644 index 000000000..6361a43a0 --- /dev/null +++ b/internal/runtime/vm/managed.go @@ -0,0 +1,259 @@ +package vm + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "path" + "strings" + "sync" + "time" + + "github.com/3xDevOps/Aether/internal/coordtransport" + "github.com/3xDevOps/Aether/internal/devexec" + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +// startManagedExec runs the staged dev-exec supervisor through the helper, +// as the Docker runtime runs it through docker exec: it waits for a claim +// before it starts the command, so a lost reply cannot orphan the command. +func (e *engine) startManagedExec(ctx context.Context, key string, spec runtime.ExecSpec, tty bool) (_ runtime.ManagedExec, resultErr error) { + if spec.CreationKey == "" || len(spec.CreationKey) > 1024 || strings.ContainsRune(spec.CreationKey, 0) { + return nil, errors.New("runtime: valid single-use execution creation key is required") + } + if len(spec.Argv) == 0 || spec.Argv[0] == "" { + return nil, errors.New("runtime: execution command is required") + } + for _, arg := range spec.Argv { + if strings.ContainsRune(arg, 0) { + return nil, errors.New("runtime: execution argument contains NUL") + } + } + if spec.WorkingDir != "" && !path.IsAbs(spec.WorkingDir) { + return nil, errors.New("runtime: execution working directory must be absolute") + } + if tty && (spec.Cols == 0 || spec.Rows == 0 || spec.Cols > 65535 || spec.Rows > 65535) { + return nil, errors.New("runtime: execution terminal dimensions must be between 1 and 65535") + } + rec, err := e.load(key) + if err != nil { + return nil, err + } + var token [32]byte + if _, err = rand.Read(token[:]); err != nil { + return nil, fmt.Errorf("runtime: create execution claim: %w", err) + } + claim := hex.EncodeToString(token[:]) + mode := "run-pipe" + var size *guest.Size + if tty { + mode, size = "run", &guest.Size{Cols: uint16(spec.Cols), Rows: uint16(spec.Rows)} + } + env := spec.Env + if spec.User != "" && spec.User != rec.User { + if _, gid, ok := strings.Cut(rec.User, ":"); ok && gid != "" { + env = append(append([]string{}, env...), devexec.ControlGroupEnv+"="+gid) + } + } + execID := processID("managed") + argv := append([]string{coordtransport.CLIPath, devexec.Command, mode, spec.CreationKey, claim}, spec.Argv...) + p, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: execID, Argv: argv, Env: env, User: spec.User, Dir: spec.WorkingDir, TTY: size, Lossless: spec.Lossless}) + if err != nil { + return nil, fmt.Errorf("runtime: create managed execution: %w", err) + } + m := &managedExec{engine: e, key: key, tty: tty, proc: p, attachment: att, identity: runtime.ExecIdentity{ + ContainerID: runtime.ID(idPrefix + key), ExecID: execID, CreationKey: spec.CreationKey, ClaimToken: claim, User: spec.User, + }} + published := false + defer func() { + if published { + return + } + cleanup, cancel := context.WithTimeout(context.Background(), devexec.StartupTimeout+5*time.Second) + defer cancel() + _, stopErr := m.control(cleanup, "stop", 0) + if stopErr == nil { + _, stopErr = m.Wait(cleanup) + } + _ = m.Detach() + if stopErr != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("runtime: cancelled execution cleanup: %w", stopErr)) + } + }() + if _, err := m.control(ctx, "start", 0); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + published = true + return m, nil +} + +// recoverExec proves the helper's claim. Like Docker's, the recovered +// execution has no attachment: callers reach a recovered command through its +// keeper. +func (e *engine) recoverExec(ctx context.Context, key string, identity runtime.ExecIdentity) (runtime.ManagedExec, error) { + if identity.ExecID == "" || identity.CreationKey == "" || identity.ClaimToken == "" { + return nil, fmt.Errorf("%w: incomplete identity", runtime.ErrExecUnavailable) + } + if _, err := e.requireRunning(ctx, key, "recover an execution in"); err != nil { + if errors.Is(err, runtime.ErrNotFound) { + return nil, err + } + return nil, fmt.Errorf("%w: %w: %w", runtime.ErrExecUnavailable, runtime.ErrNotFound, err) + } + h, err := e.helper(key) + if err != nil { + return nil, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + p := h.Process(identity.ExecID) + m := &managedExec{engine: e, key: key, proc: p, identity: identity} + _, err = p.Status(ctx) + if errors.Is(err, guest.ErrNotFound) { + return nil, fmt.Errorf("%w: %w: execution %s", runtime.ErrExecUnavailable, runtime.ErrNotFound, identity.ExecID) + } + if err != nil { + return nil, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + if _, err := m.control(ctx, "status", 0); err != nil { + return nil, fmt.Errorf("%w: cannot prove helper ownership: %w", runtime.ErrExecUnavailable, err) + } + return m, nil +} + +type managedExec struct { + engine *engine + key string + tty bool + proc process + identity runtime.ExecIdentity + + mu sync.Mutex + attachment runtime.Attachment +} + +func (m *managedExec) Identity() runtime.ExecIdentity { return m.identity } + +func (m *managedExec) Attachment() runtime.Attachment { + m.mu.Lock() + defer m.mu.Unlock() + return m.attachment +} + +func (m *managedExec) control(ctx context.Context, action string, grace time.Duration) (devexec.State, error) { + p, att, err := m.engine.spawn(ctx, m.key, guest.ProcessSpec{ + ID: processID("control"), Argv: runtime.ExecControlArgv(m.identity, action, grace), User: m.identity.User, Lossless: true, + }) + if err != nil { + return devexec.State{}, fmt.Errorf("%w: helper %s: %w", runtime.ErrExecUnavailable, action, err) + } + defer func() { _ = att.Close() }() + if err = p.CloseStdin(ctx); err != nil { + return devexec.State{}, fmt.Errorf("%w: helper %s: %w", runtime.ErrExecUnavailable, action, err) + } + stdout, stderr, err := readLimited(att) + if err != nil { + return devexec.State{}, fmt.Errorf("%w: helper %s output: %w", runtime.ErrExecUnavailable, action, err) + } + code, err := p.Wait(ctx) + if err != nil { + return devexec.State{}, fmt.Errorf("%w: helper %s: %w", runtime.ErrExecUnavailable, action, err) + } + return runtime.ExecControlState(m.identity, action, code, stdout, stderr) +} + +// Status asks the helper whether the supervisor runs; once it has ended, the +// supervisor's own record holds the command's exit. +func (m *managedExec) Status(ctx context.Context) (runtime.ExecState, error) { + st, err := m.proc.Status(ctx) + if errors.Is(err, guest.ErrNotFound) { + return runtime.ExecState{}, fmt.Errorf("%w: %w: execution %s", runtime.ErrExecUnavailable, runtime.ErrNotFound, m.identity.ExecID) + } + if err != nil { + return runtime.ExecState{}, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + state := runtime.ExecState{Running: !st.Exited, Attached: m.Attachment() != nil} + if !state.Attached { + state.UnavailableReason = "attachment is unavailable; the execution was detached" + } + if !st.Exited { + return state, nil + } + final, err := m.control(ctx, "status", 0) + if err != nil { + return state, fmt.Errorf("%w: command exit record unavailable: %w", runtime.ErrExecUnavailable, err) + } + if !final.Exited || final.ExitCode == nil { + return state, fmt.Errorf("%w: helper ended without a command exit record", runtime.ErrExecUnavailable) + } + state.Running, state.Exited, state.ExitCode = false, true, final.ExitCode + return state, nil +} + +func (m *managedExec) Wait(ctx context.Context) (runtime.ExitStatus, error) { + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + for { + state, err := m.Status(ctx) + if err != nil { + return runtime.ExitStatus{}, err + } + if state.Exited { + return runtime.ExitStatus{Code: *state.ExitCode}, nil + } + select { + case <-ctx.Done(): + return runtime.ExitStatus{}, ctx.Err() + case <-tick.C: + } + } +} + +func (m *managedExec) Stop(ctx context.Context, grace time.Duration) (runtime.ExitStatus, error) { + if grace < 0 { + grace = 3 * time.Second + } + if grace > time.Minute { + return runtime.ExitStatus{}, errors.New("runtime: managed execution stop grace exceeds one minute") + } + ctx, cancel := context.WithTimeout(ctx, devexec.StartupTimeout+grace+devexec.ReapTimeout) + defer cancel() + st, err := m.proc.Status(ctx) + if err != nil && !errors.Is(err, guest.ErrNotFound) { + return runtime.ExitStatus{}, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + if err == nil && !st.Exited { + if _, err := m.control(ctx, "stop", grace); err != nil { + return runtime.ExitStatus{}, err + } + } + return m.Wait(ctx) +} + +func (m *managedExec) Resize(ctx context.Context, cols, rows uint) error { + if cols == 0 || rows == 0 || cols > 65535 || rows > 65535 { + return errors.New("runtime: execution terminal dimensions must be between 1 and 65535") + } + if m.Attachment() == nil { + return fmt.Errorf("%w: cannot resize an unavailable PTY", runtime.ErrExecUnavailable) + } + if !m.tty { + return errors.New("runtime: resize: execution has no terminal") + } + return m.proc.Resize(ctx, guest.Size{Cols: uint16(cols), Rows: uint16(rows)}) +} + +func (m *managedExec) Detach() error { + m.mu.Lock() + attachment := m.attachment + m.attachment = nil + m.mu.Unlock() + if attachment != nil { + return attachment.Close() + } + return nil +} diff --git a/internal/runtime/vm/monitor.go b/internal/runtime/vm/monitor.go new file mode 100644 index 000000000..77f25e421 --- /dev/null +++ b/internal/runtime/vm/monitor.go @@ -0,0 +1,30 @@ +package vm + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +// MonitorCommand is the aether-server subcommand a VM's unit runs. +const MonitorCommand = "vm-monitor" + +type launch struct { + Shares []share `json:"shares,omitempty"` + Virtiofsd string `json:"virtiofsd,omitempty"` + CloudHypervisor string `json:"cloud_hypervisor"` + Args []string `json:"args"` +} + +func readLaunch(dir string) (launch, error) { + data, err := os.ReadFile(filepath.Join(dir, launchFile)) + if err != nil { + return launch{}, fmt.Errorf("vm monitor: %w", err) + } + var l launch + if err := json.Unmarshal(data, &l); err != nil { + return launch{}, fmt.Errorf("vm monitor: decode %s: %w", launchFile, err) + } + return l, nil +} diff --git a/internal/runtime/vm/monitor_linux.go b/internal/runtime/vm/monitor_linux.go new file mode 100644 index 000000000..258082b72 --- /dev/null +++ b/internal/runtime/vm/monitor_linux.go @@ -0,0 +1,148 @@ +package vm + +import ( + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strconv" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +const virtiofsdReadyTimeout = 10 * time.Second + +// Monitor runs in a VM's unit, whose private mount namespace keeps every +// bind it makes off the host: it composes the share, starts virtiofsd on +// it, then becomes Cloud Hypervisor, so the unit ends with the VM. +func Monitor(args []string) error { + if len(args) != 1 || !filepath.IsAbs(args[0]) { + return errors.New("usage: aether-server vm-monitor ") + } + dir := args[0] + l, err := readLaunch(dir) + if err != nil { + return err + } + for _, name := range []string{apiSocket, vsockSocket, fsSocket} { + if err = os.Remove(filepath.Join(dir, name)); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("vm monitor: remove stale %s: %w", name, err) + } + } + if len(l.Shares) > 0 { + root := filepath.Join(dir, shareDir) + if err = composeShare(root, l.Shares); err != nil { + return err + } + if err = startVirtiofsd(l.Virtiofsd, filepath.Join(dir, fsSocket), root); err != nil { + return err + } + } + argv := append([]string{l.CloudHypervisor}, l.Args...) + path, err := exec.LookPath(l.CloudHypervisor) + if err != nil { + return fmt.Errorf("vm monitor: %w", err) + } + return syscall.Exec(path, argv, os.Environ()) +} + +// composeShare mounts a small tmpfs at root and binds every share at +// root/: nosuid and nodev always, read-only where asked, and +// recursively, like Docker's rbind. The tmpfs itself ends read-only. +func composeShare(root string, shares []share) error { + if err := os.MkdirAll(root, 0o755); err != nil { + return fmt.Errorf("vm monitor: %w", err) + } + if err := unix.Mount("aether-share", root, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, "mode=0755,size=1m"); err != nil { + return fmt.Errorf("vm monitor: mount share root: %w", err) + } + for i, s := range shares { + if err := bindShare(filepath.Join(root, strconv.Itoa(i)), s); err != nil { + return err + } + } + if err := unix.MountSetattr(unix.AT_FDCWD, root, 0, &unix.MountAttr{Attr_set: unix.MOUNT_ATTR_RDONLY}); err != nil { + return fmt.Errorf("vm monitor: make share root read-only: %w", err) + } + return nil +} + +// bindShare resolves a subpath beneath its base without following a symlink +// out of it, as Docker's volume subpath does, because an agent may control +// the base's content. +func bindShare(target string, s share) error { + fd, err := unix.Open(s.Source, unix.O_PATH|unix.O_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("vm monitor: open %s: %w", s.Source, err) + } + defer func() { _ = unix.Close(fd) }() + if s.Subpath != "" { + sub, openErr := unix.Openat2(fd, s.Subpath, &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS, + }) + if openErr != nil { + return fmt.Errorf("vm monitor: resolve %s beneath %s: %w", s.Subpath, s.Source, openErr) + } + _ = unix.Close(fd) + fd = sub + } + var st unix.Stat_t + if err = unix.Fstat(fd, &st); err != nil { + return fmt.Errorf("vm monitor: stat %s: %w", s.Source, err) + } + if st.Mode&unix.S_IFMT == unix.S_IFDIR { + err = os.Mkdir(target, 0o755) + } else { + err = os.WriteFile(target, nil, 0o644) + } + if err != nil { + return fmt.Errorf("vm monitor: %w", err) + } + tree, err := unix.OpenTree(fd, "", unix.OPEN_TREE_CLONE|unix.OPEN_TREE_CLOEXEC|unix.AT_EMPTY_PATH|unix.AT_RECURSIVE) + if err != nil { + return fmt.Errorf("vm monitor: clone %s: %w", s.Source, err) + } + defer func() { _ = unix.Close(tree) }() + attr := unix.MountAttr{Attr_set: unix.MOUNT_ATTR_NOSUID | unix.MOUNT_ATTR_NODEV, Propagation: unix.MS_PRIVATE} + if s.ReadOnly { + attr.Attr_set |= unix.MOUNT_ATTR_RDONLY + } + if err := unix.MountSetattr(tree, "", unix.AT_EMPTY_PATH|unix.AT_RECURSIVE, &attr); err != nil { + return fmt.Errorf("vm monitor: set mount flags on %s: %w", s.Source, err) + } + if err := unix.MoveMount(tree, "", unix.AT_FDCWD, target, unix.MOVE_MOUNT_F_EMPTY_PATH); err != nil { + return fmt.Errorf("vm monitor: bind %s: %w", s.Source, err) + } + return nil +} + +// startVirtiofsd leaves virtiofsd as a child that outlives the exec into +// Cloud Hypervisor; the unit stops it with the VM. +func startVirtiofsd(binary, socket, root string) error { + cmd := exec.Command(binary, "--socket-path="+socket, "--shared-dir="+root, + "--announce-submounts", "--xattr", "--inode-file-handles=prefer", "--sandbox=namespace") + cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr + if err := cmd.Start(); err != nil { + return fmt.Errorf("vm monitor: start virtiofsd: %w", err) + } + exited := make(chan error, 1) + go func() { exited <- cmd.Wait() }() + deadline := time.After(virtiofsdReadyTimeout) + for { + if _, err := os.Stat(socket); err == nil { + return nil + } + select { + case err := <-exited: + return fmt.Errorf("vm monitor: virtiofsd exited before it listened: %v", err) + case <-deadline: + return fmt.Errorf("vm monitor: virtiofsd did not listen on %s within %s", socket, virtiofsdReadyTimeout) + case <-time.After(10 * time.Millisecond): + } + } +} diff --git a/internal/runtime/vm/monitor_other.go b/internal/runtime/vm/monitor_other.go new file mode 100644 index 000000000..e1916f2f4 --- /dev/null +++ b/internal/runtime/vm/monitor_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package vm + +import "errors" + +func Monitor([]string) error { + return errors.New("vm monitor: session VMs run only on Linux") +} diff --git a/internal/runtime/vm/network.go b/internal/runtime/vm/network.go new file mode 100644 index 000000000..7256bbebc --- /dev/null +++ b/internal/runtime/vm/network.go @@ -0,0 +1,150 @@ +package vm + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net/netip" + "os" + "strings" + + "github.com/3xDevOps/Aether/internal/guest" +) + +// vmSubnet holds one /30 per VM: .1 is the host end of its tap, .2 the +// guest. It sits outside Docker's default address pools. +var vmSubnet = netip.MustParsePrefix("10.200.0.0/16") + +const ( + tapPrefix = "aethervm" + maxNets = 1 << 14 +) + +// dockerDNS is what Docker gives a container when the host resolver is +// loopback only. +var dockerDNS = []netip.Addr{netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("8.8.4.4")} + +func tapName(index int) string { return fmt.Sprintf("%s%d", tapPrefix, index) } + +func netAddrs(index int) (host, guestAddr netip.Addr) { + base := vmSubnet.Addr().As4() + n := index * 4 + base[2], base[3] = byte(n>>8), byte(n) + host = netip.AddrFrom4(base).Next() + return host, host.Next() +} + +// chNetArg lets Cloud Hypervisor create the tap and give its host end an +// address; the tap goes away with the VM. +func chNetArg(index int) string { + host, _ := netAddrs(index) + return fmt.Sprintf("tap=%s,ip=%s,mask=255.255.255.252,mac=02:ae:00:00:%02x:%02x", tapName(index), host, byte(index>>8), byte(index)) +} + +func guestNetwork(index int, dns []netip.Addr) *guest.Network { + host, guestAddr := netAddrs(index) + return &guest.Network{Address: netip.PrefixFrom(guestAddr, 30), Gateway: host, DNS: dns} +} + +// allocateNet returns the lowest index no recorded or reserved VM holds. +// The caller releases the reservation once the record is stored. +func (e *engine) allocateNet() (int, error) { + e.netMu.Lock() + defer e.netMu.Unlock() + used := make(map[int]bool, len(e.reserved)) + for index := range e.reserved { + used[index] = true + } + keys, err := e.keys() + if err != nil { + return 0, err + } + for _, key := range keys { + rec, err := e.load(key) + if err != nil { + return 0, err + } + used[rec.Net] = true + } + for index := range maxNets { + if !used[index] { + e.reserved[index] = true + return index, nil + } + } + return 0, fmt.Errorf("vm runtime: all %d VM networks in %s are in use", maxNets, vmSubnet) +} + +func (e *engine) releaseNet(index int) { + e.netMu.Lock() + delete(e.reserved, index) + e.netMu.Unlock() +} + +// ensureNAT runs once per process; a failure is retried at the next start. +func (e *engine) ensureNAT(ctx context.Context) error { + e.netMu.Lock() + defer e.netMu.Unlock() + if e.natReady { + return nil + } + if err := e.run(ctx, "sysctl", "-q", "-w", "net.ipv4.ip_forward=1"); err != nil { + return err + } + subnet, taps := vmSubnet.String(), tapPrefix+"+" + for _, rule := range [][]string{ + {"-t", "nat", "POSTROUTING", "-s", subnet, "!", "-d", subnet, "-j", "MASQUERADE"}, + {"-t", "filter", "FORWARD", "-i", taps, "-j", "ACCEPT"}, + {"-t", "filter", "FORWARD", "-o", taps, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}, + } { + table, chain, match := rule[:2], rule[2], rule[3:] + check := append(append(append([]string{}, table...), "-C", chain), match...) + if e.run(ctx, "iptables", check...) == nil { + continue + } + insert := append(append(append([]string{}, table...), "-I", chain), match...) + if err := e.run(ctx, "iptables", insert...); err != nil { + return err + } + } + e.natReady = true + return nil +} + +// hostDNS prefers systemd-resolved's upstream servers, since its stub at +// 127.0.0.53 is unreachable from a guest. +func hostDNS() []netip.Addr { + for _, path := range []string{"/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"} { + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + break + } + servers := nameservers(f) + _ = f.Close() + if len(servers) > 0 { + return servers + } + } + return dockerDNS +} + +func nameservers(r io.Reader) []netip.Addr { + var out []netip.Addr + sc := bufio.NewScanner(r) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) < 2 || fields[0] != "nameserver" { + continue + } + addr, err := netip.ParseAddr(fields[1]) + if err == nil && addr.Is4() && !addr.IsLoopback() { + out = append(out, addr) + } + } + return out +} diff --git a/internal/runtime/vm/network_test.go b/internal/runtime/vm/network_test.go new file mode 100644 index 000000000..e10b3bf6a --- /dev/null +++ b/internal/runtime/vm/network_test.go @@ -0,0 +1,84 @@ +package vm + +import ( + "context" + "errors" + "net/netip" + "slices" + "strings" + "testing" +) + +func TestNetworkAddressing(t *testing.T) { + host, guestAddr := netAddrs(65) + if host.String() != "10.200.1.5" || guestAddr.String() != "10.200.1.6" { + t.Errorf("addresses of 65 = %s, %s", host, guestAddr) + } + if got := chNetArg(65); got != "tap=aethervm65,ip=10.200.1.5,mask=255.255.255.252,mac=02:ae:00:00:00:41" { + t.Errorf("net arg = %s", got) + } + last, _ := netAddrs(maxNets - 1) + if !vmSubnet.Contains(last) { + t.Errorf("last network %s leaves %s", last, vmSubnet) + } + n := guestNetwork(0, dockerDNS) + if n.Address != netip.MustParsePrefix("10.200.0.2/30") || n.Gateway.String() != "10.200.0.1" { + t.Errorf("guest network = %+v", n) + } + servers := nameservers(strings.NewReader("# x\nnameserver 127.0.0.53\nnameserver 10.0.0.2\nnameserver fe80::1\nsearch ec2.internal\n")) + if !slices.Equal(servers, []netip.Addr{netip.MustParseAddr("10.0.0.2")}) { + t.Errorf("nameservers = %v", servers) + } +} + +func TestAllocateNetSkipsRecordedAndReserved(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + first, err := h.engine.allocateNet() + if err != nil || first != 1 { + t.Fatalf("allocate = %d, %v", first, err) + } + second, err := h.engine.allocateNet() + if err != nil || second != 2 { + t.Fatalf("allocate while 1 is reserved = %d, %v", second, err) + } + h.engine.releaseNet(first) + if again, _ := h.engine.allocateNet(); again != 1 { + t.Errorf("allocate after release = %d", again) + } +} + +func TestEnsureNATAddsOnlyMissingRules(t *testing.T) { + h := newHarness(t) + e := h.engine + e.natReady = false + var calls []string + e.run = func(_ context.Context, name string, args ...string) error { + calls = append(calls, strings.Join(append([]string{name}, args...), " ")) + if slices.Contains(args, "-C") && slices.Contains(args, "MASQUERADE") { + return nil + } + if slices.Contains(args, "-C") { + return errors.New("no such rule") + } + return nil + } + if err := e.ensureNAT(t.Context()); err != nil { + t.Fatal(err) + } + want := []string{ + "sysctl -q -w net.ipv4.ip_forward=1", + "iptables -t nat -C POSTROUTING -s 10.200.0.0/16 ! -d 10.200.0.0/16 -j MASQUERADE", + "iptables -t filter -C FORWARD -i aethervm+ -j ACCEPT", + "iptables -t filter -I FORWARD -i aethervm+ -j ACCEPT", + "iptables -t filter -C FORWARD -o aethervm+ -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT", + "iptables -t filter -I FORWARD -o aethervm+ -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT", + } + if !slices.Equal(calls, want) { + t.Errorf("calls:\n%s", strings.Join(calls, "\n")) + } + calls = nil + if err := e.ensureNAT(t.Context()); err != nil || calls != nil { + t.Errorf("second ensureNAT ran %v, %v", calls, err) + } +} diff --git a/internal/runtime/vm/record.go b/internal/runtime/vm/record.go new file mode 100644 index 000000000..6cd65e823 --- /dev/null +++ b/internal/runtime/vm/record.go @@ -0,0 +1,149 @@ +package vm + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +// Files in a VM's directory, /vm//. +const ( + recordFile = "vm.json" + launchFile = "launch.json" + apiSocket = "api.sock" + vsockSocket = "vsock.sock" + fsSocket = "fs.sock" + shareDir = "share" + helperImage = "helper.img" + rootDisk = "root.img" + dataDisk = "data.img" + consoleLog = "console.log" + monitorLog = "monitor.log" + snapshotDir = "snapshot" +) + +// keyPattern keeps a creation key usable as a directory, a systemd unit name +// and a guest hostname. +var keyPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +// record is everything a VM needs to boot again and to answer Inspect +// without reaching the guest. Env is the image's environment overlaid with +// the spec's, which is what a container reports. +type record struct { + Key string `json:"key"` + Fingerprint string `json:"fingerprint,omitempty"` + Image string `json:"image"` + Base vmimage.Base `json:"base"` + Root vmimage.Disk `json:"root"` + Data vmimage.Disk `json:"data"` + User string `json:"user,omitempty"` + Env []string `json:"env"` + WorkingDir string `json:"working_dir,omitempty"` + Command []string `json:"command"` + TTY bool `json:"tty"` + SetupScript string `json:"setup_script,omitempty"` + SecretEnv []string `json:"secret_env,omitempty"` + NestedDocker bool `json:"nested_docker,omitempty"` + Sudo bool `json:"sudo,omitempty"` + CPUs int `json:"cpus"` + CPULimit float64 `json:"cpu_limit,omitempty"` + MemoryMiB int64 `json:"memory_mib"` + PidsLimit int64 `json:"pids_limit,omitempty"` + Mounts []runtime.Mount `json:"mounts,omitempty"` + Net int `json:"net"` + // Booted is set once a boot was launched: the next cold start builds a + // fresh root disk. + Booted bool `json:"booted,omitempty"` + // Saved means the snapshot directory holds the VM's state and the VM is + // stopped until Restore. + Saved bool `json:"saved,omitempty"` + Exited bool `json:"exited,omitempty"` + ExitCode int `json:"exit_code,omitempty"` +} + +func (e *engine) dir(key string) string { return filepath.Join(e.root, key) } + +func (e *engine) path(key, name string) string { return filepath.Join(e.root, key, name) } + +// imagesDir is the image store's directory beside the VMs. +const imagesDir = "images" + +func checkKey(key string) error { + if !keyPattern.MatchString(key) || key == imagesDir { + return fmt.Errorf("vm runtime: creation key %q is not a usable VM name (lowercase letters, digits and dashes)", key) + } + return nil +} + +// load returns runtime.ErrNotFound for a key with no VM. +func (e *engine) load(key string) (*record, error) { + if err := checkKey(key); err != nil { + return nil, fmt.Errorf("%w: %w", runtime.ErrNotFound, err) + } + data, err := os.ReadFile(e.path(key, recordFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("vm runtime: VM %q: %w", key, runtime.ErrNotFound) + } + if err != nil { + return nil, fmt.Errorf("vm runtime: read VM %q: %w", key, err) + } + var rec record + if err := json.Unmarshal(data, &rec); err != nil { + return nil, fmt.Errorf("vm runtime: decode VM %q: %w", key, err) + } + return &rec, nil +} + +func (e *engine) store(rec *record) error { + data, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return fmt.Errorf("vm runtime: encode VM %q: %w", rec.Key, err) + } + return writeFileAtomic(e.path(rec.Key, recordFile), data) +} + +func writeFileAtomic(path string, data []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*") + if err != nil { + return fmt.Errorf("vm runtime: write %s: %w", path, err) + } + _, werr := tmp.Write(data) + serr := tmp.Sync() + cerr := tmp.Close() + if err := errors.Join(werr, serr, cerr); err != nil { + _ = os.Remove(tmp.Name()) + return fmt.Errorf("vm runtime: write %s: %w", path, err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + _ = os.Remove(tmp.Name()) + return fmt.Errorf("vm runtime: write %s: %w", path, err) + } + return nil +} + +func (e *engine) keys() ([]string, error) { + entries, err := os.ReadDir(e.root) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("vm runtime: list VMs: %w", err) + } + var keys []string + for _, entry := range entries { + if !entry.IsDir() || checkKey(entry.Name()) != nil { + continue + } + if _, err := os.Stat(e.path(entry.Name(), recordFile)); err == nil { + keys = append(keys, entry.Name()) + } + } + return keys, nil +} diff --git a/internal/runtime/vm/save.go b/internal/runtime/vm/save.go new file mode 100644 index 000000000..554614857 --- /dev/null +++ b/internal/runtime/vm/save.go @@ -0,0 +1,161 @@ +package vm + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" +) + +const restoreTimeout = 60 * time.Second + +// save pauses the VM, writes its memory and device state to the snapshot +// directory and ends the monitor. A VM with a virtio-fs share is refused: a +// fresh virtiofsd after restore knows none of the guest's open files, so +// every access to the share fails with EBADF. +func (e *engine) save(ctx context.Context, key string) error { + unlock := e.lock(key) + defer unlock() + rec, err := e.requireRunning(ctx, key, "save") + if err != nil { + return err + } + p, err := planMounts(rec.Mounts, false, e.path(key, vsockSocket)) + if err != nil { + return err + } + if len(p.shares) > 0 { + targets := make([]string, len(p.mounts)) + for i, m := range p.mounts { + targets[i] = m.Target + } + return fmt.Errorf("vm runtime: cannot save VM %s: its virtio-fs mounts (%s) would come back unusable after a restore; work items 6 and 7 move them off virtio-fs", key, strings.Join(targets, ", ")) + } + api := newCHAPI(e.path(key, apiSocket)) + if err := api.pause(ctx); err != nil { + return fmt.Errorf("vm runtime: save VM %s: %w", key, err) + } + dir := e.path(key, snapshotDir) + if err := os.RemoveAll(dir); err != nil { + return e.resumeAfter(ctx, api, key, err) + } + if err := os.Mkdir(dir, 0o700); err != nil { + return e.resumeAfter(ctx, api, key, err) + } + if err := api.snapshot(ctx, dir); err != nil { + _ = os.RemoveAll(dir) + return e.resumeAfter(ctx, api, key, err) + } + rec.Saved = true + if err := e.store(rec); err != nil { + return e.resumeAfter(ctx, api, key, err) + } + e.forgetHelper(key) + if err := api.shutdownVMM(ctx); err != nil { + return e.units.Stop(context.WithoutCancel(ctx), unitName(key)) + } + return e.awaitStopped(ctx, key) +} + +func (e *engine) resumeAfter(ctx context.Context, api chAPI, key string, cause error) error { + if err := api.resume(context.WithoutCancel(ctx)); err != nil { + return fmt.Errorf("vm runtime: save VM %s: %w; and it stays paused: %w", key, cause, err) + } + return fmt.Errorf("vm runtime: save VM %s: %w", key, cause) +} + +func (e *engine) awaitStopped(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), powerOffTimeout) + defer cancel() + for { + up, err := e.running(ctx, key) + if err == nil && !up { + return nil + } + select { + case <-ctx.Done(): + return e.units.Stop(context.WithoutCancel(ctx), unitName(key)) + case <-time.After(waitPoll): + } + } +} + +// restore starts the monitor on the saved state, resumes the guest and sets +// its clock; the snapshot is deleted once the guest answers. +func (e *engine) restore(ctx context.Context, key string) (err error) { + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return err + } + if !rec.Saved { + return fmt.Errorf("vm runtime: restore VM %s: it has no saved state", key) + } + if up, upErr := e.running(ctx, key); upErr != nil || up { + return errors.Join(upErr, fmt.Errorf("vm runtime: restore VM %s: it is already running", key)) + } + dir := e.path(key, snapshotDir) + l := launch{CloudHypervisor: e.cfg.CloudHypervisor, Args: []string{ + "--api-socket", "path=" + e.path(key, apiSocket), "--restore", "source_url=file://" + dir, + }} + if err = e.ensureNAT(ctx); err != nil { + return err + } + if err = e.launch(ctx, rec, l); err != nil { + return err + } + defer func() { + if err != nil { + e.kill(context.WithoutCancel(ctx), key) + } + }() + api := newCHAPI(e.path(key, apiSocket)) + if err = e.awaitRestored(ctx, key, api); err != nil { + return err + } + if err = api.resume(ctx); err != nil { + return fmt.Errorf("vm runtime: resume restored VM %s: %w", key, err) + } + h, err := e.helper(key) + if err != nil { + return err + } + if err := e.waitHelper(ctx, key, h); err != nil { + return err + } + if err := h.SyncClock(ctx, time.Now()); err != nil { + return fmt.Errorf("vm runtime: set the clock of restored VM %s: %w", key, err) + } + rec.Saved = false + if err := e.store(rec); err != nil { + return err + } + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("vm runtime: remove the saved state of VM %s: %w", key, err) + } + return nil +} + +// awaitRestored waits until Cloud Hypervisor holds the restored, still +// paused VM. +func (e *engine) awaitRestored(ctx context.Context, key string, api chAPI) error { + ctx, cancel := context.WithTimeout(ctx, restoreTimeout) + defer cancel() + for { + state, err := api.state(ctx) + if err == nil && state == "Paused" { + return nil + } + if up, upErr := e.running(ctx, key); upErr == nil && !up { + return fmt.Errorf("vm runtime: VM %s stopped while restoring%s", key, e.logTail(key)) + } + select { + case <-ctx.Done(): + return fmt.Errorf("vm runtime: restore VM %s: %w (last state %q, %v)%s", key, ctx.Err(), state, err, e.logTail(key)) + case <-time.After(waitPoll): + } + } +} diff --git a/internal/runtime/vm/share.go b/internal/runtime/vm/share.go new file mode 100644 index 000000000..9c6767f3f --- /dev/null +++ b/internal/runtime/vm/share.go @@ -0,0 +1,120 @@ +package vm + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +// share is one host path the monitor binds into the VM's share directory, +// at share/. +type share struct { + Source string `json:"source"` + Subpath string `json:"subpath,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` +} + +// firstSocketPort leaves 1024 to the helper's own API. +const firstSocketPort = 1025 + +type plan struct { + shares []share + mounts []guest.Mount + sockets []guest.Socket + // links maps each vsock.sock_ to the host socket Cloud Hypervisor + // connects to for that port. + links map[string]string +} + +// specMounts lists a spec's mounts in Docker's order: the checkout, extra +// worktrees, then the rest. +func specMounts(spec runtime.Spec) []runtime.Mount { + var out []runtime.Mount + if spec.WorktreeHostPath != "" { + out = append(out, runtime.Mount{HostPath: spec.WorktreeHostPath, ContainerPath: spec.WorktreeMountPath}) + } + out = append(out, spec.ExtraWorktrees...) + return append(out, spec.Mounts...) +} + +// planMounts reads the host once per boot. A unix socket cannot cross +// virtio-fs, so sockets at the top of a shared directory are forwarded, and +// a directory holding nothing but sockets is not shared at all: the helper +// makes it on the guest's /run. +func planMounts(mounts []runtime.Mount, nestedDocker bool, vsock string) (plan, error) { + p := plan{links: map[string]string{}} + forward := func(host, target string) { + port := firstSocketPort + len(p.sockets) + p.sockets = append(p.sockets, guest.Socket{Port: uint32(port), Path: target}) + p.links[vsock+"_"+strconv.Itoa(port)] = host + } + for _, m := range mounts { + if m.Subpath == "" { + info, err := os.Stat(m.HostPath) + if err != nil { + return plan{}, fmt.Errorf("vm runtime: mount %s at %s: %w", m.HostPath, m.ContainerPath, err) + } + if info.Mode()&fs.ModeSocket != 0 { + forward(m.HostPath, m.ContainerPath) + continue + } + if info.IsDir() { + sockets, others, err := topSockets(m.HostPath) + if err != nil { + return plan{}, err + } + for _, name := range sockets { + forward(filepath.Join(m.HostPath, name), filepath.Join(m.ContainerPath, name)) + } + if len(sockets) > 0 && !others { + continue + } + } + } + p.mounts = append(p.mounts, guest.Mount{Volume: guest.VolumeShare, Source: strconv.Itoa(len(p.shares)), Target: m.ContainerPath, ReadOnly: m.ReadOnly}) + p.shares = append(p.shares, share{Source: m.HostPath, Subpath: m.Subpath, ReadOnly: m.ReadOnly}) + } + if nestedDocker { + p.mounts = append(p.mounts, guest.Mount{Volume: guest.VolumeData, Source: "docker", Target: "/var/lib/docker"}) + } + return p, nil +} + +func topSockets(dir string) (sockets []string, others bool, err error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, false, fmt.Errorf("vm runtime: read mount %s: %w", dir, err) + } + for _, entry := range entries { + if entry.Type()&fs.ModeSocket != 0 { + sockets = append(sockets, entry.Name()) + } else { + others = true + } + } + return sockets, others, nil +} + +func linkSockets(dir string, links map[string]string) error { + old, err := filepath.Glob(filepath.Join(dir, vsockSocket+"_*")) + if err != nil { + return err + } + for _, path := range old { + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("vm runtime: remove socket link: %w", err) + } + } + for link, target := range links { + if err := os.Symlink(target, link); err != nil { + return fmt.Errorf("vm runtime: link socket %s: %w", target, err) + } + } + return nil +} diff --git a/internal/runtime/vm/share_test.go b/internal/runtime/vm/share_test.go new file mode 100644 index 000000000..3ab928897 --- /dev/null +++ b/internal/runtime/vm/share_test.go @@ -0,0 +1,90 @@ +package vm + +import ( + "net" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +func listen(t *testing.T, path string) { + t.Helper() + l, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = l.Close() }) +} + +func TestPlanMountsSharesFilesAndForwardsSockets(t *testing.T) { + dir, err := os.MkdirTemp("", "vmp") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + home, coord, signing := filepath.Join(dir, "home"), filepath.Join(dir, "coord"), filepath.Join(dir, "sign") + for _, d := range []string{home, coord, signing} { + if err = os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + bin := filepath.Join(dir, "aether-server") + if err = os.WriteFile(bin, nil, 0o755); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(coord, "next-command"), nil, 0o644); err != nil { + t.Fatal(err) + } + listen(t, filepath.Join(coord, "coord3.sock")) + listen(t, filepath.Join(signing, "agent.sock")) + + mounts := []runtime.Mount{ + {HostPath: home, ContainerPath: "/root"}, + {HostPath: bin, ContainerPath: "/opt/aether/aether-server", ReadOnly: true}, + {HostPath: coord, ContainerPath: "/run/aether", ReadOnly: true}, + {HostPath: signing, ContainerPath: "/run/aether-signing", ReadOnly: true}, + {HostPath: home, Subpath: ".claude", ContainerPath: "/root/.claude", ReadOnly: true}, + } + vsock := filepath.Join(dir, vsockSocket) + p, err := planMounts(mounts, true, vsock) + if err != nil { + t.Fatal(err) + } + wantShares := []share{{Source: home}, {Source: bin, ReadOnly: true}, {Source: coord, ReadOnly: true}, {Source: home, Subpath: ".claude", ReadOnly: true}} + if !slices.Equal(p.shares, wantShares) { + t.Errorf("shares = %+v", p.shares) + } + wantMounts := []guest.Mount{ + {Volume: guest.VolumeShare, Source: "0", Target: "/root"}, + {Volume: guest.VolumeShare, Source: "1", Target: "/opt/aether/aether-server", ReadOnly: true}, + {Volume: guest.VolumeShare, Source: "2", Target: "/run/aether", ReadOnly: true}, + {Volume: guest.VolumeShare, Source: "3", Target: "/root/.claude", ReadOnly: true}, + {Volume: guest.VolumeData, Source: "docker", Target: "/var/lib/docker"}, + } + if !slices.Equal(p.mounts, wantMounts) { + t.Errorf("mounts = %+v", p.mounts) + } + wantSockets := []guest.Socket{{Port: 1025, Path: "/run/aether/coord3.sock"}, {Port: 1026, Path: "/run/aether-signing/agent.sock"}} + if !slices.Equal(p.sockets, wantSockets) { + t.Errorf("sockets = %+v", p.sockets) + } + if err := linkSockets(dir, p.links); err != nil { + t.Fatal(err) + } + if target, err := os.Readlink(vsock + "_1026"); err != nil || target != filepath.Join(signing, "agent.sock") { + t.Errorf("link = %q, %v", target, err) + } + if err := linkSockets(dir, map[string]string{}); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(vsock + "_1026"); !os.IsNotExist(err) { + t.Errorf("stale link kept: %v", err) + } + if _, err := planMounts([]runtime.Mount{{HostPath: filepath.Join(dir, "gone"), ContainerPath: "/x"}}, false, vsock); err == nil { + t.Error("a missing mount source was planned") + } +} diff --git a/internal/runtime/vm/stats.go b/internal/runtime/vm/stats.go new file mode 100644 index 000000000..8834454f5 --- /dev/null +++ b/internal/runtime/vm/stats.go @@ -0,0 +1,114 @@ +package vm + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/3xDevOps/Aether/internal/disk" + "github.com/3xDevOps/Aether/internal/runtime" +) + +const ( + statsConcurrency = 4 + statsTimeout = 3 * time.Second +) + +// memoryEvents reports a stopped VM as fs.ErrNotExist, as the OOM watcher +// treats a stopped container's missing cgroup. +func (e *engine) memoryEvents(ctx context.Context, key string) (runtime.MemoryEvents, error) { + if _, err := e.load(key); err != nil { + return runtime.MemoryEvents{}, err + } + if up, err := e.running(ctx, key); err != nil || !up { + return runtime.MemoryEvents{}, errors.Join(err, fmt.Errorf("vm runtime: VM %s is not running: %w", key, fs.ErrNotExist)) + } + h, err := e.helper(key) + if err != nil { + return runtime.MemoryEvents{}, err + } + st, err := h.Stats(ctx) + if err != nil { + return runtime.MemoryEvents{}, fmt.Errorf("vm runtime: memory events of VM %s: %w", key, err) + } + return runtime.MemoryEvents{OOMKills: st.OOMKills, LimitBytes: st.MemoryLimitBytes}, nil +} + +// CPU, memory and processes come from the helper, network and disk bytes +// from Cloud Hypervisor's device counters. +func (e *engine) stats(ctx context.Context) ([]runtime.ContainerStat, error) { + ctx, cancel := context.WithTimeout(ctx, statsTimeout) + defer cancel() + keys, err := e.keys() + if err != nil { + return nil, err + } + results := make([]*runtime.ContainerStat, len(keys)) + errs := make([]error, len(keys)) + var g errgroup.Group + g.SetLimit(statsConcurrency) + for i, key := range keys { + g.Go(func() error { + results[i], errs[i] = e.sample(ctx, key) + return nil + }) + } + _ = g.Wait() + var out []runtime.ContainerStat + for _, st := range results { + if st != nil { + out = append(out, *st) + } + } + return out, errors.Join(errs...) +} + +func (e *engine) sample(ctx context.Context, key string) (*runtime.ContainerStat, error) { + rec, err := e.load(key) + if err != nil { + return nil, err + } + if up, upErr := e.running(ctx, key); upErr != nil || !up { + return nil, upErr + } + h, err := e.helper(key) + if err != nil { + return nil, err + } + g, err := h.Stats(ctx) + if err != nil { + return nil, fmt.Errorf("vm runtime: stats of VM %s: %w", key, err) + } + counters, err := newCHAPI(e.path(key, apiSocket)).counters(ctx) + if err != nil { + return nil, fmt.Errorf("vm runtime: device counters of VM %s: %w", key, err) + } + cores := rec.CPULimit + if cores == 0 { + cores = float64(rec.CPUs) + } + return &runtime.ContainerStat{ + ID: idPrefix + key, CreationKey: key, Read: time.Now(), + CPUUsageNanos: g.CPUUsageNanos, CPULimitCores: cores, + MemoryBytes: g.MemoryBytes, MemoryLimitBytes: g.MemoryLimitBytes, Pids: g.Pids, PidsLimit: g.PidsLimit, + NetRxBytes: counters["net0"]["rx_bytes"], NetTxBytes: counters["net0"]["tx_bytes"], + DiskReadBytes: counters["root"]["read_bytes"] + counters["data"]["read_bytes"], + DiskWriteBytes: counters["root"]["write_bytes"] + counters["data"]["write_bytes"], + }, nil +} + +func (e *engine) filesystem() (runtime.FilesystemCapacity, error) { + if err := os.MkdirAll(e.root, 0o700); err != nil { + return runtime.FilesystemCapacity{}, err + } + usage, err := disk.Filesystem(e.root) + if err != nil { + return runtime.FilesystemCapacity{}, err + } + return runtime.FilesystemCapacity{Name: "Session VM disks", TotalBytes: usage.TotalBytes, FreeBytes: usage.FreeBytes}, nil +} diff --git a/internal/runtime/vm/unit.go b/internal/runtime/vm/unit.go new file mode 100644 index 000000000..5226bdbc1 --- /dev/null +++ b/internal/runtime/vm/unit.go @@ -0,0 +1,81 @@ +package vm + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "strconv" + "strings" + + "github.com/3xDevOps/Aether/internal/runtime" +) + +type unitState struct { + Active bool + MainPID int +} + +// units runs each VM's monitor in its own transient systemd unit, outside +// the server's process tree, so a VM outlives a server restart or update. +type units interface { + Start(ctx context.Context, name string, argv, properties []string) error + Stop(ctx context.Context, name string) error + State(ctx context.Context, name string) (unitState, error) +} + +func unitName(key string) string { return "aether-vm-" + key + ".service" } + +type systemdUnits struct{} + +// Start uses --collect so a unit that failed does not keep its name from +// the next Start. +func (systemdUnits) Start(ctx context.Context, name string, argv, properties []string) error { + args := []string{"--unit=" + name, "--slice=" + runtime.SessionSlice, "--collect", "--quiet"} + for _, p := range properties { + args = append(args, "--property="+p) + } + args = append(append(args, "--"), argv...) + return systemctl(ctx, "systemd-run", args...) +} + +// Stop treats a unit systemd no longer knows as stopped. +func (systemdUnits) Stop(ctx context.Context, name string) error { + err := systemctl(ctx, "systemctl", "stop", name) + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 5 { + return nil + } + return err +} + +// State counts a unit as active while it starts, runs or reloads, so a VM +// whose monitor is still composing its share is not mistaken for gone. +func (systemdUnits) State(ctx context.Context, name string) (unitState, error) { + out, err := exec.CommandContext(ctx, "systemctl", "show", "--property=ActiveState,MainPID", name).Output() + if err != nil { + return unitState{}, fmt.Errorf("vm runtime: systemctl show %s: %w", name, err) + } + var st unitState + for line := range strings.SplitSeq(string(out), "\n") { + key, value, _ := strings.Cut(line, "=") + switch key { + case "ActiveState": + st.Active = value == "active" || value == "activating" || value == "reloading" + case "MainPID": + st.MainPID, _ = strconv.Atoi(value) + } + } + return st, nil +} + +func systemctl(ctx context.Context, name string, args ...string) error { + var stderr bytes.Buffer + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("vm runtime: %s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return nil +} diff --git a/internal/runtime/vm/vm.go b/internal/runtime/vm/vm.go new file mode 100644 index 000000000..c583f5ad7 --- /dev/null +++ b/internal/runtime/vm/vm.go @@ -0,0 +1,270 @@ +// Package vm runs each session in its own Cloud Hypervisor virtual machine +// behind runtime.Runtime. Every other container kind, and every image +// operation, still goes to Docker until it moves into a VM as well. +package vm + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/3xDevOps/Aether/internal/coordtransport" + "github.com/3xDevOps/Aether/internal/runtime" +) + +// idPrefix marks a VM's runtime.ID; Docker IDs are hex and never carry it. +const idPrefix = "vm-" + +// Runtime sends session containers to VMs and everything else to the +// embedded Docker runtime, whose optional methods it keeps. +type Runtime struct { + *runtime.Docker + vms *engine +} + +var ( + _ runtime.Runtime = (*Runtime)(nil) + _ runtime.ManagedExecRuntime = (*Runtime)(nil) + _ runtime.SudoRuntime = (*Runtime)(nil) + _ runtime.CapacityRuntime = (*Runtime)(nil) + _ runtime.InitPIDRuntime = (*Runtime)(nil) + _ runtime.MemoryEventsRuntime = (*Runtime)(nil) + _ runtime.BrowserRuntime = (*Runtime)(nil) + _ runtime.SaveRuntime = (*Runtime)(nil) +) + +// Config names what a VM is built from. Monitor is the command that runs +// Monitor inside each VM's systemd unit: the server binary and its +// vm-monitor subcommand. +type Config struct { + Docker *runtime.Docker + DataDir string + Images Images + Monitor []string + Helper string + CloudHypervisor string + Virtiofsd string +} + +// New returns a Runtime that keeps its VMs under DataDir/vm. +func New(cfg Config) (*Runtime, error) { + if cfg.Docker == nil || cfg.Images == nil || len(cfg.Monitor) == 0 || cfg.Helper == "" { + return nil, errors.New("vm runtime: Docker, the image store, the monitor command and the helper binary are required") + } + if cfg.CloudHypervisor == "" { + cfg.CloudHypervisor = "cloud-hypervisor" + } + if cfg.Virtiofsd == "" { + cfg.Virtiofsd = "/usr/libexec/virtiofsd" + } + return &Runtime{Docker: cfg.Docker, vms: newEngine(cfg, systemdUnits{}, dialHelper)}, nil +} + +// isSession is the one routing rule: a session container carries its run ID +// both as creation key and in AETHER_RUN_ID. Environment terminals, the +// assistant, the agent updater and the cleanup helper carry no run ID. +func isSession(spec runtime.Spec) bool { + return spec.CreationKey != "" && spec.Env[coordtransport.RunIDEnv] == spec.CreationKey +} + +func vmKey(id runtime.ID) (string, bool) { + return strings.CutPrefix(string(id), idPrefix) +} + +func (r *Runtime) Create(ctx context.Context, spec runtime.Spec) (runtime.ID, error) { + if !isSession(spec) { + return r.Docker.Create(ctx, spec) + } + return r.vms.create(ctx, spec) +} + +func (r *Runtime) Start(ctx context.Context, id runtime.ID) error { + if key, ok := vmKey(id); ok { + return r.vms.start(ctx, key) + } + return r.Docker.Start(ctx, id) +} + +func (r *Runtime) Pause(ctx context.Context, id runtime.ID) error { + if key, ok := vmKey(id); ok { + return r.vms.pause(ctx, key) + } + return r.Docker.Pause(ctx, id) +} + +func (r *Runtime) Resume(ctx context.Context, id runtime.ID) error { + if key, ok := vmKey(id); ok { + return r.vms.resume(ctx, key) + } + return r.Docker.Resume(ctx, id) +} + +func (r *Runtime) Stop(ctx context.Context, id runtime.ID, grace time.Duration) error { + if key, ok := vmKey(id); ok { + return r.vms.stop(ctx, key, grace) + } + return r.Docker.Stop(ctx, id, grace) +} + +func (r *Runtime) Destroy(ctx context.Context, id runtime.ID) error { + if key, ok := vmKey(id); ok { + return r.vms.destroy(ctx, key) + } + return r.Docker.Destroy(ctx, id) +} + +func (r *Runtime) Attach(ctx context.Context, id runtime.ID) (runtime.Attachment, error) { + if key, ok := vmKey(id); ok { + return r.vms.attach(ctx, key) + } + return r.Docker.Attach(ctx, id) +} + +func (r *Runtime) ContainerIP(ctx context.Context, id runtime.ID) (string, error) { + if key, ok := vmKey(id); ok { + return r.vms.containerIP(ctx, key) + } + return r.Docker.ContainerIP(ctx, id) +} + +func (r *Runtime) ExecTTY(ctx context.Context, id runtime.ID, argv []string, workDir string, cols, rows uint) (runtime.Attachment, error) { + if key, ok := vmKey(id); ok { + return r.vms.execTTY(ctx, key, argv, workDir, cols, rows) + } + return r.Docker.ExecTTY(ctx, id, argv, workDir, cols, rows) +} + +func (r *Runtime) Exec(ctx context.Context, id runtime.ID, argv []string, workDir string) (int, string, string, error) { + if key, ok := vmKey(id); ok { + return r.vms.exec(ctx, key, argv, workDir) + } + return r.Docker.Exec(ctx, id, argv, workDir) +} + +func (r *Runtime) Wait(ctx context.Context, id runtime.ID) (runtime.ExitStatus, error) { + if key, ok := vmKey(id); ok { + return r.vms.wait(ctx, key) + } + return r.Docker.Wait(ctx, id) +} + +func (r *Runtime) Inspect(ctx context.Context, id runtime.ID) (runtime.ContainerInfo, error) { + if key, ok := vmKey(id); ok { + return r.vms.inspect(ctx, key) + } + return r.Docker.Inspect(ctx, id) +} + +// FindByCreationKey looks among VMs first: a session's key names a VM only +// when it was created under this runtime. +func (r *Runtime) FindByCreationKey(ctx context.Context, key string) (runtime.ID, error) { + id, err := r.vms.find(key) + if !errors.Is(err, runtime.ErrNotFound) { + return id, err + } + return r.Docker.FindByCreationKey(ctx, key) +} + +func (r *Runtime) Commit(ctx context.Context, id runtime.ID, tag string, labels map[string]string) error { + if _, ok := vmKey(id); ok { + return errors.New("vm runtime: Commit (Save environment) of a session VM is not supported until work item 9") + } + return r.Docker.Commit(ctx, id, tag, labels) +} + +func (r *Runtime) InitPID(ctx context.Context, id runtime.ID) (int, error) { + if key, ok := vmKey(id); ok { + return r.vms.initPID(ctx, key) + } + return r.Docker.InitPID(ctx, id) +} + +func (r *Runtime) MemoryEvents(ctx context.Context, id runtime.ID) (runtime.MemoryEvents, bool, error) { + key, ok := vmKey(id) + if !ok { + return runtime.MemoryEvents{}, false, nil + } + events, err := r.vms.memoryEvents(ctx, key) + return events, true, err +} + +func (r *Runtime) StartExecTTY(ctx context.Context, id runtime.ID, spec runtime.ExecSpec) (runtime.ManagedExec, error) { + if key, ok := vmKey(id); ok { + return r.vms.startManagedExec(ctx, key, spec, true) + } + return r.Docker.StartExecTTY(ctx, id, spec) +} + +func (r *Runtime) StartExecPipe(ctx context.Context, id runtime.ID, spec runtime.ExecSpec) (runtime.ManagedExec, error) { + if key, ok := vmKey(id); ok { + return r.vms.startManagedExec(ctx, key, spec, false) + } + return r.Docker.StartExecPipe(ctx, id, spec) +} + +func (r *Runtime) RecoverExec(ctx context.Context, identity runtime.ExecIdentity) (runtime.ManagedExec, error) { + if key, ok := vmKey(identity.ContainerID); ok { + return r.vms.recoverExec(ctx, key, identity) + } + return r.Docker.RecoverExec(ctx, identity) +} + +func (r *Runtime) SettingUp(id runtime.ID) bool { + if key, ok := vmKey(id); ok { + return r.vms.settingUp(key) + } + return r.Docker.SettingUp(id) +} + +func (r *Runtime) ExecAttached(ctx context.Context, id runtime.ID, spec runtime.AttachedExec) (int, error) { + if key, ok := vmKey(id); ok { + return r.vms.execAttached(ctx, key, spec) + } + return r.Docker.ExecAttached(ctx, id, spec) +} + +func (r *Runtime) CreateBrowser(ctx context.Context, spec runtime.BrowserSpec) (runtime.ID, error) { + if _, ok := vmKey(spec.RunContainer); ok { + return "", errors.New("vm runtime: the development browser cannot run beside a session VM until work item 9 moves it into the VM") + } + return r.Docker.CreateBrowser(ctx, spec) +} + +func (r *Runtime) Capacity(ctx context.Context) (runtime.HostCapacity, error) { + capacity, err := r.Docker.Capacity(ctx) + if err != nil { + return capacity, err + } + fs, err := r.vms.filesystem() + if err != nil { + return runtime.HostCapacity{}, fmt.Errorf("vm runtime: VM disk capacity: %w", err) + } + capacity.Filesystems = append(capacity.Filesystems, fs) + return capacity, nil +} + +// ContainerStats adds every running VM to Docker's sample. A VM that cannot +// be sampled is left out and named in the error, like a failed container. +func (r *Runtime) ContainerStats(ctx context.Context) ([]runtime.ContainerStat, error) { + stats, dockerErr := r.Docker.ContainerStats(ctx) + vms, vmErr := r.vms.stats(ctx) + return append(stats, vms...), errors.Join(dockerErr, vmErr) +} + +func (r *Runtime) Save(ctx context.Context, id runtime.ID) error { + key, ok := vmKey(id) + if !ok { + return errors.New("vm runtime: only a session VM can be saved; Docker containers cannot") + } + return r.vms.save(ctx, key) +} + +func (r *Runtime) Restore(ctx context.Context, id runtime.ID) error { + key, ok := vmKey(id) + if !ok { + return errors.New("vm runtime: only a session VM can be restored; Docker containers cannot") + } + return r.vms.restore(ctx, key) +} diff --git a/internal/runtime/vm/vm_test.go b/internal/runtime/vm/vm_test.go new file mode 100644 index 000000000..29a84a58f --- /dev/null +++ b/internal/runtime/vm/vm_test.go @@ -0,0 +1,78 @@ +package vm + +import ( + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/3xDevOps/Aether/internal/runtime" +) + +func TestIsSessionNeedsTheRunIDTwice(t *testing.T) { + for _, tc := range []struct { + spec runtime.Spec + want bool + }{ + {runtime.Spec{CreationKey: "run-a", Env: map[string]string{"AETHER_RUN_ID": "run-a"}}, true}, + {runtime.Spec{CreationKey: "terminal:m1"}, false}, + {runtime.Spec{CreationKey: "aether-agent-update-x", Env: map[string]string{"AETHER_RUN_ID": "run-a"}}, false}, + {runtime.Spec{Env: map[string]string{"AETHER_RUN_ID": ""}}, false}, + } { + if got := isSession(tc.spec); got != tc.want { + t.Errorf("isSession(%+v) = %v", tc.spec, got) + } + } +} + +// dockerOnly takes IDs that only Docker ever hands out. +var dockerOnly = []string{"DictationState", "InspectBrowser"} + +// TestEveryContainerMethodIsRouted fails when Docker gains a method that +// takes a container, unless Runtime routes it or it is listed above: a +// promoted method would hand a VM's ID to Docker. +func TestEveryContainerMethodIsRouted(t *testing.T) { + declared := map[string]bool{} + names, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, name := range names { + if strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(token.NewFileSet(), name, nil, 0) + if err != nil { + t.Fatal(err) + } + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil { + continue + } + if star, ok := fn.Recv.List[0].Type.(*ast.StarExpr); ok && star.X.(*ast.Ident).Name == "Runtime" { + declared[fn.Name.Name] = true + } + } + } + container := map[reflect.Type]bool{ + reflect.TypeFor[runtime.ID](): true, reflect.TypeFor[runtime.ExecIdentity](): true, reflect.TypeFor[runtime.BrowserSpec](): true, + reflect.TypeFor[runtime.Spec](): true, + } + docker := reflect.TypeFor[*runtime.Docker]() + for i := range docker.NumMethod() { + m := docker.Method(i) + for j := 1; j < m.Type.NumIn(); j++ { + if container[m.Type.In(j)] && !declared[m.Name] && !slices.Contains(dockerOnly, m.Name) { + t.Errorf("Runtime does not route %s, which takes a %s", m.Name, m.Type.In(j)) + } + } + } + if !declared["FindByCreationKey"] || !declared["ContainerStats"] || !declared["Capacity"] { + t.Error("Runtime must also route FindByCreationKey, ContainerStats and Capacity") + } +} diff --git a/internal/scheduler/oomwatch.go b/internal/scheduler/oomwatch.go index d022518d3..e9fe8ea8f 100644 --- a/internal/scheduler/oomwatch.go +++ b/internal/scheduler/oomwatch.go @@ -102,6 +102,12 @@ func (w *oomWatcher) forget(id runtime.ID) { } func (w *oomWatcher) read(ctx context.Context, pids runtime.InitPIDRuntime, id runtime.ID) (memoryStats, error) { + if guest, ok := pids.(runtime.MemoryEventsRuntime); ok { + events, own, err := guest.MemoryEvents(ctx, id) + if own { + return memoryStats{oomKills: events.OOMKills, max: events.LimitBytes}, err + } + } path, ok := w.paths[id] if !ok { pid, err := pids.InitPID(ctx, id) diff --git a/internal/scheduler/oomwatch_test.go b/internal/scheduler/oomwatch_test.go index 3a7ae8f2a..e6091ace2 100644 --- a/internal/scheduler/oomwatch_test.go +++ b/internal/scheduler/oomwatch_test.go @@ -1,9 +1,12 @@ package scheduler import ( + "context" "os" "path/filepath" "testing" + + "github.com/3xDevOps/Aether/internal/runtime" ) func TestReadMemoryStatsAndMessage(t *testing.T) { @@ -54,3 +57,43 @@ func TestReadMemoryStatsAndMessage(t *testing.T) { t.Fatalf("missing memory.events error = %v, want not-exist", err) } } + +type guestMemory struct { + pid int + own bool + events runtime.MemoryEvents +} + +func (g guestMemory) InitPID(context.Context, runtime.ID) (int, error) { return g.pid, nil } + +func (g guestMemory) MemoryEvents(context.Context, runtime.ID) (runtime.MemoryEvents, bool, error) { + return g.events, g.own, nil +} + +func TestOOMWatcherReadsGuestKillsFromTheRuntime(t *testing.T) { + proc, cg := t.TempDir(), t.TempDir() + if err := os.MkdirAll(filepath.Join(proc, "7"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, "7", "cgroup"), []byte("0::/aether.slice/run.scope\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(cg, "aether.slice", "run.scope"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cg, "aether.slice", "run.scope", "memory.events"), []byte("oom_kill 2\n"), 0o644); err != nil { + t.Fatal(err) + } + w := &oomWatcher{checked: true, enabled: true, cgroup: cg, proc: proc, paths: map[runtime.ID]string{}, kills: map[runtime.ID]uint64{}} + + guest := guestMemory{pid: 7, own: true, events: runtime.MemoryEvents{OOMKills: 5, LimitBytes: 4 << 30}} + st, err := w.read(t.Context(), guest, "vm-run-a") + if err != nil || st.oomKills != 5 || st.max != 4<<30 { + t.Fatalf("guest read = %+v, %v", st, err) + } + host := guestMemory{pid: 7} + st, err = w.read(t.Context(), host, "container-b") + if err != nil || st.oomKills != 2 { + t.Fatalf("cgroup fallback = %+v, %v", st, err) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index eb17dc649..d7231733b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -4,6 +4,7 @@ package server import ( + "cmp" "context" "errors" "fmt" @@ -185,6 +186,14 @@ type Config struct { // it registered, empty means the Docker daemon default. ContainerRuntime string + // SessionRuntime is where sessions run: SessionRuntimeDocker (empty + // means the same) or SessionRuntimeVM, one Cloud Hypervisor VM each. + // Every other container stays on Docker either way. + SessionRuntime string + // VMKernel is the guest kernel session VMs boot; empty means + // DefaultVMKernel. + VMKernel string + // SysboxInstaller lets admins install Sysbox on the host through the // control channel. Only cmd/aether-server sets it; an embedded server // never runs host installers. @@ -193,6 +202,11 @@ type Config struct { const DefaultContainerRuntime = runtime.RuntimeAuto +const ( + SessionRuntimeDocker = "docker" + SessionRuntimeVM = "vm" +) + // Server is the assembled Aether server. type Server struct { db *store.DB @@ -311,6 +325,15 @@ func New(ctx context.Context, cfg Config) (srv *Server, err error) { return nil, err } s.rt = s.docker + switch cfg.SessionRuntime { + case "", SessionRuntimeDocker: + case SessionRuntimeVM: + if s.rt, err = newVMRuntime(cfg.DataDir, cmp.Or(cfg.VMKernel, DefaultVMKernel), s.docker); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("server: session-runtime %q: use %s or %s", cfg.SessionRuntime, SessionRuntimeDocker, SessionRuntimeVM) + } } removeAgentFiles := runtime.Remover(s.rt, cfg.StandardImage) diff --git a/internal/server/vmruntime.go b/internal/server/vmruntime.go new file mode 100644 index 000000000..73a7b1e3e --- /dev/null +++ b/internal/server/vmruntime.go @@ -0,0 +1,31 @@ +package server + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/runtime/vm" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +// DefaultVMKernel is where the guest kernel a release ships is installed. +const DefaultVMKernel = "/usr/local/lib/aether/vmlinux" + +// newVMRuntime runs sessions in VMs whose monitor is this binary's +// vm-monitor subcommand and whose in-VM helper is this binary too. +func newVMRuntime(dataDir, kernel string, docker *runtime.Docker) (*vm.Runtime, error) { + exe, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("server: find the server binary for session VMs: %w", err) + } + images, err := vmimage.Open(filepath.Join(dataDir, "vm", "images"), kernel, docker) + if err != nil { + return nil, fmt.Errorf("server: open the VM image store: %w", err) + } + return vm.New(vm.Config{ + Docker: docker, DataDir: dataDir, Images: images, + Monitor: []string{exe, vm.MonitorCommand}, Helper: exe, + }) +} From 7bcb8d145b751c5d279029d10ffdbb9ea743db2c Mon Sep 17 00:00:00 2001 From: ilovecrayons Date: Fri, 13 Nov 2026 03:02:11 +0000 Subject: [PATCH 2/4] fix(runtime): refuse exec into a paused session VM Docker refuses an exec into a paused container at once; a paused guest would instead hold the exec until its context ends, so closed sessions, which are paused, could stall a caller. --- internal/runtime/vm/engine_test.go | 3 +++ internal/runtime/vm/exec.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/internal/runtime/vm/engine_test.go b/internal/runtime/vm/engine_test.go index 6e1d2f890..cb3bef3e6 100644 --- a/internal/runtime/vm/engine_test.go +++ b/internal/runtime/vm/engine_test.go @@ -187,6 +187,9 @@ func TestStopEndsMainRecordsItsExitAndPowersOff(t *testing.T) { if err := h.engine.pause(t.Context(), "run-a"); err != nil { t.Fatal(err) } + if _, _, _, err := h.engine.exec(t.Context(), "run-a", []string{"true"}, ""); err == nil || !strings.Contains(err.Error(), "paused") { + t.Errorf("exec in a paused VM = %v", err) + } vmm := h.units.vmm("run-a") if err := h.engine.stop(t.Context(), "run-a", time.Second); err != nil { t.Fatal(err) diff --git a/internal/runtime/vm/exec.go b/internal/runtime/vm/exec.go index 98bbe8650..1ff14c5e6 100644 --- a/internal/runtime/vm/exec.go +++ b/internal/runtime/vm/exec.go @@ -73,6 +73,10 @@ func (e *engine) spawn(ctx context.Context, key string, spec guest.ProcessSpec) if err != nil { return nil, nil, err } + // A paused guest would hold the exec until ctx ends; Docker refuses it. + if state, stateErr := newCHAPI(e.path(key, apiSocket)).state(ctx); stateErr == nil && state == "Paused" { + return nil, nil, fmt.Errorf("vm runtime: exec in VM %s: it is paused", key) + } h, err := e.helper(key) if err != nil { return nil, nil, err From 774fe50aeb94c3b9c31d73856ca3c7a8d8fe0c9e Mon Sep 17 00:00:00 2001 From: ilovecrayons Date: Fri, 13 Nov 2026 03:03:16 +0000 Subject: [PATCH 3/4] fix(runtime): drop a VM's snapshot as soon as the restored guest runs A restore that failed after resuming the guest kept the snapshot marked saved. The guest's disks had already moved past it, so restoring it again would pair old memory with newer disks. --- internal/runtime/vm/engine_test.go | 17 +++++++++++++++++ internal/runtime/vm/save.go | 22 ++++++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/internal/runtime/vm/engine_test.go b/internal/runtime/vm/engine_test.go index cb3bef3e6..e6929f76d 100644 --- a/internal/runtime/vm/engine_test.go +++ b/internal/runtime/vm/engine_test.go @@ -413,6 +413,23 @@ func TestSaveRefusesSharesAndRestoresAVMWithout(t *testing.T) { if err := h.engine.restore(t.Context(), "run-a"); err == nil || !strings.Contains(err.Error(), "no saved state") { t.Errorf("restore without state = %v", err) } + h.createStarted("run-b", sessionSpec("run-b")) + if err := h.engine.save(t.Context(), "run-b"); err != nil { + t.Fatal(err) + } + h.guest("run-b").pingErr = errors.New("helper gone") + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if err := h.engine.restore(ctx, "run-b"); err == nil { + t.Fatal("restore with a dead helper succeeded") + } + rec, _ := h.engine.load("run-b") + if rec.Saved || h.units.vmm("run-b") != nil { + t.Errorf("a restore that failed after resume left saved=%v, running=%v", rec.Saved, h.units.vmm("run-b") != nil) + } + if _, err := os.Stat(h.engine.path("run-b", snapshotDir)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a resumed snapshot was kept: %v", err) + } } func TestStatsAndMemoryEventsComeFromTheHelper(t *testing.T) { diff --git a/internal/runtime/vm/save.go b/internal/runtime/vm/save.go index 554614857..69ce4e1f1 100644 --- a/internal/runtime/vm/save.go +++ b/internal/runtime/vm/save.go @@ -83,7 +83,9 @@ func (e *engine) awaitStopped(ctx context.Context, key string) error { } // restore starts the monitor on the saved state, resumes the guest and sets -// its clock; the snapshot is deleted once the guest answers. +// its clock. Once the guest runs, its disks move past the snapshot, so the +// snapshot is dropped before anything else can fail: a second restore of it +// would pair old memory with newer disks. func (e *engine) restore(ctx context.Context, key string) (err error) { unlock := e.lock(key) defer unlock() @@ -119,23 +121,23 @@ func (e *engine) restore(ctx context.Context, key string) (err error) { if err = api.resume(ctx); err != nil { return fmt.Errorf("vm runtime: resume restored VM %s: %w", key, err) } + rec.Saved = false + if err = e.store(rec); err != nil { + return err + } + if err = os.RemoveAll(dir); err != nil { + return fmt.Errorf("vm runtime: remove the saved state of VM %s: %w", key, err) + } h, err := e.helper(key) if err != nil { return err } - if err := e.waitHelper(ctx, key, h); err != nil { + if err = e.waitHelper(ctx, key, h); err != nil { return err } - if err := h.SyncClock(ctx, time.Now()); err != nil { + if err = h.SyncClock(ctx, time.Now()); err != nil { return fmt.Errorf("vm runtime: set the clock of restored VM %s: %w", key, err) } - rec.Saved = false - if err := e.store(rec); err != nil { - return err - } - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("vm runtime: remove the saved state of VM %s: %w", key, err) - } return nil } From a633cf474bd03f297fe9bdda539804b6d597c071 Mon Sep 17 00:00:00 2001 From: ilovecrayons Date: Fri, 13 Nov 2026 03:43:51 +0000 Subject: [PATCH 4/4] fix(runtime): close the gaps Greptile found in session VM exec and waits - A pipe execution's input now closes the command's stdin, and its stdout is lossless even when the caller did not ask: stderr then drains into the Docker runtime's lossy buffer, now shared. - Exec output reads end with their context, and a failure on either stream closes both, so a quiet stdout cannot hide a stderr flood. - A wait on Main ends when the VM's unit does, and a wait on an earlier boot returns that boot's exit instead of powering off the next one. - Restore drops the snapshot before the guest resumes, so a failure after resume cannot leave old memory to pair with newer disks. - Managed exec recovery reports a systemd failure as unavailable, not gone, and reads dev-exec's own exit record once the helper forgot the supervisor. - Paused VMs are not asked for stats; their OOM count is the last one. --- internal/runtime/docker.go | 89 +---------------- internal/runtime/docker_test.go | 12 +-- internal/runtime/exectty.go | 8 +- internal/runtime/exectty_test.go | 2 +- internal/runtime/stream.go | 88 +++++++++++++++++ internal/runtime/vm/engine.go | 6 +- internal/runtime/vm/exec.go | 84 ++++++++-------- internal/runtime/vm/exec_test.go | 159 ++++++++++++++++++++++++++++++ internal/runtime/vm/fakes_test.go | 39 +++++++- internal/runtime/vm/lifecycle.go | 53 +++++++--- internal/runtime/vm/managed.go | 103 ++++++++++++++++--- internal/runtime/vm/record.go | 7 +- internal/runtime/vm/save.go | 11 ++- internal/runtime/vm/stats.go | 25 ++++- 14 files changed, 504 insertions(+), 182 deletions(-) create mode 100644 internal/runtime/stream.go create mode 100644 internal/runtime/vm/exec_test.go diff --git a/internal/runtime/docker.go b/internal/runtime/docker.go index 038be5300..e060b910b 100644 --- a/internal/runtime/docker.go +++ b/internal/runtime/docker.go @@ -1025,8 +1025,8 @@ type dockerAttachment struct { id string tty bool resp client.HijackedResponse - stdout *streamBuffer - stderr *streamBuffer + stdout *StreamBuffer + stderr *StreamBuffer closeOnce sync.Once } @@ -1036,8 +1036,8 @@ func newDockerAttachment(cli *client.Client, id string, tty bool, resp client.Hi id: id, tty: tty, resp: resp, - stdout: newStreamBuffer(), - stderr: newStreamBuffer(), + stdout: NewStreamBuffer(), + stderr: NewStreamBuffer(), } go func() { var err error @@ -1111,84 +1111,3 @@ func (h hijackStdin) WriteContext(ctx context.Context, p []byte) (int, error) { } func (h hijackStdin) Close() error { return h.resp.CloseWrite() } - -const maxStreamBuffer = 8 << 20 - -// streamBuffer writes never block, since blocking the demux goroutine would -// stall the sibling stream; past maxStreamBuffer the oldest bytes are -// dropped. A lossless buffer blocks at the cap instead, for protocol streams -// that cannot lose a byte. -type streamBuffer struct { - mu sync.Mutex - cond *sync.Cond - buf bytes.Buffer - lossless bool - // limit is where a lossless buffer blocks its writer. - limit int - closed bool - err error -} - -func newStreamBuffer() *streamBuffer { - b := &streamBuffer{limit: maxStreamBuffer} - b.cond = sync.NewCond(&b.mu) - return b -} - -func newLosslessStreamBuffer() *streamBuffer { - b := newStreamBuffer() - b.lossless = true - return b -} - -func (b *streamBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - for b.lossless && !b.closed && b.buf.Len() >= b.limit { - b.cond.Wait() - } - if b.closed { - return 0, io.ErrClosedPipe - } - n, _ := b.buf.Write(p) // bytes.Buffer.Write cannot fail - if b.lossless { - b.cond.Broadcast() - return n, nil - } - if over := b.buf.Len() - maxStreamBuffer; over > 0 { - b.buf.Next(over) - } - b.cond.Broadcast() - return n, nil -} - -func (b *streamBuffer) Read(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - for b.buf.Len() == 0 && !b.closed { - b.cond.Wait() - } - if b.buf.Len() > 0 { - if b.lossless { - b.cond.Broadcast() - } - return b.buf.Read(p) - } - if b.err != nil { - return 0, b.err - } - return 0, io.EOF -} - -// CloseWithError leaves buffered data readable first. Only the first close -// takes effect. -func (b *streamBuffer) CloseWithError(err error) { - b.mu.Lock() - defer b.mu.Unlock() - if b.closed { - return - } - b.closed = true - b.err = err - b.cond.Broadcast() -} diff --git a/internal/runtime/docker_test.go b/internal/runtime/docker_test.go index 29afb20ce..b9944574e 100644 --- a/internal/runtime/docker_test.go +++ b/internal/runtime/docker_test.go @@ -433,7 +433,7 @@ func TestNanoCPUs(t *testing.T) { } func TestStreamBufferWriteNeverBlocks(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() done := make(chan struct{}) go func() { defer close(done) @@ -452,7 +452,7 @@ func TestStreamBufferWriteNeverBlocks(t *testing.T) { } func TestStreamBufferDropsOldestPastCap(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() chunk := bytes.Repeat([]byte("a"), 4096) for range 4 * maxStreamBuffer / len(chunk) { if _, err := b.Write(chunk); err != nil { @@ -477,7 +477,7 @@ func TestStreamBufferDropsOldestPastCap(t *testing.T) { } func TestStreamBufferReadBlocksUntilData(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() got := make(chan string, 1) go func() { p := make([]byte, 16) @@ -502,7 +502,7 @@ func TestStreamBufferReadBlocksUntilData(t *testing.T) { } func TestStreamBufferCloseDrainsThenEOF(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() if _, err := b.Write([]byte("tail")); err != nil { t.Fatalf("Write() error: %v", err) } @@ -520,7 +520,7 @@ func TestStreamBufferCloseDrainsThenEOF(t *testing.T) { } func TestStreamBufferCloseWithError(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() want := errors.New("boom") b.CloseWithError(want) b.CloseWithError(nil) // only the first close takes effect @@ -530,7 +530,7 @@ func TestStreamBufferCloseWithError(t *testing.T) { } func TestStreamBufferConcurrent(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() const writers, lines = 8, 200 var wg sync.WaitGroup for range writers { diff --git a/internal/runtime/exectty.go b/internal/runtime/exectty.go index 702bc8c1c..fe6c40758 100644 --- a/internal/runtime/exectty.go +++ b/internal/runtime/exectty.go @@ -21,8 +21,8 @@ type execAttachment struct { tty bool resp client.HijackedResponse - stdout *streamBuffer - stderr *streamBuffer + stdout *StreamBuffer + stderr *StreamBuffer done chan struct{} closeOnce sync.Once } @@ -38,8 +38,8 @@ func newExecAttachment(cli *client.Client, id string, tty, lossless bool, resp c id: id, tty: tty, resp: resp, - stdout: newStreamBuffer(), - stderr: newStreamBuffer(), + stdout: NewStreamBuffer(), + stderr: NewStreamBuffer(), done: make(chan struct{}), } if !tty || lossless { diff --git a/internal/runtime/exectty_test.go b/internal/runtime/exectty_test.go index eeaa9ccb5..ead658c4c 100644 --- a/internal/runtime/exectty_test.go +++ b/internal/runtime/exectty_test.go @@ -195,7 +195,7 @@ func TestLosslessExecKeepsStderrAndTerminalOutput(t *testing.T) { time.Sleep(200 * time.Millisecond) // Unread, the stream holds its small cap and one more // write, not the megabytes a lossy stream retains. - held := tc.stream(attachment).(*streamBuffer) + held := tc.stream(attachment).(*StreamBuffer) held.mu.Lock() buffered := held.buf.Len() held.mu.Unlock() diff --git a/internal/runtime/stream.go b/internal/runtime/stream.go new file mode 100644 index 000000000..056b8f6bd --- /dev/null +++ b/internal/runtime/stream.go @@ -0,0 +1,88 @@ +package runtime + +import ( + "bytes" + "io" + "sync" +) + +const maxStreamBuffer = 8 << 20 + +// StreamBuffer writes never block, since blocking the demux goroutine would +// stall the sibling stream; past maxStreamBuffer the oldest bytes are +// dropped. A lossless buffer blocks at the cap instead, for protocol streams +// that cannot lose a byte. +type StreamBuffer struct { + mu sync.Mutex + cond *sync.Cond + buf bytes.Buffer + lossless bool + // limit is where a lossless buffer blocks its writer. + limit int + closed bool + err error +} + +func NewStreamBuffer() *StreamBuffer { + b := &StreamBuffer{limit: maxStreamBuffer} + b.cond = sync.NewCond(&b.mu) + return b +} + +func newLosslessStreamBuffer() *StreamBuffer { + b := NewStreamBuffer() + b.lossless = true + return b +} + +func (b *StreamBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + for b.lossless && !b.closed && b.buf.Len() >= b.limit { + b.cond.Wait() + } + if b.closed { + return 0, io.ErrClosedPipe + } + n, _ := b.buf.Write(p) // bytes.Buffer.Write cannot fail + if b.lossless { + b.cond.Broadcast() + return n, nil + } + if over := b.buf.Len() - maxStreamBuffer; over > 0 { + b.buf.Next(over) + } + b.cond.Broadcast() + return n, nil +} + +func (b *StreamBuffer) Read(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + for b.buf.Len() == 0 && !b.closed { + b.cond.Wait() + } + if b.buf.Len() > 0 { + if b.lossless { + b.cond.Broadcast() + } + return b.buf.Read(p) + } + if b.err != nil { + return 0, b.err + } + return 0, io.EOF +} + +// CloseWithError leaves buffered data readable first. Only the first close +// takes effect. +func (b *StreamBuffer) CloseWithError(err error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } + b.closed = true + b.err = err + b.cond.Broadcast() +} diff --git a/internal/runtime/vm/engine.go b/internal/runtime/vm/engine.go index e9e8c2679..94d8eed61 100644 --- a/internal/runtime/vm/engine.go +++ b/internal/runtime/vm/engine.go @@ -52,6 +52,7 @@ type engine struct { helpers map[string]helper booting map[string]bool fresh map[string]bool + events map[string]runtime.MemoryEvents netMu sync.Mutex reserved map[int]bool @@ -62,7 +63,7 @@ func newEngine(cfg Config, u units, dial func(string) (helper, error)) *engine { return &engine{ cfg: cfg, root: filepath.Join(cfg.DataDir, "vm"), units: u, dial: dial, run: runCommand, locks: map[string]*sync.Mutex{}, helpers: map[string]helper{}, booting: map[string]bool{}, - fresh: map[string]bool{}, reserved: map[int]bool{}, + fresh: map[string]bool{}, events: map[string]runtime.MemoryEvents{}, reserved: map[int]bool{}, } } @@ -269,7 +270,8 @@ func (e *engine) start(ctx context.Context, key string) (err error) { if err = e.ensureNAT(ctx); err != nil { return err } - rec.Booted, rec.Exited, rec.ExitCode = true, false, 0 + rec.Booted, rec.Exited = true, false + rec.Boots++ if err = e.store(rec); err != nil { return err } diff --git a/internal/runtime/vm/exec.go b/internal/runtime/vm/exec.go index 1ff14c5e6..63895a555 100644 --- a/internal/runtime/vm/exec.go +++ b/internal/runtime/vm/exec.go @@ -74,7 +74,7 @@ func (e *engine) spawn(ctx context.Context, key string, spec guest.ProcessSpec) return nil, nil, err } // A paused guest would hold the exec until ctx ends; Docker refuses it. - if state, stateErr := newCHAPI(e.path(key, apiSocket)).state(ctx); stateErr == nil && state == "Paused" { + if e.paused(ctx, key) { return nil, nil, fmt.Errorf("vm runtime: exec in VM %s: it is paused", key) } h, err := e.helper(key) @@ -108,7 +108,7 @@ func (e *engine) exec(ctx context.Context, key string, argv []string, workDir st if err = p.CloseStdin(ctx); err != nil { return 0, "", "", fmt.Errorf("vm runtime: exec input: %w", err) } - stdout, stderr, err := readLimited(att) + stdout, stderr, err := readLimited(ctx, att) if err != nil { return 0, stdout, stderr, fmt.Errorf("vm runtime: exec output: %w", err) } @@ -121,40 +121,53 @@ func (e *engine) exec(ctx context.Context, key string, argv []string, workDir st // readLimited reads both streams to their end and refuses output past // runtime.ExecOutputLimit, which the two share. -func readLimited(att runtime.Attachment) (string, string, error) { +func readLimited(ctx context.Context, att runtime.Attachment) (string, string, error) { var mu sync.Mutex total := 0 var out, errOut bytes.Buffer - read := func(r io.Reader, into *bytes.Buffer) error { - buf := make([]byte, 32<<10) - for { - n, err := r.Read(buf) + limited := func(into *bytes.Buffer) io.Writer { + return writerFunc(func(p []byte) (int, error) { mu.Lock() - total += n - over := total > runtime.ExecOutputLimit - if !over { - into.Write(buf[:n]) - } - mu.Unlock() - if over { - return fmt.Errorf("output passed the %d byte limit", runtime.ExecOutputLimit) - } - if errors.Is(err, io.EOF) { - return nil + defer mu.Unlock() + if total += len(p); total > runtime.ExecOutputLimit { + return 0, fmt.Errorf("output passed the %d byte limit", runtime.ExecOutputLimit) } + return into.Write(p) + }) + } + err := copyStreams(ctx, att, limited(&out), limited(&errOut), func() { _ = att.Close() }) + return out.String(), errOut.String(), err +} + +type writerFunc func([]byte) (int, error) + +func (f writerFunc) Write(p []byte) (int, error) { return f(p) } + +// copyStreams copies stdout and stderr at once; the first failure, or the +// end of ctx, calls abort so the other copy is not left blocked on a stream +// the command keeps open. +func copyStreams(ctx context.Context, att runtime.Attachment, stdout, stderr io.Writer, abort func()) error { + abort = sync.OnceFunc(abort) + stop := context.AfterFunc(ctx, abort) + defer stop() + errs := make(chan error, 2) + for _, pair := range [...]struct { + w io.Writer + r io.Reader + }{{stdout, att.Stdout()}, {stderr, att.Stderr()}} { + go func() { + _, err := io.Copy(pair.w, pair.r) if err != nil { - return err + abort() } - } + errs <- err + }() } - errc := make(chan error, 1) - go func() { errc <- read(att.Stderr(), &errOut) }() - outErr := read(att.Stdout(), &out) - if outErr != nil { - _ = att.Close() + err := errors.Join(<-errs, <-errs) + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr } - errErr := <-errc - return out.String(), errOut.String(), errors.Join(outErr, errErr) + return err } func (e *engine) execTTY(ctx context.Context, key string, argv []string, workDir string, cols, rows uint) (runtime.Attachment, error) { @@ -195,20 +208,7 @@ func (e *engine) execAttached(ctx context.Context, key string, spec runtime.Atta return 0, fmt.Errorf("vm runtime: exec input: %w", err) } } - copied := make(chan error, 1) - go func() { - _, copyErr := io.Copy(spec.Stderr, att.Stderr()) - copied <- copyErr - }() - _, outErr := io.Copy(spec.Stdout, att.Stdout()) - if outErr != nil { - closeStdin() - } - errErr := <-copied - if err = ctx.Err(); err != nil { - return 0, err - } - if err = errors.Join(outErr, errErr); err != nil { + if err = copyStreams(ctx, att, spec.Stdout, spec.Stderr, closeStdin); err != nil { return 0, fmt.Errorf("vm runtime: exec output: %w", err) } code, err := p.Wait(ctx) @@ -228,7 +228,7 @@ func (e *engine) runAsRoot(ctx context.Context, h helper, argv []string) (int, s if err = p.CloseStdin(ctx); err != nil { return 0, "", err } - stdout, stderr, err := readLimited(att) + stdout, stderr, err := readLimited(ctx, att) if err != nil { return 0, "", err } diff --git a/internal/runtime/vm/exec_test.go b/internal/runtime/vm/exec_test.go new file mode 100644 index 000000000..b17117460 --- /dev/null +++ b/internal/runtime/vm/exec_test.go @@ -0,0 +1,159 @@ +package vm + +import ( + "context" + "errors" + "io/fs" + "strings" + "testing" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +func TestPipeExecutionClosesInputAndKeepsStdoutLossless(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + m, err := h.engine.startManagedExec(t.Context(), "run-a", runtime.ExecSpec{Argv: []string{"cat"}, CreationKey: "ssh-1"}, false) + if err != nil { + t.Fatal(err) + } + supervisor := h.guest("run-a").procs[m.Identity().ExecID] + if !supervisor.spec.Lossless { + t.Error("a pipe execution without Lossless may drop stdout in the guest") + } + stdin := m.Attachment().Stdin() + if _, ok := stdin.(runtime.ContextWriter); !ok { + t.Error("a pipe execution's input lost WriteContext") + } + if err := stdin.Close(); err != nil { + t.Fatal(err) + } + supervisor.mu.Lock() + closed := supervisor.stdin + supervisor.mu.Unlock() + if !closed { + t.Error("closing a pipe execution's input did not close the command's stdin") + } +} + +func TestExecStopsWhenItsContextEndsOrStderrFloods(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + h.guest("run-a").output = map[string]string{"hang": "hang", "flood": "errflood:"} + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + start := time.Now() + if _, _, _, err := h.engine.exec(ctx, "run-a", []string{"hang"}, ""); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("exec of a hung command = %v", err) + } + if time.Since(start) > 5*time.Second { + t.Errorf("exec outlived its context by %s", time.Since(start)) + } + done := make(chan error, 1) + go func() { + _, _, _, err := h.engine.exec(t.Context(), "run-a", []string{"flood"}, "") + done <- err + }() + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), "byte limit") { + t.Errorf("exec flooding stderr = %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("a stderr flood left exec waiting on a quiet stdout") + } +} + +func TestWaitEndsWhenTheVMDiesMidWait(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + done := make(chan runtime.ExitStatus, 1) + go func() { + st, _ := h.engine.wait(context.WithoutCancel(t.Context()), "run-a") + done <- st + }() + time.Sleep(100 * time.Millisecond) + _ = h.units.Stop(t.Context(), unitName("run-a")) + select { + case st := <-done: + if st.Code != lostExitCode { + t.Errorf("wait on a VM that died = %+v", st) + } + case <-time.After(10 * time.Second): + t.Fatal("wait kept waiting on a VM that died") + } +} + +func TestAnOldWaitLeavesANewBootRunning(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + if err := h.engine.stop(t.Context(), "run-a", time.Second); err != nil { + t.Fatal(err) + } + if err := h.engine.start(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + h.guest("run-a").procs[guest.MainID].exit(9) + code, ok := h.engine.waitMain(t.Context(), "run-a", 1) + if !ok || code != 143 { + t.Errorf("a wait on boot 1 = %d, %v; want boot 1's exit 143", code, ok) + } + if h.units.vmm("run-a") == nil { + t.Error("a wait on the previous boot powered off the new one") + } + if info, _ := h.engine.inspect(t.Context(), "run-a"); info.Exited { + t.Errorf("the new boot inspects as exited: %+v", info) + } +} + +func TestManagedExecRecoveryKeepsTheExitRecordAndTransientErrors(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + m, err := h.engine.startManagedExec(t.Context(), "run-a", runtime.ExecSpec{Argv: []string{"bash"}, CreationKey: "terminal-1", Cols: 80, Rows: 24}, true) + if err != nil { + t.Fatal(err) + } + identity := m.Identity() + if st, stopErr := m.Stop(t.Context(), time.Second); stopErr != nil || st.Code != 143 { + t.Fatalf("stop = %+v, %v", st, stopErr) + } + g := h.guest("run-a") + g.mu.Lock() + delete(g.procs, identity.ExecID) + g.mu.Unlock() + if st, stErr := m.Status(t.Context()); stErr != nil || !st.Exited || *st.ExitCode != 143 { + t.Errorf("status once the helper forgot the supervisor = %+v, %v", st, stErr) + } + if _, err = h.engine.recoverExec(t.Context(), "run-a", identity); err != nil { + t.Errorf("recovery once the helper forgot the supervisor = %v", err) + } + h.units.stateErr = errors.New("systemctl: connection timed out") + _, err = h.engine.recoverExec(t.Context(), "run-a", identity) + if !errors.Is(err, runtime.ErrExecUnavailable) || errors.Is(err, runtime.ErrNotFound) { + t.Errorf("recovery while systemd does not answer = %v, want unavailable, not gone", err) + } +} + +func TestPausedVMsAreNotAskedForStats(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + h.guest("run-a").stats = guest.Stats{OOMKills: 3, MemoryLimitBytes: 2 << 30} + if _, err := h.engine.memoryEvents(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if err := h.engine.pause(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + h.guest("run-a").stats = guest.Stats{OOMKills: 99} + if events, err := h.engine.memoryEvents(t.Context(), "run-a"); err != nil || events.OOMKills != 3 { + t.Errorf("memory events of a paused VM = %+v, %v; want the last reading", events, err) + } + if stats, err := h.engine.stats(t.Context()); err != nil || len(stats) != 0 { + t.Errorf("stats of a paused VM = %+v, %v", stats, err) + } + if _, err := h.engine.memoryEvents(t.Context(), "run-b"); err == nil || errors.Is(err, fs.ErrNotExist) { + t.Errorf("memory events of an unknown VM = %v", err) + } +} diff --git a/internal/runtime/vm/fakes_test.go b/internal/runtime/vm/fakes_test.go index da40e6b20..1c34157b2 100644 --- a/internal/runtime/vm/fakes_test.go +++ b/internal/runtime/vm/fakes_test.go @@ -122,6 +122,7 @@ type fakeUnits struct { active map[string]*fakeVMM started []startCall failNext bool + stateErr error } type startCall struct { @@ -175,6 +176,9 @@ func (u *fakeUnits) Stop(_ context.Context, name string) error { func (u *fakeUnits) State(_ context.Context, name string) (unitState, error) { u.mu.Lock() defer u.mu.Unlock() + if u.stateErr != nil { + return unitState{}, u.stateErr + } if u.active[name] != nil { return unitState{Active: true, MainPID: 4242}, nil } @@ -285,6 +289,8 @@ type fakeHelper struct { ignoreTE bool clock time.Time stats guest.Stats + // finals are dev-exec's own exit records, which outlive the helper's. + finals map[string]int } func (g *fakeHelper) Ping(ctx context.Context) (guest.Info, error) { @@ -333,7 +339,9 @@ func (g *fakeHelper) Spawn(_ context.Context, spec guest.ProcessSpec) (process, func (g *fakeHelper) control(key, execID, claim, action string) string { state := devexec.State{CreationKey: key, ExecID: execID, ClaimToken: claim} supervisor := g.procs[execID] - if supervisor == nil { + if code, ok := g.finals[execID]; ok && supervisor == nil { + state.Exited, state.ExitCode = true, &code + } else if supervisor == nil { state.Error = "no such execution" } else { if action == "stop" { @@ -342,6 +350,10 @@ func (g *fakeHelper) control(key, execID, claim, action string) string { if st, _ := supervisor.Status(context.Background()); st.Exited { code := st.ExitCode state.Exited, state.ExitCode = true, &code + if g.finals == nil { + g.finals = map[string]int{} + } + g.finals[execID] = code } else { state.Running = true } @@ -420,8 +432,23 @@ func (p *fakeProc) exit(code int) { } } +// Attach serves the programmed output. "hang" never ends stdout, and +// "errflood:" sends stderr past the exec output limit first; both end only +// when the attachment closes. func (p *fakeProc) Attach(bool) runtime.Attachment { - return &fakeAttachment{stdout: strings.NewReader(p.out), stderr: strings.NewReader("")} + stdout, held := io.Pipe() + a := &fakeAttachment{stdout: stdout, stderr: strings.NewReader(""), closed: func() { _ = held.CloseWithError(io.ErrClosedPipe) }} + switch p.out { + case "hang": + case "errflood:": + a.stderr = strings.NewReader(strings.Repeat("e", runtime.ExecOutputLimit+1)) + default: + go func() { + _, _ = io.WriteString(held, p.out) + _ = held.Close() + }() + } + return a } func (p *fakeProc) CloseStdin(context.Context) error { @@ -475,13 +502,19 @@ func (missingProc) Wait(context.Context) (int, error) { return 0, guest.ErrNotFo type fakeAttachment struct { stdout, stderr io.Reader + closed func() } func (a *fakeAttachment) Stdin() io.WriteCloser { return nopWriteCloser{} } func (a *fakeAttachment) Stdout() io.Reader { return a.stdout } func (a *fakeAttachment) Stderr() io.Reader { return a.stderr } func (a *fakeAttachment) Resize(context.Context, uint, uint) error { return nil } -func (a *fakeAttachment) Close() error { return nil } +func (a *fakeAttachment) Close() error { + if a.closed != nil { + a.closed() + } + return nil +} type nopWriteCloser struct{} diff --git a/internal/runtime/vm/lifecycle.go b/internal/runtime/vm/lifecycle.go index e9fd8caf6..34484b2d1 100644 --- a/internal/runtime/vm/lifecycle.go +++ b/internal/runtime/vm/lifecycle.go @@ -126,16 +126,20 @@ func (e *engine) kill(ctx context.Context, key string) { _ = e.units.Stop(ctx, unitName(key)) } -// wait blocks until Main exits. On a VM that never booted it waits for the -// first boot, and on a saved VM until it is restored, so a save is not -// mistaken for an exit. +// wait blocks until Main of the current boot exits; a later boot means it +// did. On a VM that never booted it waits for the first boot, and on a saved +// VM until it is restored, so a save is not mistaken for an exit. func (e *engine) wait(ctx context.Context, key string) (runtime.ExitStatus, error) { + boot := -1 for { rec, err := e.load(key) if err != nil { return runtime.ExitStatus{}, err } - if rec.Exited { + if boot < 0 && rec.Booted { + boot = rec.Boots + } + if rec.Exited || (boot >= 0 && rec.Boots != boot) { return runtime.ExitStatus{Code: rec.ExitCode}, nil } if rec.Booted && !rec.Saved && !e.flag(e.booting, key) { @@ -149,7 +153,7 @@ func (e *engine) wait(ctx context.Context, key string) (runtime.ExitStatus, erro } continue } - if code, ok := e.waitMain(ctx, key); ok { + if code, ok := e.waitMain(ctx, key, boot); ok { return runtime.ExitStatus{Code: code}, nil } } @@ -161,14 +165,33 @@ func (e *engine) wait(ctx context.Context, key string) (runtime.ExitStatus, erro } } -// waitMain waits for Main through the helper and finishes the VM; false -// means the helper went away first and the caller looks again. -func (e *engine) waitMain(ctx context.Context, key string) (int, bool) { +// waitMain waits for Main of boot through the helper and finishes the VM; +// false means the wait ended first and the caller looks again. The helper's +// client retries a VM that died until its context ends, so the wait also +// ends when the unit does. +func (e *engine) waitMain(ctx context.Context, key string, boot int) (int, bool) { h, err := e.helper(key) if err != nil { return 0, false } - code, err := h.Process(guest.MainID).Wait(ctx) + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-waitCtx.Done(): + return + case <-tick.C: + if up, upErr := e.running(waitCtx, key); upErr == nil && !up { + cancel() + return + } + } + } + }() + code, err := h.Process(guest.MainID).Wait(waitCtx) if err != nil { return 0, false } @@ -178,7 +201,7 @@ func (e *engine) waitMain(ctx context.Context, key string) (int, bool) { if err != nil { return 0, false } - if rec.Exited { + if rec.Exited || rec.Boots != boot { return rec.ExitCode, true } if err := e.finish(ctx, rec, code); err != nil { @@ -277,10 +300,14 @@ func (e *engine) inspect(ctx context.Context, key string) (runtime.ContainerInfo for i, m := range rec.Mounts { targets[i] = m.ContainerPath } - return runtime.ContainerInfo{ + info := runtime.ContainerInfo{ Image: rec.Image, User: rec.User, Env: append([]string(nil), rec.Env...), MountTargets: targets, - Fingerprint: rec.Fingerprint, Exited: rec.Exited, ExitCode: rec.ExitCode, - }, nil + Fingerprint: rec.Fingerprint, Exited: rec.Exited, + } + if rec.Exited { + info.ExitCode = rec.ExitCode + } + return info, nil } func (e *engine) find(key string) (runtime.ID, error) { diff --git a/internal/runtime/vm/managed.go b/internal/runtime/vm/managed.go index 6361a43a0..e74ca5001 100644 --- a/internal/runtime/vm/managed.go +++ b/internal/runtime/vm/managed.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "path" "strings" "sync" @@ -60,10 +61,13 @@ func (e *engine) startManagedExec(ctx context.Context, key string, spec runtime. } execID := processID("managed") argv := append([]string{coordtransport.CLIPath, devexec.Command, mode, spec.CreationKey, claim}, spec.Argv...) - p, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: execID, Argv: argv, Env: env, User: spec.User, Dir: spec.WorkingDir, TTY: size, Lossless: spec.Lossless}) + p, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: execID, Argv: argv, Env: env, User: spec.User, Dir: spec.WorkingDir, TTY: size, Lossless: spec.Lossless || !tty}) if err != nil { return nil, fmt.Errorf("runtime: create managed execution: %w", err) } + if !tty { + att = newPipeAttachment(p, att, spec.Lossless) + } m := &managedExec{engine: e, key: key, tty: tty, proc: p, attachment: att, identity: runtime.ExecIdentity{ ContainerID: runtime.ID(idPrefix + key), ExecID: execID, CreationKey: spec.CreationKey, ClaimToken: claim, User: spec.User, }} @@ -100,11 +104,15 @@ func (e *engine) recoverExec(ctx context.Context, key string, identity runtime.E if identity.ExecID == "" || identity.CreationKey == "" || identity.ClaimToken == "" { return nil, fmt.Errorf("%w: incomplete identity", runtime.ErrExecUnavailable) } - if _, err := e.requireRunning(ctx, key, "recover an execution in"); err != nil { - if errors.Is(err, runtime.ErrNotFound) { - return nil, err - } - return nil, fmt.Errorf("%w: %w: %w", runtime.ErrExecUnavailable, runtime.ErrNotFound, err) + if _, err := e.load(key); err != nil { + return nil, err + } + up, err := e.running(ctx, key) + if err != nil { + return nil, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + if !up { + return nil, fmt.Errorf("%w: %w: VM %s is not running", runtime.ErrExecUnavailable, runtime.ErrNotFound, key) } h, err := e.helper(key) if err != nil { @@ -114,7 +122,10 @@ func (e *engine) recoverExec(ctx context.Context, key string, identity runtime.E m := &managedExec{engine: e, key: key, proc: p, identity: identity} _, err = p.Status(ctx) if errors.Is(err, guest.ErrNotFound) { - return nil, fmt.Errorf("%w: %w: execution %s", runtime.ErrExecUnavailable, runtime.ErrNotFound, identity.ExecID) + if _, err = m.final(ctx); err != nil { + return nil, fmt.Errorf("%w: execution %s: %w", runtime.ErrNotFound, identity.ExecID, err) + } + return m, nil } if err != nil { return nil, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) @@ -155,7 +166,7 @@ func (m *managedExec) control(ctx context.Context, action string, grace time.Dur if err = p.CloseStdin(ctx); err != nil { return devexec.State{}, fmt.Errorf("%w: helper %s: %w", runtime.ErrExecUnavailable, action, err) } - stdout, stderr, err := readLimited(att) + stdout, stderr, err := readLimited(ctx, att) if err != nil { return devexec.State{}, fmt.Errorf("%w: helper %s output: %w", runtime.ErrExecUnavailable, action, err) } @@ -166,23 +177,38 @@ func (m *managedExec) control(ctx context.Context, action string, grace time.Dur return runtime.ExecControlState(m.identity, action, code, stdout, stderr) } -// Status asks the helper whether the supervisor runs; once it has ended, the -// supervisor's own record holds the command's exit. +// Status asks the helper whether the supervisor runs. Once it has ended, the +// supervisor's own record holds the command's exit; it outlives the helper, +// which forgets exited processes. func (m *managedExec) Status(ctx context.Context) (runtime.ExecState, error) { st, err := m.proc.Status(ctx) if errors.Is(err, guest.ErrNotFound) { - return runtime.ExecState{}, fmt.Errorf("%w: %w: execution %s", runtime.ErrExecUnavailable, runtime.ErrNotFound, m.identity.ExecID) + state, finalErr := m.final(ctx) + if finalErr != nil { + return state, fmt.Errorf("%w: %w: execution %s: %w", runtime.ErrExecUnavailable, runtime.ErrNotFound, m.identity.ExecID, finalErr) + } + return state, nil } if err != nil { return runtime.ExecState{}, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) } - state := runtime.ExecState{Running: !st.Exited, Attached: m.Attachment() != nil} + if st.Exited { + return m.final(ctx) + } + return m.state(), nil +} + +func (m *managedExec) state() runtime.ExecState { + state := runtime.ExecState{Running: true, Attached: m.Attachment() != nil} if !state.Attached { state.UnavailableReason = "attachment is unavailable; the execution was detached" } - if !st.Exited { - return state, nil - } + return state +} + +func (m *managedExec) final(ctx context.Context) (runtime.ExecState, error) { + state := m.state() + state.Running = false final, err := m.control(ctx, "status", 0) if err != nil { return state, fmt.Errorf("%w: command exit record unavailable: %w", runtime.ErrExecUnavailable, err) @@ -190,7 +216,7 @@ func (m *managedExec) Status(ctx context.Context) (runtime.ExecState, error) { if !final.Exited || final.ExitCode == nil { return state, fmt.Errorf("%w: helper ended without a command exit record", runtime.ErrExecUnavailable) } - state.Running, state.Exited, state.ExitCode = false, true, final.ExitCode + state.Exited, state.ExitCode = true, final.ExitCode return state, nil } @@ -257,3 +283,48 @@ func (m *managedExec) Detach() error { } return nil } + +// pipeAttachment gives a pipe execution Docker's semantics: closing its input +// ends the command's stdin, and unless the caller asked for every stream to +// be lossless, stderr drains into a buffer that drops its oldest bytes, so an +// unread stderr never stalls the lossless stdout. +type pipeAttachment struct { + runtime.Attachment + stdin pipeInput + stderr io.Reader +} + +func newPipeAttachment(p process, att runtime.Attachment, lossless bool) *pipeAttachment { + a := &pipeAttachment{Attachment: att, stdin: pipeInput{WriteCloser: att.Stdin(), proc: p}, stderr: att.Stderr()} + if !lossless { + drained := runtime.NewStreamBuffer() + go func() { + _, err := io.Copy(drained, att.Stderr()) + drained.CloseWithError(err) + }() + a.stderr = drained + } + return a +} + +func (a *pipeAttachment) Stdin() io.WriteCloser { return a.stdin } +func (a *pipeAttachment) Stderr() io.Reader { return a.stderr } + +type pipeInput struct { + io.WriteCloser + proc process +} + +func (in pipeInput) WriteContext(ctx context.Context, b []byte) (int, error) { + if w, ok := in.WriteCloser.(runtime.ContextWriter); ok { + return w.WriteContext(ctx, b) + } + return in.Write(b) +} + +func (in pipeInput) Close() error { + _ = in.WriteCloser.Close() + ctx, cancel := context.WithTimeout(context.Background(), stdinCloseTimeout) + defer cancel() + return in.proc.CloseStdin(ctx) +} diff --git a/internal/runtime/vm/record.go b/internal/runtime/vm/record.go index 6cd65e823..0ad092a11 100644 --- a/internal/runtime/vm/record.go +++ b/internal/runtime/vm/record.go @@ -59,11 +59,14 @@ type record struct { Mounts []runtime.Mount `json:"mounts,omitempty"` Net int `json:"net"` // Booted is set once a boot was launched: the next cold start builds a - // fresh root disk. + // fresh root disk. Boots counts them, so a wait on one boot can tell it + // from the next. Booted bool `json:"booted,omitempty"` + Boots int `json:"boots,omitempty"` // Saved means the snapshot directory holds the VM's state and the VM is // stopped until Restore. - Saved bool `json:"saved,omitempty"` + Saved bool `json:"saved,omitempty"` + // ExitCode is the last boot's until the next boot exits. Exited bool `json:"exited,omitempty"` ExitCode int `json:"exit_code,omitempty"` } diff --git a/internal/runtime/vm/save.go b/internal/runtime/vm/save.go index 69ce4e1f1..16ba7a973 100644 --- a/internal/runtime/vm/save.go +++ b/internal/runtime/vm/save.go @@ -84,8 +84,9 @@ func (e *engine) awaitStopped(ctx context.Context, key string) error { // restore starts the monitor on the saved state, resumes the guest and sets // its clock. Once the guest runs, its disks move past the snapshot, so the -// snapshot is dropped before anything else can fail: a second restore of it -// would pair old memory with newer disks. +// snapshot is dropped durably before the guest resumes: a second restore of +// it would pair old memory with newer disks. Cloud Hypervisor has read all +// of it by then. func (e *engine) restore(ctx context.Context, key string) (err error) { unlock := e.lock(key) defer unlock() @@ -118,9 +119,6 @@ func (e *engine) restore(ctx context.Context, key string) (err error) { if err = e.awaitRestored(ctx, key, api); err != nil { return err } - if err = api.resume(ctx); err != nil { - return fmt.Errorf("vm runtime: resume restored VM %s: %w", key, err) - } rec.Saved = false if err = e.store(rec); err != nil { return err @@ -128,6 +126,9 @@ func (e *engine) restore(ctx context.Context, key string) (err error) { if err = os.RemoveAll(dir); err != nil { return fmt.Errorf("vm runtime: remove the saved state of VM %s: %w", key, err) } + if err = api.resume(ctx); err != nil { + return fmt.Errorf("vm runtime: resume restored VM %s: %w", key, err) + } h, err := e.helper(key) if err != nil { return err diff --git a/internal/runtime/vm/stats.go b/internal/runtime/vm/stats.go index 8834454f5..dcf79c73e 100644 --- a/internal/runtime/vm/stats.go +++ b/internal/runtime/vm/stats.go @@ -20,7 +20,8 @@ const ( ) // memoryEvents reports a stopped VM as fs.ErrNotExist, as the OOM watcher -// treats a stopped container's missing cgroup. +// treats a stopped container's missing cgroup. A paused guest cannot answer +// and kills nothing, so it reports what it last did. func (e *engine) memoryEvents(ctx context.Context, key string) (runtime.MemoryEvents, error) { if _, err := e.load(key); err != nil { return runtime.MemoryEvents{}, err @@ -28,6 +29,15 @@ func (e *engine) memoryEvents(ctx context.Context, key string) (runtime.MemoryEv if up, err := e.running(ctx, key); err != nil || !up { return runtime.MemoryEvents{}, errors.Join(err, fmt.Errorf("vm runtime: VM %s is not running: %w", key, fs.ErrNotExist)) } + if e.paused(ctx, key) { + e.mu.Lock() + events, known := e.events[key] + e.mu.Unlock() + if !known { + return events, fmt.Errorf("vm runtime: VM %s is paused: %w", key, fs.ErrNotExist) + } + return events, nil + } h, err := e.helper(key) if err != nil { return runtime.MemoryEvents{}, err @@ -36,7 +46,16 @@ func (e *engine) memoryEvents(ctx context.Context, key string) (runtime.MemoryEv if err != nil { return runtime.MemoryEvents{}, fmt.Errorf("vm runtime: memory events of VM %s: %w", key, err) } - return runtime.MemoryEvents{OOMKills: st.OOMKills, LimitBytes: st.MemoryLimitBytes}, nil + events := runtime.MemoryEvents{OOMKills: st.OOMKills, LimitBytes: st.MemoryLimitBytes} + e.mu.Lock() + e.events[key] = events + e.mu.Unlock() + return events, nil +} + +func (e *engine) paused(ctx context.Context, key string) bool { + state, err := newCHAPI(e.path(key, apiSocket)).state(ctx) + return err == nil && state == "Paused" } // CPU, memory and processes come from the helper, network and disk bytes @@ -73,7 +92,7 @@ func (e *engine) sample(ctx context.Context, key string) (*runtime.ContainerStat if err != nil { return nil, err } - if up, upErr := e.running(ctx, key); upErr != nil || !up { + if up, upErr := e.running(ctx, key); upErr != nil || !up || e.paused(ctx, key) { return nil, upErr } h, err := e.helper(key)