diff --git a/cmd/aether-server/main.go b/cmd/aether-server/main.go index b8c660ac7..1767fdda9 100644 --- a/cmd/aether-server/main.go +++ b/cmd/aether-server/main.go @@ -26,6 +26,7 @@ import ( "github.com/3xDevOps/Aether/internal/guest/helper" "github.com/3xDevOps/Aether/internal/harness" "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/runtime/vm" "github.com/3xDevOps/Aether/internal/scheduler" "github.com/3xDevOps/Aether/internal/server" "github.com/3xDevOps/Aether/internal/serversetup" @@ -50,6 +51,9 @@ func main() { if len(os.Args) > 1 && os.Args[1] == "guest" { os.Exit(helper.Main()) } + if len(os.Args) > 1 && os.Args[1] == vm.MonitorCommand { + exitOn(vm.Monitor(os.Args[2:])) + } if filepath.Base(os.Args[0]) == "aether-internal" { os.Exit(coordcli.Main(os.Args[1:])) } @@ -251,6 +255,8 @@ type serveOptions struct { runMemory *runCountValue runPids *runCountValue containerRuntime *string + sessionRuntime *string + vmKernel *string } // serveFlags declares the server options on fs. It is the single definition @@ -312,6 +318,10 @@ func serveFlags(fs *flag.FlagSet) *serveOptions { o.containerRuntime = fs.String("container-runtime", server.DefaultContainerRuntime, "OCI runtime for run and member environment containers: auto uses sysbox-runc when Docker has it registered "+ "(Docker inside sessions without --privileged), otherwise the daemon default; any other value is used as given, e.g. runc to opt out") + o.sessionRuntime = fs.String("session-runtime", server.SessionRuntimeDocker, + "where sessions run: docker, or vm for one Cloud Hypervisor VM per session (needs /dev/kvm, cloud-hypervisor and virtiofsd; "+ + "every other container stays on Docker)") + o.vmKernel = fs.String("vm-kernel", server.DefaultVMKernel, "the guest kernel session VMs boot with -session-runtime vm") return o } @@ -393,6 +403,8 @@ func serve(args []string) error { RunPidsLimit: int64(*o.runPids), ContainerRuntime: *o.containerRuntime, + SessionRuntime: *o.sessionRuntime, + VMKernel: *o.vmKernel, SysboxInstaller: true, }) if err != nil { diff --git a/docs/environments.md b/docs/environments.md index daa4be68f..5441b4e33 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -314,6 +314,46 @@ pressure](failure-handling.md#recover-from-disk-pressure) and [Recover from memory pressure](failure-handling.md#recover-from-memory-pressure) have the thresholds and what to do. +## Sessions in virtual machines + +`--session-runtime` (config key `session-runtime`) defaults to `docker`. With +`vm`, each new session runs in its own [Cloud +Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor) virtual +machine instead of a container. Environment terminals, the assistant, the +agent updater, the development browser and **Save environment** stay on +Docker. +This is work in progress; [the plan](plans/2026-11-13-microvm-sessions.md) +lists what is still to come. + +```sh +sudo aether-server config set session-runtime vm +sudo systemctl restart aether-server +``` + +The host needs `/dev/kvm`, `cloud-hypervisor`, `/usr/libexec/virtiofsd`, +`qemu-img`, `mkfs.ext4`, `iptables` and systemd, besides Docker, and the +release's guest kernel `aether-vmlinux-x86_64` at `/usr/local/lib/aether/vmlinux` +(`--vm-kernel`, config key `vm-kernel`, names another path). + +What changes: + +- Each VM runs in the systemd unit `aether-vm-.service` in + `aether.slice`, so it keeps running through a server restart or update. + Its disks and logs are in `/vm//`: `monitor.log` holds the + VM monitor's output and `console.log` the guest kernel's. +- The VM gets the CPU limit rounded up to whole vCPUs, held to the exact + limit by the unit's `CPUQuota`, and the memory limit as its memory. The + process limit applies inside the guest. +- The checkout, the home and the other mounts reach the VM over one virtio-fs + share; read-only mounts stay read-only. Each VM has its own address in + `10.200.0.0/16` and reaches the internet through the host. +- Starting a stopped session builds a fresh root disk from its image and runs + the setup script again: packages installed outside the mounts are gone. + Docker images inside the session stay on its data disk. +- Not yet supported, refused at launch with an error naming the work item: + the Isolated tier and egress control, and the development browser beside a + session. + ## After a session closes Reporting success or failure leaves an interactive session open for follow-up, and diff --git a/internal/runtime/docker.go b/internal/runtime/docker.go index e62508455..e060b910b 100644 --- a/internal/runtime/docker.go +++ b/internal/runtime/docker.go @@ -15,7 +15,6 @@ import ( "math" "path/filepath" "slices" - "strconv" "strings" "sync" "time" @@ -55,13 +54,6 @@ const ( setupSentinelPrefix = "/tmp/.aether-setup-" ) -// execOutputLimit covers stdout and stderr together. -const execOutputLimit = 1 << 20 - -// setupOutputLimit is how much of a setup script's output is kept, from the -// end, where the failure is. -const setupOutputLimit = 16 << 10 - // hijackWriteTimeout bounds one stdin write. Cancellation interrupts a write // only through its deadline; it never closes the shared stdin stream. const hijackWriteTimeout = 5 * time.Second @@ -290,7 +282,7 @@ func (d *Docker) containerConfig(spec Spec, containerRuntime string) (*container cfg := &container.Config{ Image: spec.Image, User: spec.User, - Env: dockerEnv(trustCheckouts(spec.Env, spec.WorktreeMountPath, spec.ExtraWorktrees)), + Env: EnvList(TrustCheckouts(spec.Env, spec.WorktreeMountPath, spec.ExtraWorktrees)), WorkingDir: spec.WorkingDir, Entrypoint: spec.Command, Labels: labels, @@ -313,7 +305,7 @@ func (d *Docker) containerConfig(spec Spec, containerRuntime string) (*container nestedDockerd := spec.NestedDocker && containerRuntime == runtimeSysbox // Gated so the agent starts only once Start has checked every subpath mount. subpaths := slices.ContainsFunc(spec.Mounts, func(m Mount) bool { return m.Subpath != "" }) - if spec.SetupScript != "" || nestedDockerd || subpaths || unprivileged(spec.User) { + if spec.SetupScript != "" || nestedDockerd || subpaths || Unprivileged(spec.User) { sentinel := newSetupSentinel() cfg.Entrypoint = gateEntrypoint(sentinel) cfg.Cmd = spec.Command @@ -486,47 +478,6 @@ func gateEntrypoint(sentinel string) []string { return []string{"/bin/sh", "-c", script, "aether-gate"} } -func dockerEnv(env map[string]string) []string { - if len(env) == 0 { - return nil - } - out := make([]string, 0, len(env)) - for _, k := range slices.Sorted(maps.Keys(env)) { - out = append(out, k+"="+env[k]) - } - return out -} - -func trustCheckouts(env map[string]string, checkout string, extras []Mount) map[string]string { - env = trustCheckout(env, checkout) - for _, m := range extras { - env = trustCheckout(env, m.ContainerPath) - } - return env -} - -// Execs inherit the container env, so this also covers every git the server -// runs inside the container. -func trustCheckout(env map[string]string, checkout string) map[string]string { - if checkout == "" { - return env - } - n := 0 - if count, ok := env["GIT_CONFIG_COUNT"]; ok { - parsed, err := strconv.Atoi(count) - if err != nil || parsed < 0 { - return env - } - n = parsed - } - out := make(map[string]string, len(env)+3) - maps.Copy(out, env) - out["GIT_CONFIG_KEY_"+strconv.Itoa(n)] = "safe.directory" - out["GIT_CONFIG_VALUE_"+strconv.Itoa(n)] = checkout - out["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1) - return out -} - func nanoCPUs(cores float64) int64 { return int64(math.Round(cores * 1e9)) } @@ -564,7 +515,7 @@ func (d *Docker) Start(ctx context.Context, id ID) error { return nil } secrets := secretEnvValues(labels[labelSecretEnv], info.Container.Config.Env) - if err := d.prepare(ctx, id, labels, sentinel, info.Container.Config.WorkingDir, info.Container.Config.User, envValue(info.Container.Config.Env, "HOME"), secrets); err != nil { + if err := d.prepare(ctx, id, labels, sentinel, info.Container.Config.WorkingDir, info.Container.Config.User, EnvValue(info.Container.Config.Env, "HOME"), secrets); err != nil { // Never kill a run that was already live before this Start call: a // redundant Start must not take down a healthy container. if !wasRunning { @@ -580,7 +531,7 @@ func (d *Docker) Start(ctx context.Context, id ID) error { } func (d *Docker) prepare(ctx context.Context, id ID, labels map[string]string, sentinel, workDir, user, home string, secrets []string) error { - if unprivileged(user) { + if Unprivileged(user) { if err := d.prepareUser(ctx, id, user, home, labels[labelSudo] != ""); err != nil { return err } @@ -641,7 +592,7 @@ func (d *Docker) startDockerd(ctx context.Context, id ID, user string) error { ctx, cancel := context.WithTimeout(ctx, dockerdStartTimeout) defer cancel() code, output, cut, err := d.execCombined(ctx, id, []string{"/bin/sh", "-c", dockerdScript, "aether-dockerd", user}, "", "0:0") - output = setupOutput(output, cut, nil) + output = SetupOutput(output, cut, nil) if err != nil { slog.Error("runtime: dockerd start failed", "container", id, "output", output, "error", err) return fmt.Errorf("runtime: start dockerd: %w", err) @@ -669,7 +620,7 @@ func (d *Docker) runSetup(ctx context.Context, id ID, script, sentinel, workDir d.settingUp.Store(id, struct{}{}) code, output, cut, err := d.execCombined(ctx, id, []string{"/bin/sh", "-ec", "exec 2>&1; " + script}, workDir, "") d.settingUp.Delete(id) - output = setupOutput(output, cut, secrets) + output = SetupOutput(output, cut, secrets) if err != nil { slog.Error("runtime: setup script execution failed", "container", id, "working_dir", workDir, "script", script, "output", output, "error", err) return fmt.Errorf("runtime: setup script: %w", err) @@ -695,51 +646,6 @@ func (d *Docker) releaseGate(ctx context.Context, id ID, sentinel string) error return nil } -func secretEnvValues(label string, env []string) []string { - var names []string - if json.Unmarshal([]byte(label), &names) != nil { - return nil - } - var values []string - for _, pair := range env { - name, value, _ := strings.Cut(pair, "=") - if value != "" && slices.Contains(names, name) { - values = append(values, value) - } - } - return values -} - -const secretMask = "***" - -// setupOutput masks exact secret values, longest first so a secret holding -// another is masked whole, then keeps the last setupOutputLimit bytes. cut -// says output already lost its beginning, which may have split a secret: the -// span that could hold its remainder is dropped. That span is measured after -// masking, where a shorter secret inside the remainder can have grown it to -// len(secretMask) times its length. -func setupOutput(output string, cut bool, secrets []string) string { - if len(secrets) > 0 { - slices.SortFunc(secrets, func(a, b string) int { return len(b) - len(a) }) - pairs := make([]string, 0, 2*len(secrets)) - for _, secret := range secrets { - pairs = append(pairs, secret, secretMask) - } - output = strings.NewReplacer(pairs...).Replace(output) - if cut { - output = output[min(len(secretMask)*len(secrets[0]), len(output)):] - } - } - if cut || len(output) > setupOutputLimit { - output = output[max(0, len(output)-setupOutputLimit):] - if i := strings.IndexByte(output, '\n'); i >= 0 { - output = output[i+1:] - } - output = "[earlier output omitted]\n" + output - } - return strings.ToValidUTF8(output, "\uFFFD") -} - // tailWriter keeps the last limit bytes written to it. The buffer grows to // twice that before it is compacted, so a script printing line by line does // not cost a copy of the whole buffer per line. @@ -764,9 +670,9 @@ func (w *tailWriter) tail() (kept string, cut bool) { } // execCombined is Exec for the setup script, which may print without limit: -// one transcript, of which the last execOutputLimit bytes are kept. +// one transcript, of which the last ExecOutputLimit bytes are kept. func (d *Docker) execCombined(ctx context.Context, id ID, cmd []string, workDir, user string) (code int, output string, cut bool, err error) { - kept := &tailWriter{limit: execOutputLimit} + kept := &tailWriter{limit: ExecOutputLimit} code, err = d.exec(ctx, id, cmd, workDir, user, nil, func(r io.Reader) error { _, copyErr := stdcopy.StdCopy(kept, kept, r) return copyErr @@ -786,10 +692,6 @@ func (d *Docker) Exec(ctx context.Context, id ID, cmd []string, workDir string) // exec writes a non-nil stdin to the command's input, which stays open until // exec returns. -// rootPath is sudo's secure_path. A root exec never searches the -// container's PATH, which puts the agent-writable ~/.local/bin first. -const rootPath = "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - func (d *Docker) exec(ctx context.Context, id ID, cmd []string, workDir, user string, stdin []byte, read func(io.Reader) error) (int, error) { opts := client.ExecCreateOptions{ Cmd: cmd, @@ -800,7 +702,7 @@ func (d *Docker) exec(ctx context.Context, id ID, cmd []string, workDir, user st AttachStderr: true, } if uid, _, _ := strings.Cut(user, ":"); uid == "0" || uid == "root" { - opts.Env = []string{rootPath} + opts.Env = []string{RootPath} } created, err := d.cli.ExecCreate(ctx, string(id), opts) if err != nil { @@ -850,16 +752,16 @@ func (d *Docker) exec(ctx context.Context, id ID, cmd []string, workDir, user st } } -// readExecOutput refuses output past execOutputLimit: the container chooses +// readExecOutput refuses output past ExecOutputLimit: the container chooses // what it prints and the caller buffers it all. func readExecOutput(r io.Reader) (stdout, stderr string, err error) { // One byte past the cap, so an output of exactly that size is not // mistaken for a truncated one. - counted := &countingReader{r: io.LimitReader(r, execOutputLimit+1)} + counted := &countingReader{r: io.LimitReader(r, ExecOutputLimit+1)} var out, errOut bytes.Buffer _, copyErr := stdcopy.StdCopy(&out, &errOut, counted) - if counted.read > execOutputLimit { - return out.String(), errOut.String(), fmt.Errorf("output passed the %d byte limit", execOutputLimit) + if counted.read > ExecOutputLimit { + return out.String(), errOut.String(), fmt.Errorf("output passed the %d byte limit", ExecOutputLimit) } return out.String(), errOut.String(), copyErr } @@ -1123,8 +1025,8 @@ type dockerAttachment struct { id string tty bool resp client.HijackedResponse - stdout *streamBuffer - stderr *streamBuffer + stdout *StreamBuffer + stderr *StreamBuffer closeOnce sync.Once } @@ -1134,8 +1036,8 @@ func newDockerAttachment(cli *client.Client, id string, tty bool, resp client.Hi id: id, tty: tty, resp: resp, - stdout: newStreamBuffer(), - stderr: newStreamBuffer(), + stdout: NewStreamBuffer(), + stderr: NewStreamBuffer(), } go func() { var err error @@ -1209,84 +1111,3 @@ func (h hijackStdin) WriteContext(ctx context.Context, p []byte) (int, error) { } func (h hijackStdin) Close() error { return h.resp.CloseWrite() } - -const maxStreamBuffer = 8 << 20 - -// streamBuffer writes never block, since blocking the demux goroutine would -// stall the sibling stream; past maxStreamBuffer the oldest bytes are -// dropped. A lossless buffer blocks at the cap instead, for protocol streams -// that cannot lose a byte. -type streamBuffer struct { - mu sync.Mutex - cond *sync.Cond - buf bytes.Buffer - lossless bool - // limit is where a lossless buffer blocks its writer. - limit int - closed bool - err error -} - -func newStreamBuffer() *streamBuffer { - b := &streamBuffer{limit: maxStreamBuffer} - b.cond = sync.NewCond(&b.mu) - return b -} - -func newLosslessStreamBuffer() *streamBuffer { - b := newStreamBuffer() - b.lossless = true - return b -} - -func (b *streamBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - for b.lossless && !b.closed && b.buf.Len() >= b.limit { - b.cond.Wait() - } - if b.closed { - return 0, io.ErrClosedPipe - } - n, _ := b.buf.Write(p) // bytes.Buffer.Write cannot fail - if b.lossless { - b.cond.Broadcast() - return n, nil - } - if over := b.buf.Len() - maxStreamBuffer; over > 0 { - b.buf.Next(over) - } - b.cond.Broadcast() - return n, nil -} - -func (b *streamBuffer) Read(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - for b.buf.Len() == 0 && !b.closed { - b.cond.Wait() - } - if b.buf.Len() > 0 { - if b.lossless { - b.cond.Broadcast() - } - return b.buf.Read(p) - } - if b.err != nil { - return 0, b.err - } - return 0, io.EOF -} - -// CloseWithError leaves buffered data readable first. Only the first close -// takes effect. -func (b *streamBuffer) CloseWithError(err error) { - b.mu.Lock() - defer b.mu.Unlock() - if b.closed { - return - } - b.closed = true - b.err = err - b.cond.Broadcast() -} diff --git a/internal/runtime/docker_integration_test.go b/internal/runtime/docker_integration_test.go index e607b8ea7..e099610f3 100644 --- a/internal/runtime/docker_integration_test.go +++ b/internal/runtime/docker_integration_test.go @@ -343,7 +343,7 @@ func TestDockerSetupFailure(t *testing.T) { } // TestDockerSetupOutputPastTheBuffer verifies a setup script may print more -// than execOutputLimit: a passing one still starts the container, and a +// than ExecOutputLimit: a passing one still starts the container, and a // failing one reports the end of what it printed. func TestDockerSetupOutputPastTheBuffer(t *testing.T) { t.Parallel() diff --git a/internal/runtime/docker_managed_exec.go b/internal/runtime/docker_managed_exec.go index fd09809ef..f2abbb088 100644 --- a/internal/runtime/docker_managed_exec.go +++ b/internal/runtime/docker_managed_exec.go @@ -4,13 +4,11 @@ import ( "context" "crypto/rand" "encoding/hex" - "encoding/json" "errors" "fmt" "io" "path" "slices" - "strconv" "strings" "sync" "time" @@ -199,36 +197,15 @@ func (e *dockerManagedExec) forgotten(ctx context.Context, cause error) (devexec } func (e *dockerManagedExec) control(ctx context.Context, action string, grace time.Duration) (devexec.State, error) { - args := []string{coordtransport.CLIPath, devexec.Command, "control", e.identity.CreationKey, e.identity.ExecID, e.identity.ClaimToken, action, strconv.FormatInt(grace.Milliseconds(), 10)} var stdout, stderr string - code, err := e.docker.exec(ctx, e.identity.ContainerID, args, "", e.identity.User, nil, func(r io.Reader) (copyErr error) { + code, err := e.docker.exec(ctx, e.identity.ContainerID, ExecControlArgv(e.identity, action, grace), "", e.identity.User, nil, func(r io.Reader) (copyErr error) { stdout, stderr, copyErr = readExecOutput(r) return copyErr }) if err != nil { return devexec.State{}, err } - var state devexec.State - decodeErr := json.Unmarshal([]byte(stdout), &state) - if decodeErr == nil && state.ExecID == e.identity.ExecID && state.CreationKey == e.identity.CreationKey && state.ClaimToken == e.identity.ClaimToken { - if state.Error != "" { - if state.Exited && state.ExitCode != nil && (*state.ExitCode == 126 || *state.ExitCode == 127) { - if action == "start" { - return state, &ExecExitError{Code: *state.ExitCode} - } - return state, nil - } - return state, errors.New(state.Error) - } - if code == 0 { - return state, nil - } - } - failed := fmt.Errorf("%w: helper %s failed (status %d): %s", ErrExecUnavailable, action, code, strings.TrimSpace(stderr)) - if strings.Contains(stderr, devexec.ErrNoSupervisor.Error()) { - return state, fmt.Errorf("%w: %w", devexec.ErrNoSupervisor, failed) - } - return state, failed + return ExecControlState(e.identity, action, code, stdout, stderr) } func (e *dockerManagedExec) Status(ctx context.Context) (ExecState, error) { diff --git a/internal/runtime/docker_sudo.go b/internal/runtime/docker_sudo.go index 324d88020..8e5d7fde7 100644 --- a/internal/runtime/docker_sudo.go +++ b/internal/runtime/docker_sudo.go @@ -14,42 +14,12 @@ import ( "github.com/3xDevOps/Aether/internal/devexec" ) -// userScript gives uid $1 a passwd entry whose home is $2: git refuses to -// commit and ssh and ssh-keygen refuse to run for a uid with none, and ssh -// reads ~/.ssh from the passwd home, never $HOME. With $3 set it also lets -// the uid sudo without a password; the sudoers file is checked before it -// goes live, since sudo refuses to run at all with a broken one. -// -// The home at /root is the member's, shared by every container they run, -// and systemd's postinst runs systemd-tmpfiles, whose provision.conf hands -// /root to root: installing any package that pulls in systemd would take -// the home from all of them. The /etc entry masks it. -const userScript = `uid=${1%%:*} gid=${1#*:} -if grep -q "^[^:]*:[^:]*:$uid:" /etc/passwd; then - awk -F: -v OFS=: -v u="$uid" -v h="$2" '$3 == u { $6 = h } { print }' /etc/passwd > /etc/passwd.aether && cat /etc/passwd.aether > /etc/passwd && rm -f /etc/passwd.aether || exit 1 -else - echo "aether:x:$uid:$gid::$2:/bin/sh" >> /etc/passwd || exit 1 -fi -grep -q "^[^:]*:[^:]*:$gid:" /etc/group || echo "aether:x:$gid:" >> /etc/group || exit 1 -mkdir -p /etc/tmpfiles.d && ln -sfn /dev/null /etc/tmpfiles.d/provision.conf || exit 1 -[ -n "$3" ] || exit 0 -mkdir -p /etc/sudoers.d -printf '#%s ALL=(ALL:ALL) NOPASSWD: ALL\n' "$uid" > /etc/sudoers.d/.aether.new -chmod 0440 /etc/sudoers.d/.aether.new -if command -v visudo >/dev/null 2>&1 && ! visudo -cqf /etc/sudoers.d/.aether.new; then rm -f /etc/sudoers.d/.aether.new; exit 1; fi -mv -f /etc/sudoers.d/.aether.new /etc/sudoers.d/aether` - -func unprivileged(user string) bool { - uid, _, _ := strings.Cut(user, ":") - return uid != "" && uid != "0" -} - func (d *Docker) prepareUser(ctx context.Context, id ID, user, home string, sudo bool) error { allow := "" if sudo { allow = "sudo" } - code, output, _, err := d.execCombined(ctx, id, []string{"/bin/sh", "-c", userScript, "aether-user", user, cmp.Or(home, "/"), allow}, "", "0:0") + code, output, _, err := d.execCombined(ctx, id, []string{"/bin/sh", "-c", UserScript, "aether-user", user, cmp.Or(home, "/"), allow}, "", "0:0") if err != nil { return fmt.Errorf("runtime: prepare user %s: %w", user, err) } @@ -80,12 +50,3 @@ func (d *Docker) ExecAttached(ctx context.Context, id ID, spec AttachedExec) (in return err }) } - -func envValue(env []string, name string) string { - for _, pair := range env { - if key, value, _ := strings.Cut(pair, "="); key == name { - return value - } - } - return "" -} diff --git a/internal/runtime/docker_test.go b/internal/runtime/docker_test.go index b367ad447..b9944574e 100644 --- a/internal/runtime/docker_test.go +++ b/internal/runtime/docker_test.go @@ -314,7 +314,7 @@ func TestSetupOutputMasksSecretEnv(t *testing.T) { cfg, _ := d.containerConfig(spec, "") secrets := secretEnvValues(cfg.Labels[labelSecretEnv], cfg.Env) - got := setupOutput("+ curl -H abc-and-more\nmode plain, token abc\n", false, secrets) + got := SetupOutput("+ curl -H abc-and-more\nmode plain, token abc\n", false, secrets) if want := "+ curl -H ***\nmode plain, token ***\n"; got != want { t.Errorf("setupOutput = %q, want %q", got, want) } @@ -327,7 +327,7 @@ func TestSetupOutputMasksSecretEnv(t *testing.T) { func TestSetupOutputKeepsTheEnd(t *testing.T) { long := strings.Repeat("noise\n", setupOutputLimit) + "npm error missing script\n" - got := setupOutput(long, false, nil) + got := SetupOutput(long, false, nil) if len(got) > setupOutputLimit+64 || !strings.HasPrefix(got, "[earlier output omitted]\nnoise\n") || !strings.HasSuffix(got, "npm error missing script\n") { t.Errorf("setupOutput kept %d bytes, starting %q", len(got), got[:min(len(got), 40)]) } @@ -354,12 +354,12 @@ func TestSetupOutputKeepsTheEnd(t *testing.T) { if !cut || !strings.HasPrefix(text, "et-value\n") { t.Fatalf("tail = %q, cut %v, want it to start inside the secret", text, cut) } - if got := setupOutput(text, cut, []string{"secret-value"}); strings.Contains(got, "value") || !strings.HasSuffix(got, "last line\n") { + if got := SetupOutput(text, cut, []string{"secret-value"}); strings.Contains(got, "value") || !strings.HasSuffix(got, "last line\n") { t.Errorf("setupOutput after a cut inside the secret = %q", got) } // Nor when a shorter secret, masked inside that second half, makes it longer. grown := "qqqqqq\nZ9\n" + strings.Repeat("padding line\n", 4) + "last line\n" - if got := setupOutput(grown, true, []string{"q", "qqqqqqqq\nZ9"}); strings.Contains(got, "Z9") || !strings.HasSuffix(got, "last line\n") { + if got := SetupOutput(grown, true, []string{"q", "qqqqqqqq\nZ9"}); strings.Contains(got, "Z9") || !strings.HasSuffix(got, "last line\n") { t.Errorf("setupOutput after a cut inside a secret that holds another = %q", got) } } @@ -397,7 +397,7 @@ func TestContainerConfigNoMountNoLimits(t *testing.T) { func TestTrustCheckoutKeepsWorkspaceGitConfig(t *testing.T) { env := map[string]string{"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "core.editor", "GIT_CONFIG_VALUE_0": "vi"} - got := dockerEnv(trustCheckout(env, "/workspace")) + got := EnvList(trustCheckout(env, "/workspace")) want := []string{"GIT_CONFIG_COUNT=2", "GIT_CONFIG_KEY_0=core.editor", "GIT_CONFIG_KEY_1=safe.directory", "GIT_CONFIG_VALUE_0=vi", "GIT_CONFIG_VALUE_1=/workspace"} if !slices.Equal(got, want) { t.Errorf("env = %v, want %v", got, want) @@ -408,7 +408,7 @@ func TestTrustCheckoutKeepsWorkspaceGitConfig(t *testing.T) { } func TestDockerEnvSorted(t *testing.T) { - got := dockerEnv(map[string]string{"ZED": "1", "ALPHA": "2", "MID": "3"}) + got := EnvList(map[string]string{"ZED": "1", "ALPHA": "2", "MID": "3"}) want := []string{"ALPHA=2", "MID=3", "ZED=1"} if !slices.Equal(got, want) { t.Errorf("dockerEnv = %v, want %v", got, want) @@ -433,7 +433,7 @@ func TestNanoCPUs(t *testing.T) { } func TestStreamBufferWriteNeverBlocks(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() done := make(chan struct{}) go func() { defer close(done) @@ -452,7 +452,7 @@ func TestStreamBufferWriteNeverBlocks(t *testing.T) { } func TestStreamBufferDropsOldestPastCap(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() chunk := bytes.Repeat([]byte("a"), 4096) for range 4 * maxStreamBuffer / len(chunk) { if _, err := b.Write(chunk); err != nil { @@ -477,7 +477,7 @@ func TestStreamBufferDropsOldestPastCap(t *testing.T) { } func TestStreamBufferReadBlocksUntilData(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() got := make(chan string, 1) go func() { p := make([]byte, 16) @@ -502,7 +502,7 @@ func TestStreamBufferReadBlocksUntilData(t *testing.T) { } func TestStreamBufferCloseDrainsThenEOF(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() if _, err := b.Write([]byte("tail")); err != nil { t.Fatalf("Write() error: %v", err) } @@ -520,7 +520,7 @@ func TestStreamBufferCloseDrainsThenEOF(t *testing.T) { } func TestStreamBufferCloseWithError(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() want := errors.New("boom") b.CloseWithError(want) b.CloseWithError(nil) // only the first close takes effect @@ -530,7 +530,7 @@ func TestStreamBufferCloseWithError(t *testing.T) { } func TestStreamBufferConcurrent(t *testing.T) { - b := newStreamBuffer() + b := NewStreamBuffer() const writers, lines = 8, 200 var wg sync.WaitGroup for range writers { @@ -694,7 +694,7 @@ func TestReadExecOutput(t *testing.T) { var flood bytes.Buffer chunk := bytes.Repeat([]byte("y"), 64<<10) - for flood.Len() <= execOutputLimit+1 { + for flood.Len() <= ExecOutputLimit+1 { if writeErr := writeExecFrame(&flood, stdcopy.Stdout, chunk); writeErr != nil { t.Fatal(writeErr) } @@ -703,8 +703,8 @@ func TestReadExecOutput(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "1048576 byte limit") { t.Fatalf("readExecOutput on a flood = %v, want the cap named", err) } - if len(stdout) > execOutputLimit { - t.Errorf("buffered %d bytes, want no more than %d", len(stdout), execOutputLimit) + if len(stdout) > ExecOutputLimit { + t.Errorf("buffered %d bytes, want no more than %d", len(stdout), ExecOutputLimit) } } diff --git a/internal/runtime/exectty.go b/internal/runtime/exectty.go index 702bc8c1c..fe6c40758 100644 --- a/internal/runtime/exectty.go +++ b/internal/runtime/exectty.go @@ -21,8 +21,8 @@ type execAttachment struct { tty bool resp client.HijackedResponse - stdout *streamBuffer - stderr *streamBuffer + stdout *StreamBuffer + stderr *StreamBuffer done chan struct{} closeOnce sync.Once } @@ -38,8 +38,8 @@ func newExecAttachment(cli *client.Client, id string, tty, lossless bool, resp c id: id, tty: tty, resp: resp, - stdout: newStreamBuffer(), - stderr: newStreamBuffer(), + stdout: NewStreamBuffer(), + stderr: NewStreamBuffer(), done: make(chan struct{}), } if !tty || lossless { diff --git a/internal/runtime/exectty_test.go b/internal/runtime/exectty_test.go index eeaa9ccb5..ead658c4c 100644 --- a/internal/runtime/exectty_test.go +++ b/internal/runtime/exectty_test.go @@ -195,7 +195,7 @@ func TestLosslessExecKeepsStderrAndTerminalOutput(t *testing.T) { time.Sleep(200 * time.Millisecond) // Unread, the stream holds its small cap and one more // write, not the megabytes a lossy stream retains. - held := tc.stream(attachment).(*streamBuffer) + held := tc.stream(attachment).(*StreamBuffer) held.mu.Lock() buffered := held.buf.Len() held.mu.Unlock() diff --git a/internal/runtime/managed_exec.go b/internal/runtime/managed_exec.go index e6ce40030..8543a1efc 100644 --- a/internal/runtime/managed_exec.go +++ b/internal/runtime/managed_exec.go @@ -2,8 +2,15 @@ package runtime import ( "context" + "encoding/json" "errors" + "fmt" + "strconv" + "strings" "time" + + "github.com/3xDevOps/Aether/internal/coordtransport" + "github.com/3xDevOps/Aether/internal/devexec" ) // ErrExecUnavailable means execution identity or its terminal cannot be proved. @@ -83,3 +90,35 @@ type ManagedExec interface { Resize(context.Context, uint, uint) error Detach() error } + +// ExecControlArgv is the staged helper's control command for one action on +// the execution identity names. +func ExecControlArgv(identity ExecIdentity, action string, grace time.Duration) []string { + return []string{coordtransport.CLIPath, devexec.Command, "control", identity.CreationKey, identity.ExecID, identity.ClaimToken, action, strconv.FormatInt(grace.Milliseconds(), 10)} +} + +// ExecControlState reads the staged helper's reply to a control command that +// exited with code and printed stdout and stderr. +func ExecControlState(identity ExecIdentity, action string, code int, stdout, stderr string) (devexec.State, error) { + var state devexec.State + decodeErr := json.Unmarshal([]byte(stdout), &state) + if decodeErr == nil && state.ExecID == identity.ExecID && state.CreationKey == identity.CreationKey && state.ClaimToken == identity.ClaimToken { + if state.Error != "" { + if state.Exited && state.ExitCode != nil && (*state.ExitCode == 126 || *state.ExitCode == 127) { + if action == "start" { + return state, &ExecExitError{Code: *state.ExitCode} + } + return state, nil + } + return state, errors.New(state.Error) + } + if code == 0 { + return state, nil + } + } + failed := fmt.Errorf("%w: helper %s failed (status %d): %s", ErrExecUnavailable, action, code, strings.TrimSpace(stderr)) + if strings.Contains(stderr, devexec.ErrNoSupervisor.Error()) { + return state, fmt.Errorf("%w: %w", devexec.ErrNoSupervisor, failed) + } + return state, failed +} diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 911304faa..8c8d6c533 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -52,6 +52,28 @@ type InitPIDRuntime interface { InitPID(context.Context, ID) (int, error) } +// MemoryEventsRuntime optionally reports out-of-memory kills inside a +// container whose own kernel does the killing, where its host cgroup cannot +// see them. ok is false for a container its host cgroup accounts for. +type MemoryEventsRuntime interface { + MemoryEvents(context.Context, ID) (events MemoryEvents, ok bool, err error) +} + +// MemoryEvents counts kills since the container started; LimitBytes is 0 +// when unknown. +type MemoryEvents struct { + OOMKills uint64 + LimitBytes uint64 +} + +// SaveRuntime optionally writes a running container's memory and device +// state to disk and ends it, then later restores it with every process +// where it was. A container that was saved and not restored is stopped. +type SaveRuntime interface { + Save(context.Context, ID) error + Restore(context.Context, ID) error +} + // Mount is one additional mount into a run container. Host paths are // validated against Aether-owned roots by the mount validator (see // mounts.go) before a spec reaches any runtime; the runtime itself only diff --git a/internal/runtime/session.go b/internal/runtime/session.go new file mode 100644 index 000000000..8f60140be --- /dev/null +++ b/internal/runtime/session.go @@ -0,0 +1,149 @@ +package runtime + +import ( + "encoding/json" + "maps" + "slices" + "strconv" + "strings" +) + +// ExecOutputLimit covers stdout and stderr together. +const ExecOutputLimit = 1 << 20 + +// setupOutputLimit is how much of a setup script's output is kept, from the +// end, where the failure is. +const setupOutputLimit = 16 << 10 + +// RootPath is sudo's secure_path. A root exec never searches the +// container's PATH, which puts the agent-writable ~/.local/bin first. +const RootPath = "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +// UserScript gives uid $1 a passwd entry whose home is $2: git refuses to +// commit and ssh and ssh-keygen refuse to run for a uid with none, and ssh +// reads ~/.ssh from the passwd home, never $HOME. With $3 set it also lets +// the uid sudo without a password; the sudoers file is checked before it +// goes live, since sudo refuses to run at all with a broken one. +// +// The home at /root is the member's, shared by every container they run, +// and systemd's postinst runs systemd-tmpfiles, whose provision.conf hands +// /root to root: installing any package that pulls in systemd would take +// the home from all of them. The /etc entry masks it. +const UserScript = `uid=${1%%:*} gid=${1#*:} +if grep -q "^[^:]*:[^:]*:$uid:" /etc/passwd; then + awk -F: -v OFS=: -v u="$uid" -v h="$2" '$3 == u { $6 = h } { print }' /etc/passwd > /etc/passwd.aether && cat /etc/passwd.aether > /etc/passwd && rm -f /etc/passwd.aether || exit 1 +else + echo "aether:x:$uid:$gid::$2:/bin/sh" >> /etc/passwd || exit 1 +fi +grep -q "^[^:]*:[^:]*:$gid:" /etc/group || echo "aether:x:$gid:" >> /etc/group || exit 1 +mkdir -p /etc/tmpfiles.d && ln -sfn /dev/null /etc/tmpfiles.d/provision.conf || exit 1 +[ -n "$3" ] || exit 0 +mkdir -p /etc/sudoers.d +printf '#%s ALL=(ALL:ALL) NOPASSWD: ALL\n' "$uid" > /etc/sudoers.d/.aether.new +chmod 0440 /etc/sudoers.d/.aether.new +if command -v visudo >/dev/null 2>&1 && ! visudo -cqf /etc/sudoers.d/.aether.new; then rm -f /etc/sudoers.d/.aether.new; exit 1; fi +mv -f /etc/sudoers.d/.aether.new /etc/sudoers.d/aether` + +func Unprivileged(user string) bool { + uid, _, _ := strings.Cut(user, ":") + return uid != "" && uid != "0" +} + +func EnvList(env map[string]string) []string { + if len(env) == 0 { + return nil + } + out := make([]string, 0, len(env)) + for _, k := range slices.Sorted(maps.Keys(env)) { + out = append(out, k+"="+env[k]) + } + return out +} + +func EnvValue(env []string, name string) string { + for _, pair := range env { + if key, value, _ := strings.Cut(pair, "="); key == name { + return value + } + } + return "" +} + +func TrustCheckouts(env map[string]string, checkout string, extras []Mount) map[string]string { + env = trustCheckout(env, checkout) + for _, m := range extras { + env = trustCheckout(env, m.ContainerPath) + } + return env +} + +// Execs inherit the container env, so this also covers every git the server +// runs inside the container. +func trustCheckout(env map[string]string, checkout string) map[string]string { + if checkout == "" { + return env + } + n := 0 + if count, ok := env["GIT_CONFIG_COUNT"]; ok { + parsed, err := strconv.Atoi(count) + if err != nil || parsed < 0 { + return env + } + n = parsed + } + out := make(map[string]string, len(env)+3) + maps.Copy(out, env) + out["GIT_CONFIG_KEY_"+strconv.Itoa(n)] = "safe.directory" + out["GIT_CONFIG_VALUE_"+strconv.Itoa(n)] = checkout + out["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1) + return out +} + +func secretEnvValues(label string, env []string) []string { + var names []string + if json.Unmarshal([]byte(label), &names) != nil { + return nil + } + return SecretValues(names, env) +} + +func SecretValues(names, env []string) []string { + var values []string + for _, pair := range env { + name, value, _ := strings.Cut(pair, "=") + if value != "" && slices.Contains(names, name) { + values = append(values, value) + } + } + return values +} + +const secretMask = "***" + +// SetupOutput masks exact secret values, longest first so a secret holding +// another is masked whole, then keeps the last setupOutputLimit bytes. cut +// says output already lost its beginning, which may have split a secret: the +// span that could hold its remainder is dropped. That span is measured after +// masking, where a shorter secret inside the remainder can have grown it to +// len(secretMask) times its length. +func SetupOutput(output string, cut bool, secrets []string) string { + if len(secrets) > 0 { + slices.SortFunc(secrets, func(a, b string) int { return len(b) - len(a) }) + pairs := make([]string, 0, 2*len(secrets)) + for _, secret := range secrets { + pairs = append(pairs, secret, secretMask) + } + output = strings.NewReplacer(pairs...).Replace(output) + if cut { + output = output[min(len(secretMask)*len(secrets[0]), len(output)):] + } + } + if cut || len(output) > setupOutputLimit { + output = output[max(0, len(output)-setupOutputLimit):] + if i := strings.IndexByte(output, '\n'); i >= 0 { + output = output[i+1:] + } + output = "[earlier output omitted]\n" + output + } + return strings.ToValidUTF8(output, "�") +} diff --git a/internal/runtime/stream.go b/internal/runtime/stream.go new file mode 100644 index 000000000..056b8f6bd --- /dev/null +++ b/internal/runtime/stream.go @@ -0,0 +1,88 @@ +package runtime + +import ( + "bytes" + "io" + "sync" +) + +const maxStreamBuffer = 8 << 20 + +// StreamBuffer writes never block, since blocking the demux goroutine would +// stall the sibling stream; past maxStreamBuffer the oldest bytes are +// dropped. A lossless buffer blocks at the cap instead, for protocol streams +// that cannot lose a byte. +type StreamBuffer struct { + mu sync.Mutex + cond *sync.Cond + buf bytes.Buffer + lossless bool + // limit is where a lossless buffer blocks its writer. + limit int + closed bool + err error +} + +func NewStreamBuffer() *StreamBuffer { + b := &StreamBuffer{limit: maxStreamBuffer} + b.cond = sync.NewCond(&b.mu) + return b +} + +func newLosslessStreamBuffer() *StreamBuffer { + b := NewStreamBuffer() + b.lossless = true + return b +} + +func (b *StreamBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + for b.lossless && !b.closed && b.buf.Len() >= b.limit { + b.cond.Wait() + } + if b.closed { + return 0, io.ErrClosedPipe + } + n, _ := b.buf.Write(p) // bytes.Buffer.Write cannot fail + if b.lossless { + b.cond.Broadcast() + return n, nil + } + if over := b.buf.Len() - maxStreamBuffer; over > 0 { + b.buf.Next(over) + } + b.cond.Broadcast() + return n, nil +} + +func (b *StreamBuffer) Read(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + for b.buf.Len() == 0 && !b.closed { + b.cond.Wait() + } + if b.buf.Len() > 0 { + if b.lossless { + b.cond.Broadcast() + } + return b.buf.Read(p) + } + if b.err != nil { + return 0, b.err + } + return 0, io.EOF +} + +// CloseWithError leaves buffered data readable first. Only the first close +// takes effect. +func (b *StreamBuffer) CloseWithError(err error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } + b.closed = true + b.err = err + b.cond.Broadcast() +} diff --git a/internal/runtime/vm/chapi.go b/internal/runtime/vm/chapi.go new file mode 100644 index 000000000..cbfa609fe --- /dev/null +++ b/internal/runtime/vm/chapi.go @@ -0,0 +1,98 @@ +package vm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" +) + +type chAPI struct { + client *http.Client +} + +func newCHAPI(socket string) chAPI { + return chAPI{client: &http.Client{Transport: &http.Transport{ + DisableKeepAlives: true, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socket) + }, + }}} +} + +// state is vm.info's state: Created, Running, Paused or Shutdown. +func (c chAPI) state(ctx context.Context) (string, error) { + var info struct { + State string `json:"state"` + } + if err := c.call(ctx, http.MethodGet, "vm.info", nil, &info); err != nil { + return "", err + } + return info.State, nil +} + +func (c chAPI) pause(ctx context.Context) error { + return c.call(ctx, http.MethodPut, "vm.pause", nil, nil) +} + +func (c chAPI) resume(ctx context.Context) error { + return c.call(ctx, http.MethodPut, "vm.resume", nil, nil) +} + +func (c chAPI) snapshot(ctx context.Context, dir string) error { + return c.call(ctx, http.MethodPut, "vm.snapshot", map[string]string{"destination_url": "file://" + dir}, nil) +} + +// shutdownVMM ends the monitor process; the guest gets no warning. +func (c chAPI) shutdownVMM(ctx context.Context) error { + return c.call(ctx, http.MethodPut, "vmm.shutdown", nil, nil) +} + +// counters maps each device ID to its counters, such as a disk's read_bytes +// or a net device's rx_bytes. +func (c chAPI) counters(ctx context.Context) (map[string]map[string]uint64, error) { + var out map[string]map[string]uint64 + return out, c.call(ctx, http.MethodGet, "vm.counters", nil, &out) +} + +func (c chAPI) call(ctx context.Context, method, endpoint string, body, out any) error { + var payload io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("cloud-hypervisor %s: %w", endpoint, err) + } + payload = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, "http://localhost/api/v1/"+endpoint, payload) + if err != nil { + return fmt.Errorf("cloud-hypervisor %s: %w", endpoint, err) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("cloud-hypervisor %s: %w", endpoint, err) + } + defer func() { _ = resp.Body.Close() }() + data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return fmt.Errorf("cloud-hypervisor %s: %w", endpoint, err) + } + if resp.StatusCode/100 != 2 { + return fmt.Errorf("cloud-hypervisor %s: %s: %s", endpoint, resp.Status, strings.TrimSpace(string(data))) + } + if out == nil || len(data) == 0 { + return nil + } + if err := json.Unmarshal(data, out); err != nil { + return fmt.Errorf("cloud-hypervisor %s: decode reply: %w", endpoint, err) + } + return nil +} diff --git a/internal/runtime/vm/engine.go b/internal/runtime/vm/engine.go new file mode 100644 index 000000000..94d8eed61 --- /dev/null +++ b/internal/runtime/vm/engine.go @@ -0,0 +1,450 @@ +package vm + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + goruntime "runtime" + "strings" + "sync" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +type Images interface { + Kernel() (string, error) + HelperDisk(ctx context.Context, binary, path string) (vmimage.Disk, error) + BaseDisk(ctx context.Context, ref string) (vmimage.Base, error) + NewRootDisk(ctx context.Context, base vmimage.Base, path string) (vmimage.Disk, error) + NewDataDisk(ctx context.Context, path string, size int64) (vmimage.Disk, error) +} + +const ( + // dataDiskBytes is sparse: it costs only what the guest writes. + dataDiskBytes = 128 << 30 + minMemoryMiB = 256 + bootTimeout = 90 * time.Second + agentOOMScore = 500 + // cmdlineBase boots the read-only helper disk, whose /init is the helper. + // Page reporting order 0 lets freed guest page cache back to the host; + // at the default order it stalls with most of it still held. + cmdlineBase = "root=/dev/vda ro rootfstype=ext4 init=/init console=ttyS0 quiet page_reporting.page_reporting_order=0" +) + +type engine struct { + cfg Config + root string + units units + dial func(vsockPath string) (helper, error) + run func(ctx context.Context, name string, args ...string) error + + mu sync.Mutex + locks map[string]*sync.Mutex + helpers map[string]helper + booting map[string]bool + fresh map[string]bool + events map[string]runtime.MemoryEvents + + netMu sync.Mutex + reserved map[int]bool + natReady bool +} + +func newEngine(cfg Config, u units, dial func(string) (helper, error)) *engine { + return &engine{ + cfg: cfg, root: filepath.Join(cfg.DataDir, "vm"), units: u, dial: dial, run: runCommand, + locks: map[string]*sync.Mutex{}, helpers: map[string]helper{}, booting: map[string]bool{}, + fresh: map[string]bool{}, events: map[string]runtime.MemoryEvents{}, reserved: map[int]bool{}, + } +} + +func runCommand(ctx context.Context, name string, args ...string) error { + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if err != nil { + return fmt.Errorf("vm runtime: %s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) + } + return nil +} + +func (e *engine) lock(key string) func() { + e.mu.Lock() + l := e.locks[key] + if l == nil { + l = &sync.Mutex{} + e.locks[key] = l + } + e.mu.Unlock() + l.Lock() + return l.Unlock +} + +// helper returns the key's client, dialing once per VM lifetime; the client +// redials by itself across a save and restore. +func (e *engine) helper(key string) (helper, error) { + e.mu.Lock() + defer e.mu.Unlock() + if h := e.helpers[key]; h != nil { + return h, nil + } + h, err := e.dial(e.path(key, vsockSocket)) + if err != nil { + return nil, fmt.Errorf("vm runtime: connect to the helper of VM %s: %w", key, err) + } + e.helpers[key] = h + return h, nil +} + +func (e *engine) forgetHelper(key string) { + e.mu.Lock() + h := e.helpers[key] + delete(e.helpers, key) + e.mu.Unlock() + if h != nil { + _ = h.Close() + } +} + +func (e *engine) setFlag(m map[string]bool, key string, on bool) { + e.mu.Lock() + defer e.mu.Unlock() + if on { + m[key] = true + } else { + delete(m, key) + } +} + +func (e *engine) flag(m map[string]bool, key string) bool { + e.mu.Lock() + defer e.mu.Unlock() + return m[key] +} + +func (e *engine) create(ctx context.Context, spec runtime.Spec) (_ runtime.ID, err error) { + if err = spec.Validate(); err != nil { + return "", err + } + key := spec.CreationKey + if err = checkKey(key); err != nil { + return "", err + } + if spec.Egress { + return "", errors.New("vm runtime: egress control (the Isolated tier) is not supported in session VMs until work item 5") + } + for _, m := range spec.Mounts { + if m.Overlay != "" { + return "", fmt.Errorf("vm runtime: the overlay mount at %s (the Isolated tier) is not supported in session VMs until work item 7", m.ContainerPath) + } + } + memMiB, err := memoryMiB(spec.MemoryLimitBytes) + if err != nil { + return "", err + } + unlock := e.lock(key) + defer unlock() + if _, err = os.Stat(e.dir(key)); err == nil { + return "", fmt.Errorf("vm runtime: a VM named %s already exists", key) + } + base, err := e.cfg.Images.BaseDisk(ctx, spec.Image) + if err != nil { + return "", fmt.Errorf("vm runtime: base disk for %s: %w", spec.Image, err) + } + index, err := e.allocateNet() + if err != nil { + return "", err + } + defer e.releaseNet(index) + if err = os.MkdirAll(e.dir(key), 0o700); err != nil { + return "", fmt.Errorf("vm runtime: %w", err) + } + defer func() { + if err != nil { + _ = os.RemoveAll(e.dir(key)) + } + }() + root, err := e.cfg.Images.NewRootDisk(ctx, base, e.path(key, rootDisk)) + if err != nil { + return "", fmt.Errorf("vm runtime: root disk: %w", err) + } + data, err := e.cfg.Images.NewDataDisk(ctx, e.path(key, dataDisk), dataDiskBytes) + if err != nil { + return "", fmt.Errorf("vm runtime: data disk: %w", err) + } + cpus, limit := vcpus(spec.CPULimit) + rec := &record{ + Key: key, Fingerprint: spec.Fingerprint, Image: spec.Image, Base: base, Root: root, Data: data, + User: spec.User, + Env: overrideEnv(base.Config.Env, runtime.EnvList(runtime.TrustCheckouts(spec.Env, spec.WorktreeMountPath, spec.ExtraWorktrees))), + WorkingDir: cmp.Or(spec.WorkingDir, base.Config.WorkingDir, "/"), Command: spec.Command, TTY: spec.TTY, + SetupScript: spec.SetupScript, SecretEnv: spec.SecretEnv, NestedDocker: spec.NestedDocker, Sudo: spec.Sudo, + CPUs: cpus, CPULimit: limit, MemoryMiB: memMiB, PidsLimit: spec.PidsLimit, Mounts: specMounts(spec), Net: index, + } + if err = e.store(rec); err != nil { + return "", err + } + return runtime.ID(idPrefix + key), nil +} + +// vcpus rounds a fractional CPU limit up to whole vCPUs and keeps the limit +// itself as the unit's CPU quota; no limit means every host CPU. +func vcpus(limit float64) (int, float64) { + host := goruntime.NumCPU() + if limit <= 0 { + return host, 0 + } + return min(host, max(1, int(math.Ceil(limit)))), limit +} + +func memoryMiB(limit int64) (int64, error) { + if limit <= 0 { + total, err := hostMemoryBytes() + if err != nil { + return 0, err + } + return total >> 20, nil + } + mib := (limit + (1<<20 - 1)) >> 20 + if mib < minMemoryMiB { + return 0, fmt.Errorf("vm runtime: a memory limit of %d bytes is below the %d MiB a session VM needs to boot", limit, minMemoryMiB) + } + return mib, nil +} + +func hostMemoryBytes() (int64, error) { + var kb int64 + data, err := os.ReadFile("/proc/meminfo") + if err != nil { + return 0, fmt.Errorf("vm runtime: host memory: %w", err) + } + for line := range strings.SplitSeq(string(data), "\n") { + if rest, ok := strings.CutPrefix(line, "MemTotal:"); ok { + if _, err := fmt.Sscanf(strings.TrimSpace(rest), "%d kB", &kb); err != nil { + return 0, fmt.Errorf("vm runtime: parse MemTotal: %w", err) + } + return kb << 10, nil + } + } + return 0, errors.New("vm runtime: /proc/meminfo has no MemTotal") +} + +// start cold-starts a stopped VM; on a running one it only makes sure Main +// was started, as a redundant docker start does. +func (e *engine) start(ctx context.Context, key string) (err error) { + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return err + } + st, err := e.units.State(ctx, unitName(key)) + if err != nil { + return err + } + if st.Active { + return e.boot(ctx, rec) + } + if rec.Saved { + if err = os.RemoveAll(e.path(key, snapshotDir)); err != nil { + return fmt.Errorf("vm runtime: discard saved state: %w", err) + } + rec.Saved = false + } + if rec.Booted { + if rec.Root, err = e.cfg.Images.NewRootDisk(ctx, rec.Base, e.path(key, rootDisk)); err != nil { + return fmt.Errorf("vm runtime: root disk: %w", err) + } + } + l, err := e.bootLaunch(ctx, rec) + if err != nil { + return err + } + if err = e.ensureNAT(ctx); err != nil { + return err + } + rec.Booted, rec.Exited = true, false + rec.Boots++ + if err = e.store(rec); err != nil { + return err + } + if err = e.launch(ctx, rec, l); err != nil { + return err + } + defer func() { + if err != nil { + e.kill(context.WithoutCancel(ctx), key) + } + }() + if err = e.boot(ctx, rec); err != nil { + return err + } + e.setFlag(e.fresh, key, true) + return nil +} + +func (e *engine) bootLaunch(ctx context.Context, rec *record) (launch, error) { + p, err := planMounts(rec.Mounts, rec.NestedDocker, e.path(rec.Key, vsockSocket)) + if err != nil { + return launch{}, err + } + if err = linkSockets(e.dir(rec.Key), p.links); err != nil { + return launch{}, err + } + kernel, err := e.cfg.Images.Kernel() + if err != nil { + return launch{}, fmt.Errorf("vm runtime: guest kernel: %w", err) + } + helperDisk, err := e.cfg.Images.HelperDisk(ctx, e.cfg.Helper, e.path(rec.Key, helperImage)) + if err != nil { + return launch{}, fmt.Errorf("vm runtime: helper disk: %w", err) + } + cmdline, err := guest.Config{ + Hostname: rec.Key, Network: guestNetwork(rec.Net, hostDNS()), Mounts: p.mounts, Sockets: p.sockets, + }.KernelArgs(cmdlineBase) + if err != nil { + return launch{}, fmt.Errorf("vm runtime: kernel command line: %w", err) + } + args := []string{ + "--api-socket", "path=" + e.path(rec.Key, apiSocket), + "--vsock", "cid=3,socket=" + e.path(rec.Key, vsockSocket), + "--serial", "file=" + e.path(rec.Key, consoleLog), "--console", "off", + "--kernel", kernel, "--cmdline", cmdline, + "--disk", diskArg(helperDisk, "helper") + ",readonly=on", diskArg(rec.Root, "root"), diskArg(rec.Data, "data"), + "--cpus", fmt.Sprintf("boot=%d", rec.CPUs), + "--memory", fmt.Sprintf("size=%dM,shared=on", rec.MemoryMiB), + "--balloon", "size=0,free_page_reporting=on", + "--net", "id=net0," + chNetArg(rec.Net), + } + l := launch{CloudHypervisor: e.cfg.CloudHypervisor, Args: args, Shares: p.shares} + if len(p.shares) > 0 { + l.Args = append(l.Args, "--fs", "tag="+guest.ShareTag+",socket="+e.path(rec.Key, fsSocket)) + l.Virtiofsd = e.cfg.Virtiofsd + } + return l, nil +} + +func diskArg(d vmimage.Disk, id string) string { + arg := fmt.Sprintf("path=%s,image_type=%s,id=%s,serial=%s", d.Path, d.Format, id, id) + if d.Format == vmimage.FormatQcow2 { + arg += ",backing_files=on" + } + return arg +} + +func (e *engine) launch(ctx context.Context, rec *record, l launch) error { + data, err := json.MarshalIndent(l, "", " ") + if err != nil { + return fmt.Errorf("vm runtime: encode launch: %w", err) + } + if err := writeFileAtomic(e.path(rec.Key, launchFile), data); err != nil { + return err + } + logPath := e.path(rec.Key, monitorLog) + properties := []string{ + "Type=exec", "PrivateMounts=yes", "KillMode=control-group", "TimeoutStopSec=15", + fmt.Sprintf("OOMScoreAdjust=%d", agentOOMScore), + "StandardOutput=append:" + logPath, "StandardError=append:" + logPath, + } + if rec.CPULimit > 0 { + properties = append(properties, fmt.Sprintf("CPUQuota=%d%%", int(math.Ceil(rec.CPULimit*100)))) + } + argv := append(append([]string{}, e.cfg.Monitor...), e.dir(rec.Key)) + if err := e.units.Start(ctx, unitName(rec.Key), argv, properties); err != nil { + return err + } + return nil +} + +func (e *engine) boot(ctx context.Context, rec *record) error { + h, err := e.helper(rec.Key) + if err != nil { + return err + } + if err = e.waitHelper(ctx, rec.Key, h); err != nil { + return err + } + if runtime.Unprivileged(rec.User) { + allow := "" + if rec.Sudo { + allow = "sudo" + } + home := cmp.Or(runtime.EnvValue(rec.Env, "HOME"), "/") + code, out, prepErr := e.runAsRoot(ctx, h, []string{"/bin/sh", "-c", runtime.UserScript, "aether-user", rec.User, home, allow}) + if prepErr != nil { + return fmt.Errorf("vm runtime: prepare user %s: %w", rec.User, prepErr) + } + if code != 0 { + return fmt.Errorf("vm runtime: prepare user %s exited %d: %s", rec.User, code, strings.TrimSpace(out)) + } + } + var tty *guest.Size + if rec.TTY { + tty = &guest.Size{Cols: 80, Rows: 24} + } + e.setFlag(e.booting, rec.Key, true) + err = h.Boot(ctx, guest.BootSpec{ + Main: guest.ProcessSpec{ID: guest.MainID, Argv: rec.Command, Env: rec.Env, User: rec.User, Dir: rec.WorkingDir, TTY: tty}, + SetupScript: rec.SetupScript, NestedDocker: rec.NestedDocker, PidsLimit: rec.PidsLimit, + }) + e.setFlag(e.booting, rec.Key, false) + var setup *guest.SetupError + if errors.As(err, &setup) { + return &runtime.SetupError{ExitCode: setup.ExitCode, Output: runtime.SetupOutput(setup.Output, setup.Cut, runtime.SecretValues(rec.SecretEnv, rec.Env))} + } + if err != nil { + return fmt.Errorf("vm runtime: boot VM %s: %w", rec.Key, err) + } + return nil +} + +func (e *engine) waitHelper(ctx context.Context, key string, h helper) error { + ctx, cancel := context.WithTimeout(ctx, bootTimeout) + defer cancel() + answered := make(chan error, 1) + var info guest.Info + go func() { + var err error + info, err = h.Ping(ctx) + answered <- err + }() + tick := time.NewTicker(250 * time.Millisecond) + defer tick.Stop() + for { + select { + case err := <-answered: + if err != nil { + return fmt.Errorf("vm runtime: helper of VM %s did not answer: %w%s", key, err, e.logTail(key)) + } + if info.BootError != "" { + return fmt.Errorf("vm runtime: VM %s failed to boot: %s", key, info.BootError) + } + return nil + case <-tick.C: + if st, err := e.units.State(ctx, unitName(key)); err == nil && !st.Active { + cancel() + return fmt.Errorf("vm runtime: VM %s stopped while booting%s", key, e.logTail(key)) + } + } + } +} + +func (e *engine) logTail(key string) string { + var out strings.Builder + for _, name := range []string{monitorLog, consoleLog} { + data, err := os.ReadFile(e.path(key, name)) + if err != nil || len(data) == 0 { + continue + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + fmt.Fprintf(&out, "\n%s:\n%s", name, strings.Join(lines[max(0, len(lines)-15):], "\n")) + } + return out.String() +} diff --git a/internal/runtime/vm/engine_test.go b/internal/runtime/vm/engine_test.go new file mode 100644 index 000000000..e6929f76d --- /dev/null +++ b/internal/runtime/vm/engine_test.go @@ -0,0 +1,458 @@ +package vm + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "testing" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +func (h *harness) createStarted(key string, spec runtime.Spec) runtime.ID { + h.t.Helper() + id, err := h.engine.create(h.t.Context(), spec) + if err != nil { + h.t.Fatalf("create: %v", err) + } + if err := h.engine.start(h.t.Context(), key); err != nil { + h.t.Fatalf("start: %v", err) + } + return id +} + +func TestCreateRecordsWhatAContainerWouldReport(t *testing.T) { + h := newHarness(t) + spec := sessionSpec("run-a") + spec.WorktreeHostPath, spec.WorktreeMountPath = t.TempDir(), "/workspace" + spec.User = "1000:1000" + id, err := h.engine.create(t.Context(), spec) + if err != nil || id != "vm-run-a" { + t.Fatalf("create = %q, %v", id, err) + } + info, err := h.engine.inspect(t.Context(), "run-a") + if err != nil { + t.Fatal(err) + } + if info.Image != "aether/standard:1" || info.User != "1000:1000" || info.Fingerprint != "fp-1" || info.Exited { + t.Fatalf("inspect = %+v", info) + } + for _, want := range []string{"HOME=/root", "PATH=/usr/bin:/bin", "GIT_CONFIG_KEY_0=safe.directory", "GIT_CONFIG_VALUE_0=/workspace"} { + if !slices.Contains(info.Env, want) { + t.Errorf("env %v lacks %s", info.Env, want) + } + } + if !slices.Equal(info.MountTargets, []string{"/workspace"}) { + t.Errorf("mount targets = %v", info.MountTargets) + } + rec, _ := h.engine.load("run-a") + if rec.CPUs != 2 || rec.CPULimit != 1.5 || rec.MemoryMiB != 2048 || rec.WorkingDir != "/srv" { + t.Errorf("record = %+v", rec) + } + if _, err := h.engine.create(t.Context(), spec); err == nil || !strings.Contains(err.Error(), "already exists") { + t.Errorf("second create = %v", err) + } + if found, err := h.engine.find("run-a"); err != nil || found != id { + t.Errorf("find = %q, %v", found, err) + } + if _, err := h.engine.find("run-b"); !errors.Is(err, runtime.ErrNotFound) { + t.Errorf("find missing = %v", err) + } +} + +func TestCreateRefusesWhatLaterItemsAdd(t *testing.T) { + h := newHarness(t) + egress := sessionSpec("run-e") + egress.Egress = true + if _, err := h.engine.create(t.Context(), egress); err == nil || !strings.Contains(err.Error(), "work item 5") { + t.Errorf("egress create = %v", err) + } + overlay := sessionSpec("run-o") + overlay.Mounts = []runtime.Mount{{HostPath: "/h", ContainerPath: "/root", Overlay: "/scratch"}} + if _, err := h.engine.create(t.Context(), overlay); err == nil || !strings.Contains(err.Error(), "work item 7") { + t.Errorf("overlay create = %v", err) + } + missing := sessionSpec("run-m") + missing.Image = "missing:1" + if _, err := h.engine.create(t.Context(), missing); err == nil || !strings.Contains(err.Error(), "manifest unknown") { + t.Errorf("missing image create = %v", err) + } + if _, err := os.Stat(h.engine.dir("run-m")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("failed create left %s: %v", h.engine.dir("run-m"), err) + } +} + +func TestStartBootsTheVMInItsUnit(t *testing.T) { + h := newHarness(t) + home := t.TempDir() + spec := sessionSpec("run-a") + spec.User, spec.Sudo, spec.NestedDocker = "1000:1000", true, true + spec.SetupScript = "make deps" + spec.Mounts = []runtime.Mount{{HostPath: home, ContainerPath: "/root"}} + h.createStarted("run-a", spec) + + call := h.units.last() + if call.name != "aether-vm-run-a.service" || call.argv[1] != MonitorCommand || call.argv[2] != h.engine.dir("run-a") { + t.Fatalf("unit start = %+v", call) + } + for _, want := range []string{"PrivateMounts=yes", "CPUQuota=150%", "OOMScoreAdjust=500"} { + if !slices.Contains(call.properties, want) { + t.Errorf("properties %v lack %s", call.properties, want) + } + } + args := strings.Join(call.launch.Args, " ") + for _, want := range []string{ + "--cpus boot=2", "--memory size=2048M,shared=on", "--balloon size=0,free_page_reporting=on", + "--kernel /k/vmlinux", "page_reporting.page_reporting_order=0", "--fs tag=aether,socket=", + "helper.img,image_type=raw,id=helper,serial=helper,readonly=on", "root.img,image_type=qcow2,id=root,serial=root,backing_files=on", + "--net id=net0,tap=aethervm0,ip=10.200.0.1,mask=255.255.255.252", + } { + if !strings.Contains(args, want) { + t.Errorf("launch args lack %q:\n%s", want, args) + } + } + if len(call.launch.Shares) != 1 || call.launch.Shares[0].Source != home || call.launch.Virtiofsd != "virtiofsd" { + t.Errorf("shares = %+v", call.launch) + } + g := h.guest("run-a") + if len(g.boots) != 1 { + t.Fatalf("boots = %d", len(g.boots)) + } + boot := g.boots[0] + if boot.SetupScript != "make deps" || !boot.NestedDocker || boot.PidsLimit != 4096 || boot.Main.ID != guest.MainID || + boot.Main.User != "1000:1000" || boot.Main.Dir != "/srv" || boot.Main.TTY == nil || !slices.Equal(boot.Main.Argv, spec.Command) { + t.Errorf("boot = %+v", boot) + } + var prepared bool + for _, p := range g.spawned() { + if p.User == "0:0" && slices.Contains(p.Argv, runtime.UserScript) { + prepared = slices.Equal(p.Argv[len(p.Argv)-3:], []string{"1000:1000", "/root", "sudo"}) + } + } + if !prepared { + t.Errorf("no root user prep among %+v", g.spawned()) + } + if !h.engine.flag(h.engine.fresh, "run-a") || h.engine.settingUp("run-a") { + t.Errorf("fresh/settingUp after start") + } + if err := h.engine.start(t.Context(), "run-a"); err != nil || len(h.units.started) != 1 || len(g.boots) != 2 { + t.Errorf("redundant start: %v, %d unit starts, %d boots", err, len(h.units.started), len(g.boots)) + } +} + +func TestStartReportsSetupFailureMaskedAndKillsTheVM(t *testing.T) { + h := newHarness(t) + spec := sessionSpec("run-a") + spec.SecretEnv = []string{"TOKEN"} + if _, err := h.engine.create(t.Context(), spec); err != nil { + t.Fatal(err) + } + h.guest("run-a").bootErr = &guest.SetupError{ExitCode: 3, Output: "using s3cret\nfailed\n"} + err := h.engine.start(t.Context(), "run-a") + var setup *runtime.SetupError + if !errors.As(err, &setup) || setup.ExitCode != 3 || strings.Contains(setup.Output, "s3cret") || !strings.Contains(setup.Output, "using ***") { + t.Fatalf("start = %v", err) + } + if h.units.vmm("run-a") != nil { + t.Errorf("a failed start left the VM running") + } +} + +func TestStartFailsWithTheLogWhenTheVMDiesWhileBooting(t *testing.T) { + h := newHarness(t) + if _, err := h.engine.create(t.Context(), sessionSpec("run-a")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(h.engine.path("run-a", monitorLog), []byte("cloud-hypervisor: Error booting VM: KvmNotAvailable\n"), 0o600); err != nil { + t.Fatal(err) + } + h.units.failNext = true + h.guest("run-a").pingErr = errors.New("connection refused") + err := h.engine.start(t.Context(), "run-a") + if err == nil || !strings.Contains(err.Error(), "stopped while booting") || !strings.Contains(err.Error(), "KvmNotAvailable") { + t.Fatalf("start = %v", err) + } +} + +func TestStopEndsMainRecordsItsExitAndPowersOff(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + if err := h.engine.pause(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if _, _, _, err := h.engine.exec(t.Context(), "run-a", []string{"true"}, ""); err == nil || !strings.Contains(err.Error(), "paused") { + t.Errorf("exec in a paused VM = %v", err) + } + vmm := h.units.vmm("run-a") + if err := h.engine.stop(t.Context(), "run-a", time.Second); err != nil { + t.Fatal(err) + } + if !slices.Contains(vmm.called(), "vm.resume") { + t.Errorf("stop did not resume the paused VM first: %v", vmm.called()) + } + if h.units.vmm("run-a") != nil { + t.Errorf("VM still running after stop") + } + info, err := h.engine.inspect(t.Context(), "run-a") + if err != nil || !info.Exited || info.ExitCode != 128+int(syscall.SIGTERM) { + t.Fatalf("inspect = %+v, %v", info, err) + } + if st, err := h.engine.wait(t.Context(), "run-a"); err != nil || st.Code != info.ExitCode { + t.Errorf("wait = %+v, %v", st, err) + } + if err := h.engine.stop(t.Context(), "run-a", 0); err != nil { + t.Errorf("stop of a stopped VM = %v", err) + } + if err := h.engine.start(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if h.images.roots() != 2 { + t.Errorf("cold start used %d root disks, want a fresh one", h.images.roots()) + } + if info, _ := h.engine.inspect(t.Context(), "run-a"); info.Exited { + t.Errorf("restarted VM still reports exited") + } +} + +func TestStopKillsMainThatIgnoresTerm(t *testing.T) { + h := newHarness(t) + h.guest("run-a").ignoreTE = true + h.createStarted("run-a", sessionSpec("run-a")) + if err := h.engine.stop(t.Context(), "run-a", 50*time.Millisecond); err != nil { + t.Fatal(err) + } + if info, _ := h.engine.inspect(t.Context(), "run-a"); info.ExitCode != 128+int(syscall.SIGKILL) { + t.Errorf("exit code = %d, want SIGKILL's", info.ExitCode) + } +} + +func TestWaitReportsMainsExitAndALostVM(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + main := h.guest("run-a").procs[guest.MainID] + go func() { + time.Sleep(50 * time.Millisecond) + main.exit(7) + }() + st, err := h.engine.wait(t.Context(), "run-a") + if err != nil || st.Code != 7 { + t.Fatalf("wait = %+v, %v", st, err) + } + if h.units.vmm("run-a") != nil { + t.Errorf("VM still running after Main exited") + } + + h.createStarted("run-b", sessionSpec("run-b")) + _ = h.units.Stop(t.Context(), unitName("run-b")) + if st, err := h.engine.wait(t.Context(), "run-b"); err != nil || st.Code != lostExitCode { + t.Fatalf("wait on a lost VM = %+v, %v", st, err) + } +} + +func TestAnotherEngineFindsAndDrivesARunningVM(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + after := h.newEngine(filepath.Dir(h.engine.root)) + id, err := after.find("run-a") + if err != nil || id != "vm-run-a" { + t.Fatalf("find after restart = %q, %v", id, err) + } + info, err := after.inspect(t.Context(), "run-a") + if err != nil || info.Exited { + t.Fatalf("inspect after restart = %+v, %v", info, err) + } + if pid, pidErr := after.initPID(t.Context(), "run-a"); pidErr != nil || pid != 4242 { + t.Errorf("initPID = %d, %v", pid, pidErr) + } + if ip, ipErr := after.containerIP(t.Context(), "run-a"); ipErr != nil || ip != "10.200.0.2" { + t.Errorf("containerIP = %q, %v", ip, ipErr) + } + h.guest("run-a").output = map[string]string{"echo": "hi\n"} + code, out, _, err := after.exec(t.Context(), "run-a", []string{"echo", "hi"}, "") + if err != nil || code != 0 || out != "hi\n" { + t.Errorf("exec after restart = %d %q %v", code, out, err) + } + if err := after.stop(t.Context(), "run-a", time.Second); err != nil { + t.Fatal(err) + } + if err := after.destroy(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(after.dir("run-a")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("destroy left %s", after.dir("run-a")) + } + if err := after.destroy(t.Context(), "run-a"); err != nil { + t.Errorf("second destroy = %v", err) + } +} + +func TestExecRunsAsTheVMsUserWithItsEnvironment(t *testing.T) { + h := newHarness(t) + spec := sessionSpec("run-a") + spec.User = "1000:1000" + h.createStarted("run-a", spec) + g := h.guest("run-a") + g.output = map[string]string{"id": "uid=1000\n", "nope": "start-error:"} + if code, out, _, err := h.engine.exec(t.Context(), "run-a", []string{"id"}, "/workspace"); err != nil || code != 0 || out != "uid=1000\n" { + t.Fatalf("exec = %d %q %v", code, out, err) + } + var ran guest.ProcessSpec + for _, p := range g.spawned() { + if p.Argv[0] == "id" { + ran = p + } + } + if ran.User != "1000:1000" || ran.Dir != "/workspace" || !slices.Contains(ran.Env, "TOKEN=s3cret") || !slices.Contains(ran.Env, "PATH=/usr/bin:/bin") { + t.Errorf("exec spawned %+v", ran) + } + if code, _, _, err := h.engine.exec(t.Context(), "run-a", []string{"nope"}, ""); err != nil || code != 127 { + t.Errorf("exec of a missing command = %d, %v", code, err) + } + var exit *runtime.ExecExitError + if _, err := h.engine.execTTY(t.Context(), "run-a", []string{"nope"}, "", 80, 24); !errors.As(err, &exit) || exit.Code != 127 { + t.Errorf("execTTY of a missing command = %v", err) + } + g.output["big"] = strings.Repeat("x", runtime.ExecOutputLimit+1) + if _, _, _, err := h.engine.exec(t.Context(), "run-a", []string{"big"}, ""); err == nil || !strings.Contains(err.Error(), "byte limit") { + t.Errorf("oversized exec = %v", err) + } + if env := execEnv(&record{Env: []string{"PATH=/home/agent/.local/bin:/usr/bin"}}, "0:0", nil); !slices.Equal(env, []string{runtime.RootPath}) { + t.Errorf("root exec env = %v", env) + } +} + +func TestManagedExecGoesThroughDevExec(t *testing.T) { + h := newHarness(t) + spec := sessionSpec("run-a") + spec.User = "1000:1000" + id := h.createStarted("run-a", spec) + m, err := h.engine.startManagedExec(t.Context(), "run-a", runtime.ExecSpec{Argv: []string{"bash"}, User: "0", CreationKey: "terminal-1", Cols: 80, Rows: 24}, true) + if err != nil { + t.Fatal(err) + } + identity := m.Identity() + if identity.ContainerID != id || identity.CreationKey != "terminal-1" || identity.User != "0" { + t.Errorf("identity = %+v", identity) + } + supervisor := h.guest("run-a").procs[identity.ExecID] + if supervisor == nil || !slices.Equal(supervisor.spec.Argv[1:4], []string{"dev-exec", "run", "terminal-1"}) || + !slices.Contains(supervisor.spec.Env, "AETHER_DEVEXEC_CONTROL_GROUP=1000") { + t.Fatalf("supervisor = %+v", supervisor) + } + if st, stErr := m.Status(t.Context()); stErr != nil || !st.Running || !st.Attached { + t.Errorf("status = %+v, %v", st, stErr) + } + recovered, err := h.newEngine(filepath.Dir(h.engine.root)).recoverExec(t.Context(), "run-a", identity) + if err != nil || recovered.Attachment() != nil { + t.Fatalf("recover = %v, attachment %v", err, recovered) + } + st, err := recovered.Stop(t.Context(), time.Second) + if err != nil || st.Code != 143 { + t.Errorf("stop = %+v, %v", st, err) + } + if _, err := h.engine.recoverExec(t.Context(), "run-a", runtime.ExecIdentity{ExecID: "gone", CreationKey: "k", ClaimToken: "c"}); !errors.Is(err, runtime.ErrNotFound) { + t.Errorf("recover of an unknown exec = %v", err) + } +} + +func TestSaveRefusesSharesAndRestoresAVMWithout(t *testing.T) { + h := newHarness(t) + shared := sessionSpec("run-s") + shared.WorktreeHostPath, shared.WorktreeMountPath = t.TempDir(), "/workspace" + h.createStarted("run-s", shared) + if err := h.engine.save(t.Context(), "run-s"); err == nil || !strings.Contains(err.Error(), "/workspace") { + t.Fatalf("save with a share = %v", err) + } + if h.units.vmm("run-s") == nil { + t.Error("a refused save stopped the VM") + } + + h.createStarted("run-a", sessionSpec("run-a")) + vmm := h.units.vmm("run-a") + if err := h.engine.save(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if !slices.Equal(vmm.called(), []string{"vm.pause", "vm.snapshot", "vmm.shutdown"}) { + t.Errorf("save calls = %v", vmm.called()) + } + if _, err := os.Stat(h.engine.path("run-a", snapshotDir+"/state.json")); err != nil { + t.Errorf("no saved state: %v", err) + } + if info, _ := h.engine.inspect(t.Context(), "run-a"); info.Exited { + t.Errorf("a saved VM reports exited") + } + waited := make(chan struct{}) + go func() { + _, _ = h.engine.wait(context.WithoutCancel(t.Context()), "run-a") + close(waited) + }() + if err := h.engine.restore(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + restored := h.units.vmm("run-a") + if !restored.restored || !slices.Contains(restored.called(), "vm.resume") || h.guest("run-a").clock.IsZero() { + t.Errorf("restore: vmm %+v calls %v", restored, restored.called()) + } + if _, err := os.Stat(h.engine.path("run-a", snapshotDir)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("snapshot kept after restore: %v", err) + } + select { + case <-waited: + t.Fatal("wait returned across a save and restore") + case <-time.After(300 * time.Millisecond): + } + h.guest("run-a").procs[guest.MainID].exit(0) + <-waited + if err := h.engine.restore(t.Context(), "run-a"); err == nil || !strings.Contains(err.Error(), "no saved state") { + t.Errorf("restore without state = %v", err) + } + h.createStarted("run-b", sessionSpec("run-b")) + if err := h.engine.save(t.Context(), "run-b"); err != nil { + t.Fatal(err) + } + h.guest("run-b").pingErr = errors.New("helper gone") + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if err := h.engine.restore(ctx, "run-b"); err == nil { + t.Fatal("restore with a dead helper succeeded") + } + rec, _ := h.engine.load("run-b") + if rec.Saved || h.units.vmm("run-b") != nil { + t.Errorf("a restore that failed after resume left saved=%v, running=%v", rec.Saved, h.units.vmm("run-b") != nil) + } + if _, err := os.Stat(h.engine.path("run-b", snapshotDir)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a resumed snapshot was kept: %v", err) + } +} + +func TestStatsAndMemoryEventsComeFromTheHelper(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + h.guest("run-a").stats = guest.Stats{CPUUsageNanos: 5, MemoryBytes: 300 << 20, MemoryLimitBytes: 2 << 30, Pids: 12, PidsLimit: 4096, OOMKills: 2} + stats, err := h.engine.stats(t.Context()) + if err != nil || len(stats) != 1 { + t.Fatalf("stats = %+v, %v", stats, err) + } + st := stats[0] + if st.ID != "vm-run-a" || st.CreationKey != "run-a" || st.CPULimitCores != 1.5 || st.MemoryBytes != 300<<20 || st.Pids != 12 || + st.NetRxBytes != 10 || st.NetTxBytes != 20 || st.DiskReadBytes != 4 || st.DiskWriteBytes != 6 { + t.Errorf("stat = %+v", st) + } + events, err := h.engine.memoryEvents(t.Context(), "run-a") + if err != nil || events.OOMKills != 2 || events.LimitBytes != 2<<30 { + t.Errorf("memory events = %+v, %v", events, err) + } + if err := h.engine.stop(t.Context(), "run-a", time.Second); err != nil { + t.Fatal(err) + } + if _, err := h.engine.memoryEvents(t.Context(), "run-a"); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("memory events of a stopped VM = %v, want not-exist", err) + } +} diff --git a/internal/runtime/vm/exec.go b/internal/runtime/vm/exec.go new file mode 100644 index 000000000..63895a555 --- /dev/null +++ b/internal/runtime/vm/exec.go @@ -0,0 +1,237 @@ +package vm + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "strconv" + "strings" + "sync" + "time" + + "github.com/3xDevOps/Aether/internal/coordtransport" + "github.com/3xDevOps/Aether/internal/devexec" + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +const stdinCloseTimeout = 5 * time.Second + +func processID(kind string) string { + var b [8]byte + _, _ = rand.Read(b[:]) + return kind + "-" + hex.EncodeToString(b[:]) +} + +// A root exec gets sudo's secure_path instead of the VM's PATH, as with +// Docker. +func execEnv(rec *record, user string, extra []string) []string { + if uid, _, _ := strings.Cut(user, ":"); uid == "0" || uid == "root" { + extra = append(append([]string{}, extra...), runtime.RootPath) + } + return overrideEnv(rec.Env, extra) +} + +func overrideEnv(base, extra []string) []string { + merged := make(map[string]string, len(base)+len(extra)) + for _, list := range [][]string{base, extra} { + for _, pair := range list { + if k, v, ok := strings.Cut(pair, "="); ok { + merged[k] = v + } + } + } + return runtime.EnvList(merged) +} + +func (e *engine) settingUp(key string) bool { return e.flag(e.booting, key) } + +// attach replays Main's output from its start only to the first attach after +// this process booted the VM; later ones, like docker attach, see output from +// now on. +func (e *engine) attach(ctx context.Context, key string) (runtime.Attachment, error) { + if _, err := e.requireRunning(ctx, key, "attach to"); err != nil { + return nil, err + } + h, err := e.helper(key) + if err != nil { + return nil, err + } + e.mu.Lock() + fromStart := e.fresh[key] + delete(e.fresh, key) + e.mu.Unlock() + return h.Process(guest.MainID).Attach(fromStart), nil +} + +func (e *engine) spawn(ctx context.Context, key string, spec guest.ProcessSpec) (process, runtime.Attachment, error) { + rec, err := e.requireRunning(ctx, key, "exec in") + if err != nil { + return nil, nil, err + } + // A paused guest would hold the exec until ctx ends; Docker refuses it. + if e.paused(ctx, key) { + return nil, nil, fmt.Errorf("vm runtime: exec in VM %s: it is paused", key) + } + h, err := e.helper(key) + if err != nil { + return nil, nil, err + } + if spec.User == "" { + spec.User = rec.User + } + spec.Env = execEnv(rec, spec.User, spec.Env) + if spec.Dir == "" { + spec.Dir = rec.WorkingDir + } + p, err := h.Spawn(ctx, spec) + if err != nil { + return nil, nil, err + } + return p, p.Attach(true), nil +} + +func (e *engine) exec(ctx context.Context, key string, argv []string, workDir string) (int, string, string, error) { + p, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: processID("exec"), Argv: argv, Dir: workDir, Lossless: true}) + var start *guest.StartError + if errors.As(err, &start) { + return start.Code, "", err.Error(), nil + } + if err != nil { + return 0, "", "", fmt.Errorf("vm runtime: exec: %w", err) + } + defer func() { _ = att.Close() }() + if err = p.CloseStdin(ctx); err != nil { + return 0, "", "", fmt.Errorf("vm runtime: exec input: %w", err) + } + stdout, stderr, err := readLimited(ctx, att) + if err != nil { + return 0, stdout, stderr, fmt.Errorf("vm runtime: exec output: %w", err) + } + code, err := p.Wait(ctx) + if err != nil { + return 0, stdout, stderr, fmt.Errorf("vm runtime: exec wait: %w", err) + } + return code, stdout, stderr, nil +} + +// readLimited reads both streams to their end and refuses output past +// runtime.ExecOutputLimit, which the two share. +func readLimited(ctx context.Context, att runtime.Attachment) (string, string, error) { + var mu sync.Mutex + total := 0 + var out, errOut bytes.Buffer + limited := func(into *bytes.Buffer) io.Writer { + return writerFunc(func(p []byte) (int, error) { + mu.Lock() + defer mu.Unlock() + if total += len(p); total > runtime.ExecOutputLimit { + return 0, fmt.Errorf("output passed the %d byte limit", runtime.ExecOutputLimit) + } + return into.Write(p) + }) + } + err := copyStreams(ctx, att, limited(&out), limited(&errOut), func() { _ = att.Close() }) + return out.String(), errOut.String(), err +} + +type writerFunc func([]byte) (int, error) + +func (f writerFunc) Write(p []byte) (int, error) { return f(p) } + +// copyStreams copies stdout and stderr at once; the first failure, or the +// end of ctx, calls abort so the other copy is not left blocked on a stream +// the command keeps open. +func copyStreams(ctx context.Context, att runtime.Attachment, stdout, stderr io.Writer, abort func()) error { + abort = sync.OnceFunc(abort) + stop := context.AfterFunc(ctx, abort) + defer stop() + errs := make(chan error, 2) + for _, pair := range [...]struct { + w io.Writer + r io.Reader + }{{stdout, att.Stdout()}, {stderr, att.Stderr()}} { + go func() { + _, err := io.Copy(pair.w, pair.r) + if err != nil { + abort() + } + errs <- err + }() + } + err := errors.Join(<-errs, <-errs) + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return err +} + +func (e *engine) execTTY(ctx context.Context, key string, argv []string, workDir string, cols, rows uint) (runtime.Attachment, error) { + size := guest.Size{Cols: 80, Rows: 24} + if cols != 0 && rows != 0 { + size = guest.Size{Cols: uint16(min(cols, 65535)), Rows: uint16(min(rows, 65535))} + } + _, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: processID("tty"), Argv: argv, Dir: workDir, TTY: &size}) + var start *guest.StartError + if errors.As(err, &start) && (start.Code == 126 || start.Code == 127) { + return nil, &runtime.ExecExitError{Code: start.Code} + } + if err != nil { + return nil, fmt.Errorf("vm runtime: exec: %w", err) + } + return att, nil +} + +// execAttached keeps the staged helper's input open while it runs: closing +// it, when ctx ends or the output cannot be delivered, kills the command. +func (e *engine) execAttached(ctx context.Context, key string, spec runtime.AttachedExec) (int, error) { + argv := append([]string{coordtransport.CLIPath, devexec.Command, devexec.Attached, strconv.Itoa(len(spec.Stdin))}, spec.Argv...) + p, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: processID("attached"), Argv: argv, User: spec.User, Dir: spec.WorkingDir, Lossless: true}) + if err != nil { + return 0, fmt.Errorf("vm runtime: exec: %w", err) + } + defer func() { _ = att.Close() }() + closeStdin := sync.OnceFunc(func() { + closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stdinCloseTimeout) + defer cancel() + _ = p.CloseStdin(closeCtx) + }) + stop := context.AfterFunc(ctx, closeStdin) + defer stop() + if len(spec.Stdin) > 0 { + if _, err = att.Stdin().Write(spec.Stdin); err != nil { + closeStdin() + return 0, fmt.Errorf("vm runtime: exec input: %w", err) + } + } + if err = copyStreams(ctx, att, spec.Stdout, spec.Stderr, closeStdin); err != nil { + return 0, fmt.Errorf("vm runtime: exec output: %w", err) + } + code, err := p.Wait(ctx) + if err != nil { + return 0, fmt.Errorf("vm runtime: exec wait: %w", err) + } + return code, nil +} + +func (e *engine) runAsRoot(ctx context.Context, h helper, argv []string) (int, string, error) { + p, err := h.Spawn(ctx, guest.ProcessSpec{ID: processID("prepare"), Argv: argv, User: "0:0", Env: []string{runtime.RootPath}, Dir: "/", Lossless: true}) + if err != nil { + return 0, "", err + } + att := p.Attach(true) + defer func() { _ = att.Close() }() + if err = p.CloseStdin(ctx); err != nil { + return 0, "", err + } + stdout, stderr, err := readLimited(ctx, att) + if err != nil { + return 0, "", err + } + code, err := p.Wait(ctx) + return code, stdout + stderr, err +} diff --git a/internal/runtime/vm/exec_test.go b/internal/runtime/vm/exec_test.go new file mode 100644 index 000000000..b17117460 --- /dev/null +++ b/internal/runtime/vm/exec_test.go @@ -0,0 +1,159 @@ +package vm + +import ( + "context" + "errors" + "io/fs" + "strings" + "testing" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +func TestPipeExecutionClosesInputAndKeepsStdoutLossless(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + m, err := h.engine.startManagedExec(t.Context(), "run-a", runtime.ExecSpec{Argv: []string{"cat"}, CreationKey: "ssh-1"}, false) + if err != nil { + t.Fatal(err) + } + supervisor := h.guest("run-a").procs[m.Identity().ExecID] + if !supervisor.spec.Lossless { + t.Error("a pipe execution without Lossless may drop stdout in the guest") + } + stdin := m.Attachment().Stdin() + if _, ok := stdin.(runtime.ContextWriter); !ok { + t.Error("a pipe execution's input lost WriteContext") + } + if err := stdin.Close(); err != nil { + t.Fatal(err) + } + supervisor.mu.Lock() + closed := supervisor.stdin + supervisor.mu.Unlock() + if !closed { + t.Error("closing a pipe execution's input did not close the command's stdin") + } +} + +func TestExecStopsWhenItsContextEndsOrStderrFloods(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + h.guest("run-a").output = map[string]string{"hang": "hang", "flood": "errflood:"} + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + start := time.Now() + if _, _, _, err := h.engine.exec(ctx, "run-a", []string{"hang"}, ""); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("exec of a hung command = %v", err) + } + if time.Since(start) > 5*time.Second { + t.Errorf("exec outlived its context by %s", time.Since(start)) + } + done := make(chan error, 1) + go func() { + _, _, _, err := h.engine.exec(t.Context(), "run-a", []string{"flood"}, "") + done <- err + }() + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), "byte limit") { + t.Errorf("exec flooding stderr = %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("a stderr flood left exec waiting on a quiet stdout") + } +} + +func TestWaitEndsWhenTheVMDiesMidWait(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + done := make(chan runtime.ExitStatus, 1) + go func() { + st, _ := h.engine.wait(context.WithoutCancel(t.Context()), "run-a") + done <- st + }() + time.Sleep(100 * time.Millisecond) + _ = h.units.Stop(t.Context(), unitName("run-a")) + select { + case st := <-done: + if st.Code != lostExitCode { + t.Errorf("wait on a VM that died = %+v", st) + } + case <-time.After(10 * time.Second): + t.Fatal("wait kept waiting on a VM that died") + } +} + +func TestAnOldWaitLeavesANewBootRunning(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + if err := h.engine.stop(t.Context(), "run-a", time.Second); err != nil { + t.Fatal(err) + } + if err := h.engine.start(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + h.guest("run-a").procs[guest.MainID].exit(9) + code, ok := h.engine.waitMain(t.Context(), "run-a", 1) + if !ok || code != 143 { + t.Errorf("a wait on boot 1 = %d, %v; want boot 1's exit 143", code, ok) + } + if h.units.vmm("run-a") == nil { + t.Error("a wait on the previous boot powered off the new one") + } + if info, _ := h.engine.inspect(t.Context(), "run-a"); info.Exited { + t.Errorf("the new boot inspects as exited: %+v", info) + } +} + +func TestManagedExecRecoveryKeepsTheExitRecordAndTransientErrors(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + m, err := h.engine.startManagedExec(t.Context(), "run-a", runtime.ExecSpec{Argv: []string{"bash"}, CreationKey: "terminal-1", Cols: 80, Rows: 24}, true) + if err != nil { + t.Fatal(err) + } + identity := m.Identity() + if st, stopErr := m.Stop(t.Context(), time.Second); stopErr != nil || st.Code != 143 { + t.Fatalf("stop = %+v, %v", st, stopErr) + } + g := h.guest("run-a") + g.mu.Lock() + delete(g.procs, identity.ExecID) + g.mu.Unlock() + if st, stErr := m.Status(t.Context()); stErr != nil || !st.Exited || *st.ExitCode != 143 { + t.Errorf("status once the helper forgot the supervisor = %+v, %v", st, stErr) + } + if _, err = h.engine.recoverExec(t.Context(), "run-a", identity); err != nil { + t.Errorf("recovery once the helper forgot the supervisor = %v", err) + } + h.units.stateErr = errors.New("systemctl: connection timed out") + _, err = h.engine.recoverExec(t.Context(), "run-a", identity) + if !errors.Is(err, runtime.ErrExecUnavailable) || errors.Is(err, runtime.ErrNotFound) { + t.Errorf("recovery while systemd does not answer = %v, want unavailable, not gone", err) + } +} + +func TestPausedVMsAreNotAskedForStats(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + h.guest("run-a").stats = guest.Stats{OOMKills: 3, MemoryLimitBytes: 2 << 30} + if _, err := h.engine.memoryEvents(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + if err := h.engine.pause(t.Context(), "run-a"); err != nil { + t.Fatal(err) + } + h.guest("run-a").stats = guest.Stats{OOMKills: 99} + if events, err := h.engine.memoryEvents(t.Context(), "run-a"); err != nil || events.OOMKills != 3 { + t.Errorf("memory events of a paused VM = %+v, %v; want the last reading", events, err) + } + if stats, err := h.engine.stats(t.Context()); err != nil || len(stats) != 0 { + t.Errorf("stats of a paused VM = %+v, %v", stats, err) + } + if _, err := h.engine.memoryEvents(t.Context(), "run-b"); err == nil || errors.Is(err, fs.ErrNotExist) { + t.Errorf("memory events of an unknown VM = %v", err) + } +} diff --git a/internal/runtime/vm/fakes_test.go b/internal/runtime/vm/fakes_test.go new file mode 100644 index 000000000..1c34157b2 --- /dev/null +++ b/internal/runtime/vm/fakes_test.go @@ -0,0 +1,522 @@ +package vm + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/3xDevOps/Aether/internal/devexec" + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +// harness wires an engine to fakes: systemd units that serve a fake Cloud +// Hypervisor API on the VM's real API socket, one fake helper per VM, and a +// fake image store. +type harness struct { + t *testing.T + engine *engine + units *fakeUnits + images *fakeImages + mu sync.Mutex + guests map[string]*fakeHelper +} + +func newHarness(t *testing.T) *harness { + t.Helper() + dir, err := os.MkdirTemp("", "vmt") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + h := &harness{t: t, images: &fakeImages{}, guests: map[string]*fakeHelper{}} + h.units = &fakeUnits{t: t, active: map[string]*fakeVMM{}} + h.engine = h.newEngine(dir) + t.Cleanup(h.units.stopAll) + return h +} + +// newEngine is a second engine process over the same data directory. +func (h *harness) newEngine(dir string) *engine { + cfg := Config{DataDir: dir, Images: h.images, Monitor: []string{"/srv/aether-server", MonitorCommand}, Helper: "/srv/aether-server", CloudHypervisor: "cloud-hypervisor", Virtiofsd: "virtiofsd"} + e := newEngine(cfg, h.units, func(vsock string) (helper, error) { + return h.guest(filepath.Base(filepath.Dir(vsock))), nil + }) + e.run = func(context.Context, string, ...string) error { return nil } + e.natReady = true + return e +} + +func (h *harness) guest(key string) *fakeHelper { + h.mu.Lock() + defer h.mu.Unlock() + g := h.guests[key] + if g == nil { + g = &fakeHelper{key: key, units: h.units, procs: map[string]*fakeProc{}} + h.guests[key] = g + } + return g +} + +func sessionSpec(key string) runtime.Spec { + return runtime.Spec{ + Image: "aether/standard:1", Command: []string{"/bin/sh", "-c", "agent"}, TTY: true, + Env: map[string]string{"AETHER_RUN_ID": key, "HOME": "/root", "TOKEN": "s3cret"}, CreationKey: key, + Fingerprint: "fp-1", CPULimit: 1.5, MemoryLimitBytes: 2 << 30, PidsLimit: 4096, + } +} + +type fakeImages struct { + mu sync.Mutex + rootDisks int +} + +func (f *fakeImages) Kernel() (string, error) { return "/k/vmlinux", nil } + +func (f *fakeImages) HelperDisk(_ context.Context, _, path string) (vmimage.Disk, error) { + return vmimage.Disk{Path: path, Format: "raw"}, os.WriteFile(path, nil, 0o600) +} + +func (f *fakeImages) BaseDisk(_ context.Context, ref string) (vmimage.Base, error) { + if ref == "missing:1" { + return vmimage.Base{}, errors.New("manifest unknown") + } + return vmimage.Base{ID: "sha256:b", Path: "/images/b.raw", Config: vmimage.ImageConfig{ + Env: []string{"PATH=/usr/bin:/bin", "HOME=/image-home"}, WorkingDir: "/srv", + }}, nil +} + +func (f *fakeImages) NewRootDisk(_ context.Context, _ vmimage.Base, path string) (vmimage.Disk, error) { + f.mu.Lock() + f.rootDisks++ + f.mu.Unlock() + return vmimage.Disk{Path: path, Format: "qcow2"}, os.WriteFile(path, nil, 0o600) +} + +func (f *fakeImages) NewDataDisk(_ context.Context, path string, _ int64) (vmimage.Disk, error) { + return vmimage.Disk{Path: path, Format: "raw"}, os.WriteFile(path, nil, 0o600) +} + +func (f *fakeImages) roots() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.rootDisks +} + +// fakeUnits starts a fake VMM per unit: it reads the launch the engine +// wrote and serves Cloud Hypervisor's API on the API socket it names. +type fakeUnits struct { + t *testing.T + mu sync.Mutex + active map[string]*fakeVMM + started []startCall + failNext bool + stateErr error +} + +type startCall struct { + name string + argv []string + properties []string + launch launch +} + +func (u *fakeUnits) Start(_ context.Context, name string, argv, properties []string) error { + data, err := os.ReadFile(filepath.Join(argv[len(argv)-1], launchFile)) + if err != nil { + return err + } + var l launch + if err = json.Unmarshal(data, &l); err != nil { + return err + } + u.mu.Lock() + defer u.mu.Unlock() + u.started = append(u.started, startCall{name, argv, properties, l}) + if u.failNext { + u.failNext = false + return nil + } + socket := "" + for i, arg := range l.Args { + if arg == "--api-socket" { + socket = strings.TrimPrefix(l.Args[i+1], "path=") + } + } + vmm, err := startVMM(socket, l.Args, func() { go func() { _ = u.Stop(context.Background(), name) }() }) + if err != nil { + return err + } + u.active[name] = vmm + return nil +} + +func (u *fakeUnits) Stop(_ context.Context, name string) error { + u.mu.Lock() + vmm := u.active[name] + delete(u.active, name) + u.mu.Unlock() + if vmm != nil { + vmm.close() + } + return nil +} + +func (u *fakeUnits) State(_ context.Context, name string) (unitState, error) { + u.mu.Lock() + defer u.mu.Unlock() + if u.stateErr != nil { + return unitState{}, u.stateErr + } + if u.active[name] != nil { + return unitState{Active: true, MainPID: 4242}, nil + } + return unitState{}, nil +} + +func (u *fakeUnits) stopAll() { + u.mu.Lock() + names := make([]string, 0, len(u.active)) + for name := range u.active { + names = append(names, name) + } + u.mu.Unlock() + for _, name := range names { + _ = u.Stop(context.Background(), name) + } +} + +func (u *fakeUnits) vmm(key string) *fakeVMM { + u.mu.Lock() + defer u.mu.Unlock() + return u.active[unitName(key)] +} + +func (u *fakeUnits) last() startCall { + u.mu.Lock() + defer u.mu.Unlock() + return u.started[len(u.started)-1] +} + +// fakeVMM is Cloud Hypervisor's REST API with the state transitions the +// engine relies on. +type fakeVMM struct { + mu sync.Mutex + state string + calls []string + restored bool + srv *http.Server +} + +func startVMM(socket string, args []string, exit func()) (*fakeVMM, error) { + _ = os.Remove(socket) + l, err := net.Listen("unix", socket) + if err != nil { + return nil, err + } + v := &fakeVMM{state: "Running"} + for _, arg := range args { + if arg == "--restore" { + v.state, v.restored = "Paused", true + } + } + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/", func(w http.ResponseWriter, r *http.Request) { + endpoint := strings.TrimPrefix(r.URL.Path, "/api/v1/") + body, _ := io.ReadAll(r.Body) + v.mu.Lock() + defer v.mu.Unlock() + v.calls = append(v.calls, endpoint) + switch endpoint { + case "vm.info": + _ = json.NewEncoder(w).Encode(map[string]string{"state": v.state}) + case "vm.pause": + v.state = "Paused" + case "vm.resume": + v.state = "Running" + case "vm.snapshot": + var req struct { + Destination string `json:"destination_url"` + } + _ = json.Unmarshal(body, &req) + dir := strings.TrimPrefix(req.Destination, "file://") + _ = os.WriteFile(filepath.Join(dir, "state.json"), []byte("{}"), 0o600) + case "vm.counters": + _ = json.NewEncoder(w).Encode(map[string]map[string]uint64{ + "net0": {"rx_bytes": 10, "tx_bytes": 20}, "root": {"read_bytes": 1, "write_bytes": 2}, "data": {"read_bytes": 3, "write_bytes": 4}, + }) + case "vmm.shutdown": + exit() + default: + http.Error(w, "unknown endpoint", http.StatusNotFound) + } + }) + v.srv = &http.Server{Handler: mux, ReadHeaderTimeout: time.Second} + go func() { _ = v.srv.Serve(l) }() + return v, nil +} + +func (v *fakeVMM) close() { _ = v.srv.Close() } + +func (v *fakeVMM) called() []string { + v.mu.Lock() + defer v.mu.Unlock() + return append([]string(nil), v.calls...) +} + +// fakeHelper is the in-VM helper. Main exits when it is signalled or told to. +type fakeHelper struct { + key string + units *fakeUnits + mu sync.Mutex + boots []guest.BootSpec + procs map[string]*fakeProc + // bootErr and output program the next Boot and Spawn. + bootErr error + pingErr error + output map[string]string + ignoreTE bool + clock time.Time + stats guest.Stats + // finals are dev-exec's own exit records, which outlive the helper's. + finals map[string]int +} + +func (g *fakeHelper) Ping(ctx context.Context) (guest.Info, error) { + if g.pingErr != nil { + <-ctx.Done() + return guest.Info{}, g.pingErr + } + return guest.Info{BootID: "b1", Protocol: 1}, nil +} + +func (g *fakeHelper) Boot(_ context.Context, spec guest.BootSpec) error { + g.mu.Lock() + defer g.mu.Unlock() + g.boots = append(g.boots, spec) + if g.bootErr != nil { + return g.bootErr + } + if main := g.procs[guest.MainID]; main == nil || main.exited() { + g.procs[guest.MainID] = newFakeProc(spec.Main, "", 0, g.ignoreTE) + } + return nil +} + +func (g *fakeHelper) Spawn(_ context.Context, spec guest.ProcessSpec) (process, error) { + g.mu.Lock() + defer g.mu.Unlock() + if _, ok := g.procs[spec.ID]; ok { + return nil, errors.New("already exists") + } + out := g.output[spec.Argv[0]] + if len(spec.Argv) >= 8 && spec.Argv[1] == devexec.Command && spec.Argv[2] == "control" { + out = g.control(spec.Argv[3], spec.Argv[4], spec.Argv[5], spec.Argv[6]) + } + if strings.HasPrefix(out, "start-error:") { + return nil, &guest.StartError{Code: 127, Err: errors.New("exec: no such file")} + } + p := newFakeProc(spec, out, 0, false) + g.procs[spec.ID] = p + if !strings.Contains(strings.Join(spec.Argv, " "), "dev-exec run") { + p.exit(0) + } + return p, nil +} + +// control answers like dev-exec's control command; it holds g.mu. +func (g *fakeHelper) control(key, execID, claim, action string) string { + state := devexec.State{CreationKey: key, ExecID: execID, ClaimToken: claim} + supervisor := g.procs[execID] + if code, ok := g.finals[execID]; ok && supervisor == nil { + state.Exited, state.ExitCode = true, &code + } else if supervisor == nil { + state.Error = "no such execution" + } else { + if action == "stop" { + supervisor.exit(143) + } + if st, _ := supervisor.Status(context.Background()); st.Exited { + code := st.ExitCode + state.Exited, state.ExitCode = true, &code + if g.finals == nil { + g.finals = map[string]int{} + } + g.finals[execID] = code + } else { + state.Running = true + } + } + data, _ := json.Marshal(state) + return string(data) +} + +func (g *fakeHelper) Process(id string) process { + g.mu.Lock() + defer g.mu.Unlock() + if p := g.procs[id]; p != nil { + return p + } + return missingProc{} +} + +func (g *fakeHelper) spawned() []guest.ProcessSpec { + g.mu.Lock() + defer g.mu.Unlock() + var out []guest.ProcessSpec + for _, p := range g.procs { + out = append(out, p.spec) + } + return out +} + +func (g *fakeHelper) Stats(context.Context) (guest.Stats, error) { return g.stats, nil } + +func (g *fakeHelper) SyncClock(_ context.Context, t time.Time) error { + g.mu.Lock() + g.clock = t + g.mu.Unlock() + return nil +} + +// PowerOff ends the VM the way a guest poweroff ends Cloud Hypervisor. +func (g *fakeHelper) PowerOff(ctx context.Context) error { + return g.units.Stop(ctx, unitName(g.key)) +} + +func (g *fakeHelper) Close() error { return nil } + +type fakeProc struct { + spec guest.ProcessSpec + out string + ignoreTE bool + mu sync.Mutex + signals []syscall.Signal + done chan struct{} + code int + stdin bool +} + +func newFakeProc(spec guest.ProcessSpec, out string, code int, ignoreTerm bool) *fakeProc { + return &fakeProc{spec: spec, out: out, code: code, ignoreTE: ignoreTerm, done: make(chan struct{})} +} + +func (p *fakeProc) exited() bool { + select { + case <-p.done: + return true + default: + return false + } +} + +func (p *fakeProc) exit(code int) { + p.mu.Lock() + defer p.mu.Unlock() + select { + case <-p.done: + default: + p.code = code + close(p.done) + } +} + +// Attach serves the programmed output. "hang" never ends stdout, and +// "errflood:" sends stderr past the exec output limit first; both end only +// when the attachment closes. +func (p *fakeProc) Attach(bool) runtime.Attachment { + stdout, held := io.Pipe() + a := &fakeAttachment{stdout: stdout, stderr: strings.NewReader(""), closed: func() { _ = held.CloseWithError(io.ErrClosedPipe) }} + switch p.out { + case "hang": + case "errflood:": + a.stderr = strings.NewReader(strings.Repeat("e", runtime.ExecOutputLimit+1)) + default: + go func() { + _, _ = io.WriteString(held, p.out) + _ = held.Close() + }() + } + return a +} + +func (p *fakeProc) CloseStdin(context.Context) error { + p.mu.Lock() + p.stdin = true + p.mu.Unlock() + return nil +} + +func (p *fakeProc) Resize(context.Context, guest.Size) error { return nil } + +func (p *fakeProc) Signal(_ context.Context, sig syscall.Signal) error { + p.mu.Lock() + p.signals = append(p.signals, sig) + ignore := p.ignoreTE && sig == syscall.SIGTERM + p.mu.Unlock() + if !ignore { + p.exit(128 + int(sig)) + } + return nil +} + +func (p *fakeProc) Status(context.Context) (guest.State, error) { + select { + case <-p.done: + return guest.State{Exited: true, ExitCode: p.code}, nil + default: + return guest.State{}, nil + } +} + +func (p *fakeProc) Wait(ctx context.Context) (int, error) { + select { + case <-p.done: + return p.code, nil + case <-ctx.Done(): + return 0, ctx.Err() + } +} + +type missingProc struct{} + +func (missingProc) Attach(bool) runtime.Attachment { return &fakeAttachment{} } +func (missingProc) CloseStdin(context.Context) error { return guest.ErrNotFound } +func (missingProc) Resize(context.Context, guest.Size) error { return guest.ErrNotFound } +func (missingProc) Signal(context.Context, syscall.Signal) error { return guest.ErrNotFound } +func (missingProc) Status(context.Context) (guest.State, error) { + return guest.State{}, guest.ErrNotFound +} +func (missingProc) Wait(context.Context) (int, error) { return 0, guest.ErrNotFound } + +type fakeAttachment struct { + stdout, stderr io.Reader + closed func() +} + +func (a *fakeAttachment) Stdin() io.WriteCloser { return nopWriteCloser{} } +func (a *fakeAttachment) Stdout() io.Reader { return a.stdout } +func (a *fakeAttachment) Stderr() io.Reader { return a.stderr } +func (a *fakeAttachment) Resize(context.Context, uint, uint) error { return nil } +func (a *fakeAttachment) Close() error { + if a.closed != nil { + a.closed() + } + return nil +} + +type nopWriteCloser struct{} + +func (nopWriteCloser) Write(p []byte) (int, error) { return len(p), nil } +func (nopWriteCloser) Close() error { return nil } diff --git a/internal/runtime/vm/helper.go b/internal/runtime/vm/helper.go new file mode 100644 index 000000000..d7fc60453 --- /dev/null +++ b/internal/runtime/vm/helper.go @@ -0,0 +1,54 @@ +package vm + +import ( + "context" + "syscall" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +type helper interface { + Ping(context.Context) (guest.Info, error) + Boot(context.Context, guest.BootSpec) error + Spawn(context.Context, guest.ProcessSpec) (process, error) + Process(id string) process + Stats(context.Context) (guest.Stats, error) + SyncClock(context.Context, time.Time) error + PowerOff(context.Context) error + Close() error +} + +type process interface { + Attach(fromStart bool) runtime.Attachment + CloseStdin(context.Context) error + Resize(context.Context, guest.Size) error + Signal(context.Context, syscall.Signal) error + Status(context.Context) (guest.State, error) + Wait(context.Context) (int, error) +} + +func dialHelper(vsockPath string) (helper, error) { + c, err := guest.Dial(vsockPath) + if err != nil { + return nil, err + } + return guestClient{c}, nil +} + +type guestClient struct{ *guest.Client } + +func (c guestClient) Spawn(ctx context.Context, spec guest.ProcessSpec) (process, error) { + p, err := c.Client.Spawn(ctx, spec) + if err != nil { + return nil, err + } + return guestProcess{p}, nil +} + +func (c guestClient) Process(id string) process { return guestProcess{c.Client.Process(id)} } + +type guestProcess struct{ *guest.Process } + +func (p guestProcess) Attach(fromStart bool) runtime.Attachment { return p.Process.Attach(fromStart) } diff --git a/internal/runtime/vm/kvm_integration_test.go b/internal/runtime/vm/kvm_integration_test.go new file mode 100644 index 000000000..2a83f0af9 --- /dev/null +++ b/internal/runtime/vm/kvm_integration_test.go @@ -0,0 +1,216 @@ +//go:build integration + +package vm + +import ( + "bufio" + "context" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +// kvmEngine runs the engine as the server does: real systemd units, the +// server binary as monitor and helper, and the image store. It needs root +// on a KVM host with cloud-hypervisor and virtiofsd, plus: +// +// AETHER_VM_TEST_DIR data directory, on the filesystem the store uses +// AETHER_VM_TEST_KERNEL guest kernel +// AETHER_VM_TEST_SERVER aether-server built from this tree +func kvmEngine(t *testing.T) *engine { + t.Helper() + dir, kernel, server := os.Getenv("AETHER_VM_TEST_DIR"), os.Getenv("AETHER_VM_TEST_KERNEL"), os.Getenv("AETHER_VM_TEST_SERVER") + if dir == "" || kernel == "" || server == "" { + t.Skip("set AETHER_VM_TEST_DIR, AETHER_VM_TEST_KERNEL and AETHER_VM_TEST_SERVER to boot real VMs") + } + if _, err := os.Stat("/dev/kvm"); err != nil || os.Geteuid() != 0 { + t.Skip("needs root and /dev/kvm") + } + images, err := vmimage.Open(filepath.Join(dir, "vm", "images"), kernel, nil) + if err != nil { + t.Fatal(err) + } + return newEngine(Config{DataDir: dir, Images: images, Monitor: []string{server, MonitorCommand}, Helper: server, + CloudHypervisor: "cloud-hypervisor", Virtiofsd: "/usr/libexec/virtiofsd"}, systemdUnits{}, dialHelper) +} + +func vmRun(t *testing.T, e *engine, key, script string) (string, int) { + t.Helper() + code, out, errOut, err := e.exec(t.Context(), key, []string{"/bin/sh", "-c", script}, "") + if err != nil { + t.Fatalf("exec %q: %v", script, err) + } + return out + errOut, code +} + +func readUntil(t *testing.T, att runtime.Attachment, want string) { + t.Helper() + found := make(chan bool, 1) + go func() { + sc := bufio.NewScanner(att.Stdout()) + for sc.Scan() { + if strings.Contains(sc.Text(), want) { + found <- true + return + } + } + found <- false + }() + select { + case ok := <-found: + if !ok { + t.Fatalf("main's output ended before %q", want) + } + case <-time.After(30 * time.Second): + t.Fatalf("no %q from main", want) + } +} + +func TestKVMSessionLifecycle(t *testing.T) { + e := kvmEngine(t) + key := "run-vmtest-a" + t.Cleanup(func() { _ = e.destroy(context.Background(), key) }) + src := filepath.Join(e.root, "..", "src-"+key) + if err := os.MkdirAll(filepath.Join(src, "checkout"), 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(src) }) + if err := os.WriteFile(filepath.Join(src, "checkout", "hello"), []byte("from host\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "ro"), []byte("read only\n"), 0o644); err != nil { + t.Fatal(err) + } + coord := filepath.Join(src, "coord") + if err := os.MkdirAll(coord, 0o755); err != nil { + t.Fatal(err) + } + l, err := net.Listen("unix", filepath.Join(coord, "coord3.sock")) + if err != nil { + t.Fatal(err) + } + defer func() { _ = l.Close() }() + go func() { + for c, acceptErr := l.Accept(); acceptErr == nil; c, acceptErr = l.Accept() { + _, _ = fmt.Fprintln(c, "host answered") + _ = c.Close() + } + }() + spec := runtime.Spec{ + Image: "docker.io/library/ubuntu:24.04", CreationKey: key, TTY: true, + Command: []string{"/bin/sh", "-c", "echo main-started; exec sleep infinity"}, + SetupScript: "echo setup-ran >> /var/tmp/setup-log", + Env: map[string]string{"AETHER_RUN_ID": key, "HOME": "/root"}, + CPULimit: 1.5, MemoryLimitBytes: 1 << 30, PidsLimit: 512, + WorktreeHostPath: filepath.Join(src, "checkout"), WorktreeMountPath: "/workspace", WorkingDir: "/workspace", + Mounts: []runtime.Mount{ + {HostPath: filepath.Join(src, "ro"), ContainerPath: "/opt/ro", ReadOnly: true}, + {HostPath: coord, ContainerPath: "/run/aether", ReadOnly: true}, + }, + } + start := time.Now() + id, err := e.create(t.Context(), spec) + if err != nil { + t.Fatal(err) + } + t.Logf("create (base disk, root and data disks) took %s", time.Since(start)) + start = time.Now() + if err = e.start(t.Context(), key); err != nil { + t.Fatal(err) + } + t.Logf("start (unit, boot, setup, main) took %s", time.Since(start)) + att, err := e.attach(t.Context(), key) + if err != nil { + t.Fatal(err) + } + readUntil(t, att, "main-started") + _ = att.Close() + + for script, want := range map[string]string{ + "cat /workspace/hello": "from host", + "pwd": "/workspace", + "cat /var/tmp/setup-log": "setup-ran", + "echo x >> /opt/ro || echo refused": "refused", + "nproc": "2", + "cat /proc/self/cgroup": "session", + "test -S /run/aether/coord3.sock && echo socket": "socket", + "getent hosts example.com >/dev/null && echo resolved": "resolved", + } { + if out, _ := vmRun(t, e, key, script); !strings.Contains(out, want) { + t.Errorf("%s: %q, want %q", script, out, want) + } + } + if out, code := vmRun(t, e, key, "echo from-guest > /workspace/written"); code != 0 { + t.Fatalf("write to the checkout: %d %s", code, out) + } + if data, _ := os.ReadFile(filepath.Join(src, "checkout", "written")); string(data) != "from-guest\n" { + t.Errorf("host sees %q", data) + } + + after := kvmEngine(t) + if found, err := after.find(key); err != nil || found != id { + t.Fatalf("a new engine finds %q, %v", found, err) + } + if info, err := after.inspect(t.Context(), key); err != nil || info.Exited { + t.Fatalf("a new engine inspects %+v, %v", info, err) + } + if out, _ := vmRun(t, after, key, "cat /var/tmp/setup-log | wc -l"); strings.TrimSpace(out) != "1" { + t.Errorf("setup ran %s times before the engine restart", out) + } + start = time.Now() + if err := after.stop(t.Context(), key, 2*time.Second); err != nil { + t.Fatal(err) + } + t.Logf("stop took %s", time.Since(start)) + if info, _ := after.inspect(t.Context(), key); !info.Exited { + t.Errorf("stopped VM inspects as %+v", info) + } + if err := after.start(t.Context(), key); err != nil { + t.Fatal(err) + } + if out, _ := vmRun(t, after, key, "cat /var/tmp/setup-log | wc -l"); strings.TrimSpace(out) != "1" { + t.Errorf("a cold start kept the old root disk: setup log has %s lines", out) + } +} + +func TestKVMSaveRestoreKeepsProcesses(t *testing.T) { + e := kvmEngine(t) + key := "run-vmtest-b" + t.Cleanup(func() { _ = e.destroy(context.Background(), key) }) + spec := runtime.Spec{ + Image: "docker.io/library/ubuntu:24.04", CreationKey: key, Command: []string{"/bin/sh", "-c", "exec sleep infinity"}, + Env: map[string]string{"AETHER_RUN_ID": key}, CPULimit: 2, MemoryLimitBytes: 1 << 30, + } + if _, err := e.create(t.Context(), spec); err != nil { + t.Fatal(err) + } + if err := e.start(t.Context(), key); err != nil { + t.Fatal(err) + } + before, _ := vmRun(t, e, key, "setsid sleep 4000 /dev/null 2>&1 & sleep 0.5; pgrep -x sleep | sort | tr '\\n' ' '; cat /proc/sys/kernel/random/boot_id") + start := time.Now() + if err := e.save(t.Context(), key); err != nil { + t.Fatal(err) + } + t.Logf("save took %s", time.Since(start)) + if up, _ := e.running(t.Context(), key); up { + t.Fatal("the VM still runs after save") + } + start = time.Now() + if err := e.restore(t.Context(), key); err != nil { + t.Fatal(err) + } + t.Logf("restore took %s", time.Since(start)) + after, _ := vmRun(t, e, key, "pgrep -x sleep | sort | tr '\\n' ' '; cat /proc/sys/kernel/random/boot_id") + if before != after { + t.Errorf("processes or boot changed across save and restore:\n%s\nvs\n%s", before, after) + } + t.Logf("same processes and boot after restore: %s", strings.TrimSpace(after)) +} diff --git a/internal/runtime/vm/lifecycle.go b/internal/runtime/vm/lifecycle.go new file mode 100644 index 000000000..34484b2d1 --- /dev/null +++ b/internal/runtime/vm/lifecycle.go @@ -0,0 +1,339 @@ +package vm + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "syscall" + "time" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +const ( + defaultStopGrace = 10 * time.Second + killGrace = 5 * time.Second + powerOffTimeout = 15 * time.Second + // lostExitCode reports a VM that ended without its helper reporting + // Main's exit: the host killed it, the guest crashed or the host + // rebooted. + lostExitCode = 255 + // killedExitCode is 128+SIGKILL, what a container reports after a stop + // had to kill. + killedExitCode = 137 + waitPoll = 200 * time.Millisecond +) + +func (e *engine) running(ctx context.Context, key string) (bool, error) { + st, err := e.units.State(ctx, unitName(key)) + return st.Active, err +} + +func (e *engine) requireRunning(ctx context.Context, key, action string) (*record, error) { + rec, err := e.load(key) + if err != nil { + return nil, err + } + up, err := e.running(ctx, key) + if err != nil { + return nil, err + } + if !up { + return nil, fmt.Errorf("vm runtime: %s VM %s: it is not running", action, key) + } + return rec, nil +} + +// stop ends Main as docker stop ends a container's main process. +func (e *engine) stop(ctx context.Context, key string, grace time.Duration) error { + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return err + } + up, err := e.running(ctx, key) + if err != nil || !up { + return err + } + api := newCHAPI(e.path(key, apiSocket)) + if state, stateErr := api.state(ctx); stateErr == nil && state == "Paused" { + if err = api.resume(ctx); err != nil { + return fmt.Errorf("vm runtime: resume before stop: %w", err) + } + } + if grace < 0 { + grace = defaultStopGrace + } + h, err := e.helper(key) + if err != nil { + return err + } + code := e.endMain(ctx, h.Process(guest.MainID), grace) + return e.finish(ctx, rec, code) +} + +func (e *engine) endMain(ctx context.Context, main process, grace time.Duration) int { + if err := main.Signal(ctx, syscall.SIGTERM); errors.Is(err, guest.ErrNotFound) { + return 0 + } + waitFor := func(d time.Duration) (int, bool) { + waitCtx, cancel := context.WithTimeout(ctx, d) + defer cancel() + code, err := main.Wait(waitCtx) + return code, err == nil + } + if code, ok := waitFor(grace); ok { + return code + } + _ = main.Signal(ctx, syscall.SIGKILL) + if code, ok := waitFor(killGrace); ok { + return code + } + return killedExitCode +} + +// finish records Main's exit and powers the VM off; it holds the key's lock. +func (e *engine) finish(ctx context.Context, rec *record, code int) error { + rec.Exited, rec.ExitCode = true, code + if err := e.store(rec); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), powerOffTimeout) + defer cancel() + if h, err := e.helper(rec.Key); err == nil { + _ = h.PowerOff(ctx) + } + e.forgetHelper(rec.Key) + for { + up, err := e.running(ctx, rec.Key) + if err == nil && !up { + return nil + } + select { + case <-ctx.Done(): + return e.units.Stop(context.WithoutCancel(ctx), unitName(rec.Key)) + case <-time.After(waitPoll): + } + } +} + +func (e *engine) kill(ctx context.Context, key string) { + e.forgetHelper(key) + _ = e.units.Stop(ctx, unitName(key)) +} + +// wait blocks until Main of the current boot exits; a later boot means it +// did. On a VM that never booted it waits for the first boot, and on a saved +// VM until it is restored, so a save is not mistaken for an exit. +func (e *engine) wait(ctx context.Context, key string) (runtime.ExitStatus, error) { + boot := -1 + for { + rec, err := e.load(key) + if err != nil { + return runtime.ExitStatus{}, err + } + if boot < 0 && rec.Booted { + boot = rec.Boots + } + if rec.Exited || (boot >= 0 && rec.Boots != boot) { + return runtime.ExitStatus{Code: rec.ExitCode}, nil + } + if rec.Booted && !rec.Saved && !e.flag(e.booting, key) { + up, err := e.running(ctx, key) + if err != nil { + return runtime.ExitStatus{}, err + } + if !up { + if code, err := e.lost(ctx, key); err != nil || code >= 0 { + return runtime.ExitStatus{Code: code}, err + } + continue + } + if code, ok := e.waitMain(ctx, key, boot); ok { + return runtime.ExitStatus{Code: code}, nil + } + } + select { + case <-ctx.Done(): + return runtime.ExitStatus{}, ctx.Err() + case <-time.After(waitPoll): + } + } +} + +// waitMain waits for Main of boot through the helper and finishes the VM; +// false means the wait ended first and the caller looks again. The helper's +// client retries a VM that died until its context ends, so the wait also +// ends when the unit does. +func (e *engine) waitMain(ctx context.Context, key string, boot int) (int, bool) { + h, err := e.helper(key) + if err != nil { + return 0, false + } + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-waitCtx.Done(): + return + case <-tick.C: + if up, upErr := e.running(waitCtx, key); upErr == nil && !up { + cancel() + return + } + } + } + }() + code, err := h.Process(guest.MainID).Wait(waitCtx) + if err != nil { + return 0, false + } + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return 0, false + } + if rec.Exited || rec.Boots != boot { + return rec.ExitCode, true + } + if err := e.finish(ctx, rec, code); err != nil { + return 0, false + } + return code, true +} + +// lost records a booted VM whose unit is gone without an exit, and returns +// -1 when it turns out to have been stopped or saved properly meanwhile. +func (e *engine) lost(ctx context.Context, key string) (int, error) { + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return 0, err + } + if rec.Exited { + return rec.ExitCode, nil + } + if rec.Saved { + return -1, nil + } + up, err := e.running(ctx, key) + if err != nil || up { + return -1, err + } + rec.Exited, rec.ExitCode = true, lostExitCode + e.forgetHelper(key) + return lostExitCode, e.store(rec) +} + +// destroy removes the VM and every file it had; a missing VM is not an +// error. +func (e *engine) destroy(ctx context.Context, key string) error { + if checkKey(key) != nil { + return nil + } + unlock := e.lock(key) + defer unlock() + if _, err := os.Stat(e.dir(key)); errors.Is(err, fs.ErrNotExist) { + return nil + } + e.forgetHelper(key) + if err := e.units.Stop(ctx, unitName(key)); err != nil { + return err + } + if err := os.RemoveAll(e.dir(key)); err != nil { + return fmt.Errorf("vm runtime: remove VM %s: %w", key, err) + } + return nil +} + +// pause stops the vCPUs; the guest keeps its memory, as a frozen container +// does. +func (e *engine) pause(ctx context.Context, key string) error { + if _, err := e.requireRunning(ctx, key, "pause"); err != nil { + return err + } + if err := newCHAPI(e.path(key, apiSocket)).pause(ctx); err != nil { + return fmt.Errorf("vm runtime: pause VM %s: %w", key, err) + } + return nil +} + +func (e *engine) resume(ctx context.Context, key string) error { + if _, err := e.requireRunning(ctx, key, "resume"); err != nil { + return err + } + if err := newCHAPI(e.path(key, apiSocket)).resume(ctx); err != nil { + return fmt.Errorf("vm runtime: resume VM %s: %w", key, err) + } + return nil +} + +// inspect answers from the record and the unit, never from the guest. +func (e *engine) inspect(ctx context.Context, key string) (runtime.ContainerInfo, error) { + rec, err := e.load(key) + if err != nil { + return runtime.ContainerInfo{}, err + } + if rec.Booted && !rec.Exited && !rec.Saved && !e.flag(e.booting, key) { + up, err := e.running(ctx, key) + if err != nil { + return runtime.ContainerInfo{}, err + } + if !up { + if code, err := e.lost(ctx, key); err != nil { + return runtime.ContainerInfo{}, err + } else if code >= 0 { + rec.Exited, rec.ExitCode = true, code + } + } + } + targets := make([]string, len(rec.Mounts)) + for i, m := range rec.Mounts { + targets[i] = m.ContainerPath + } + info := runtime.ContainerInfo{ + Image: rec.Image, User: rec.User, Env: append([]string(nil), rec.Env...), MountTargets: targets, + Fingerprint: rec.Fingerprint, Exited: rec.Exited, + } + if rec.Exited { + info.ExitCode = rec.ExitCode + } + return info, nil +} + +func (e *engine) find(key string) (runtime.ID, error) { + if _, err := e.load(key); err != nil { + return "", err + } + return runtime.ID(idPrefix + key), nil +} + +// initPID is Cloud Hypervisor's PID, whose cgroup is the VM's unit. +func (e *engine) initPID(ctx context.Context, key string) (int, error) { + if _, err := e.load(key); err != nil { + return 0, err + } + st, err := e.units.State(ctx, unitName(key)) + if err != nil || !st.Active { + return 0, err + } + return st.MainPID, nil +} + +func (e *engine) containerIP(ctx context.Context, key string) (string, error) { + rec, err := e.requireRunning(ctx, key, "address") + if err != nil { + return "", err + } + _, addr := netAddrs(rec.Net) + return addr.String(), nil +} diff --git a/internal/runtime/vm/managed.go b/internal/runtime/vm/managed.go new file mode 100644 index 000000000..e74ca5001 --- /dev/null +++ b/internal/runtime/vm/managed.go @@ -0,0 +1,330 @@ +package vm + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "path" + "strings" + "sync" + "time" + + "github.com/3xDevOps/Aether/internal/coordtransport" + "github.com/3xDevOps/Aether/internal/devexec" + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +// startManagedExec runs the staged dev-exec supervisor through the helper, +// as the Docker runtime runs it through docker exec: it waits for a claim +// before it starts the command, so a lost reply cannot orphan the command. +func (e *engine) startManagedExec(ctx context.Context, key string, spec runtime.ExecSpec, tty bool) (_ runtime.ManagedExec, resultErr error) { + if spec.CreationKey == "" || len(spec.CreationKey) > 1024 || strings.ContainsRune(spec.CreationKey, 0) { + return nil, errors.New("runtime: valid single-use execution creation key is required") + } + if len(spec.Argv) == 0 || spec.Argv[0] == "" { + return nil, errors.New("runtime: execution command is required") + } + for _, arg := range spec.Argv { + if strings.ContainsRune(arg, 0) { + return nil, errors.New("runtime: execution argument contains NUL") + } + } + if spec.WorkingDir != "" && !path.IsAbs(spec.WorkingDir) { + return nil, errors.New("runtime: execution working directory must be absolute") + } + if tty && (spec.Cols == 0 || spec.Rows == 0 || spec.Cols > 65535 || spec.Rows > 65535) { + return nil, errors.New("runtime: execution terminal dimensions must be between 1 and 65535") + } + rec, err := e.load(key) + if err != nil { + return nil, err + } + var token [32]byte + if _, err = rand.Read(token[:]); err != nil { + return nil, fmt.Errorf("runtime: create execution claim: %w", err) + } + claim := hex.EncodeToString(token[:]) + mode := "run-pipe" + var size *guest.Size + if tty { + mode, size = "run", &guest.Size{Cols: uint16(spec.Cols), Rows: uint16(spec.Rows)} + } + env := spec.Env + if spec.User != "" && spec.User != rec.User { + if _, gid, ok := strings.Cut(rec.User, ":"); ok && gid != "" { + env = append(append([]string{}, env...), devexec.ControlGroupEnv+"="+gid) + } + } + execID := processID("managed") + argv := append([]string{coordtransport.CLIPath, devexec.Command, mode, spec.CreationKey, claim}, spec.Argv...) + p, att, err := e.spawn(ctx, key, guest.ProcessSpec{ID: execID, Argv: argv, Env: env, User: spec.User, Dir: spec.WorkingDir, TTY: size, Lossless: spec.Lossless || !tty}) + if err != nil { + return nil, fmt.Errorf("runtime: create managed execution: %w", err) + } + if !tty { + att = newPipeAttachment(p, att, spec.Lossless) + } + m := &managedExec{engine: e, key: key, tty: tty, proc: p, attachment: att, identity: runtime.ExecIdentity{ + ContainerID: runtime.ID(idPrefix + key), ExecID: execID, CreationKey: spec.CreationKey, ClaimToken: claim, User: spec.User, + }} + published := false + defer func() { + if published { + return + } + cleanup, cancel := context.WithTimeout(context.Background(), devexec.StartupTimeout+5*time.Second) + defer cancel() + _, stopErr := m.control(cleanup, "stop", 0) + if stopErr == nil { + _, stopErr = m.Wait(cleanup) + } + _ = m.Detach() + if stopErr != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("runtime: cancelled execution cleanup: %w", stopErr)) + } + }() + if _, err := m.control(ctx, "start", 0); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + published = true + return m, nil +} + +// recoverExec proves the helper's claim. Like Docker's, the recovered +// execution has no attachment: callers reach a recovered command through its +// keeper. +func (e *engine) recoverExec(ctx context.Context, key string, identity runtime.ExecIdentity) (runtime.ManagedExec, error) { + if identity.ExecID == "" || identity.CreationKey == "" || identity.ClaimToken == "" { + return nil, fmt.Errorf("%w: incomplete identity", runtime.ErrExecUnavailable) + } + if _, err := e.load(key); err != nil { + return nil, err + } + up, err := e.running(ctx, key) + if err != nil { + return nil, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + if !up { + return nil, fmt.Errorf("%w: %w: VM %s is not running", runtime.ErrExecUnavailable, runtime.ErrNotFound, key) + } + h, err := e.helper(key) + if err != nil { + return nil, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + p := h.Process(identity.ExecID) + m := &managedExec{engine: e, key: key, proc: p, identity: identity} + _, err = p.Status(ctx) + if errors.Is(err, guest.ErrNotFound) { + if _, err = m.final(ctx); err != nil { + return nil, fmt.Errorf("%w: execution %s: %w", runtime.ErrNotFound, identity.ExecID, err) + } + return m, nil + } + if err != nil { + return nil, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + if _, err := m.control(ctx, "status", 0); err != nil { + return nil, fmt.Errorf("%w: cannot prove helper ownership: %w", runtime.ErrExecUnavailable, err) + } + return m, nil +} + +type managedExec struct { + engine *engine + key string + tty bool + proc process + identity runtime.ExecIdentity + + mu sync.Mutex + attachment runtime.Attachment +} + +func (m *managedExec) Identity() runtime.ExecIdentity { return m.identity } + +func (m *managedExec) Attachment() runtime.Attachment { + m.mu.Lock() + defer m.mu.Unlock() + return m.attachment +} + +func (m *managedExec) control(ctx context.Context, action string, grace time.Duration) (devexec.State, error) { + p, att, err := m.engine.spawn(ctx, m.key, guest.ProcessSpec{ + ID: processID("control"), Argv: runtime.ExecControlArgv(m.identity, action, grace), User: m.identity.User, Lossless: true, + }) + if err != nil { + return devexec.State{}, fmt.Errorf("%w: helper %s: %w", runtime.ErrExecUnavailable, action, err) + } + defer func() { _ = att.Close() }() + if err = p.CloseStdin(ctx); err != nil { + return devexec.State{}, fmt.Errorf("%w: helper %s: %w", runtime.ErrExecUnavailable, action, err) + } + stdout, stderr, err := readLimited(ctx, att) + if err != nil { + return devexec.State{}, fmt.Errorf("%w: helper %s output: %w", runtime.ErrExecUnavailable, action, err) + } + code, err := p.Wait(ctx) + if err != nil { + return devexec.State{}, fmt.Errorf("%w: helper %s: %w", runtime.ErrExecUnavailable, action, err) + } + return runtime.ExecControlState(m.identity, action, code, stdout, stderr) +} + +// Status asks the helper whether the supervisor runs. Once it has ended, the +// supervisor's own record holds the command's exit; it outlives the helper, +// which forgets exited processes. +func (m *managedExec) Status(ctx context.Context) (runtime.ExecState, error) { + st, err := m.proc.Status(ctx) + if errors.Is(err, guest.ErrNotFound) { + state, finalErr := m.final(ctx) + if finalErr != nil { + return state, fmt.Errorf("%w: %w: execution %s: %w", runtime.ErrExecUnavailable, runtime.ErrNotFound, m.identity.ExecID, finalErr) + } + return state, nil + } + if err != nil { + return runtime.ExecState{}, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + if st.Exited { + return m.final(ctx) + } + return m.state(), nil +} + +func (m *managedExec) state() runtime.ExecState { + state := runtime.ExecState{Running: true, Attached: m.Attachment() != nil} + if !state.Attached { + state.UnavailableReason = "attachment is unavailable; the execution was detached" + } + return state +} + +func (m *managedExec) final(ctx context.Context) (runtime.ExecState, error) { + state := m.state() + state.Running = false + final, err := m.control(ctx, "status", 0) + if err != nil { + return state, fmt.Errorf("%w: command exit record unavailable: %w", runtime.ErrExecUnavailable, err) + } + if !final.Exited || final.ExitCode == nil { + return state, fmt.Errorf("%w: helper ended without a command exit record", runtime.ErrExecUnavailable) + } + state.Exited, state.ExitCode = true, final.ExitCode + return state, nil +} + +func (m *managedExec) Wait(ctx context.Context) (runtime.ExitStatus, error) { + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + for { + state, err := m.Status(ctx) + if err != nil { + return runtime.ExitStatus{}, err + } + if state.Exited { + return runtime.ExitStatus{Code: *state.ExitCode}, nil + } + select { + case <-ctx.Done(): + return runtime.ExitStatus{}, ctx.Err() + case <-tick.C: + } + } +} + +func (m *managedExec) Stop(ctx context.Context, grace time.Duration) (runtime.ExitStatus, error) { + if grace < 0 { + grace = 3 * time.Second + } + if grace > time.Minute { + return runtime.ExitStatus{}, errors.New("runtime: managed execution stop grace exceeds one minute") + } + ctx, cancel := context.WithTimeout(ctx, devexec.StartupTimeout+grace+devexec.ReapTimeout) + defer cancel() + st, err := m.proc.Status(ctx) + if err != nil && !errors.Is(err, guest.ErrNotFound) { + return runtime.ExitStatus{}, fmt.Errorf("%w: %w", runtime.ErrExecUnavailable, err) + } + if err == nil && !st.Exited { + if _, err := m.control(ctx, "stop", grace); err != nil { + return runtime.ExitStatus{}, err + } + } + return m.Wait(ctx) +} + +func (m *managedExec) Resize(ctx context.Context, cols, rows uint) error { + if cols == 0 || rows == 0 || cols > 65535 || rows > 65535 { + return errors.New("runtime: execution terminal dimensions must be between 1 and 65535") + } + if m.Attachment() == nil { + return fmt.Errorf("%w: cannot resize an unavailable PTY", runtime.ErrExecUnavailable) + } + if !m.tty { + return errors.New("runtime: resize: execution has no terminal") + } + return m.proc.Resize(ctx, guest.Size{Cols: uint16(cols), Rows: uint16(rows)}) +} + +func (m *managedExec) Detach() error { + m.mu.Lock() + attachment := m.attachment + m.attachment = nil + m.mu.Unlock() + if attachment != nil { + return attachment.Close() + } + return nil +} + +// pipeAttachment gives a pipe execution Docker's semantics: closing its input +// ends the command's stdin, and unless the caller asked for every stream to +// be lossless, stderr drains into a buffer that drops its oldest bytes, so an +// unread stderr never stalls the lossless stdout. +type pipeAttachment struct { + runtime.Attachment + stdin pipeInput + stderr io.Reader +} + +func newPipeAttachment(p process, att runtime.Attachment, lossless bool) *pipeAttachment { + a := &pipeAttachment{Attachment: att, stdin: pipeInput{WriteCloser: att.Stdin(), proc: p}, stderr: att.Stderr()} + if !lossless { + drained := runtime.NewStreamBuffer() + go func() { + _, err := io.Copy(drained, att.Stderr()) + drained.CloseWithError(err) + }() + a.stderr = drained + } + return a +} + +func (a *pipeAttachment) Stdin() io.WriteCloser { return a.stdin } +func (a *pipeAttachment) Stderr() io.Reader { return a.stderr } + +type pipeInput struct { + io.WriteCloser + proc process +} + +func (in pipeInput) WriteContext(ctx context.Context, b []byte) (int, error) { + if w, ok := in.WriteCloser.(runtime.ContextWriter); ok { + return w.WriteContext(ctx, b) + } + return in.Write(b) +} + +func (in pipeInput) Close() error { + _ = in.WriteCloser.Close() + ctx, cancel := context.WithTimeout(context.Background(), stdinCloseTimeout) + defer cancel() + return in.proc.CloseStdin(ctx) +} diff --git a/internal/runtime/vm/monitor.go b/internal/runtime/vm/monitor.go new file mode 100644 index 000000000..77f25e421 --- /dev/null +++ b/internal/runtime/vm/monitor.go @@ -0,0 +1,30 @@ +package vm + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +// MonitorCommand is the aether-server subcommand a VM's unit runs. +const MonitorCommand = "vm-monitor" + +type launch struct { + Shares []share `json:"shares,omitempty"` + Virtiofsd string `json:"virtiofsd,omitempty"` + CloudHypervisor string `json:"cloud_hypervisor"` + Args []string `json:"args"` +} + +func readLaunch(dir string) (launch, error) { + data, err := os.ReadFile(filepath.Join(dir, launchFile)) + if err != nil { + return launch{}, fmt.Errorf("vm monitor: %w", err) + } + var l launch + if err := json.Unmarshal(data, &l); err != nil { + return launch{}, fmt.Errorf("vm monitor: decode %s: %w", launchFile, err) + } + return l, nil +} diff --git a/internal/runtime/vm/monitor_linux.go b/internal/runtime/vm/monitor_linux.go new file mode 100644 index 000000000..258082b72 --- /dev/null +++ b/internal/runtime/vm/monitor_linux.go @@ -0,0 +1,148 @@ +package vm + +import ( + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strconv" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +const virtiofsdReadyTimeout = 10 * time.Second + +// Monitor runs in a VM's unit, whose private mount namespace keeps every +// bind it makes off the host: it composes the share, starts virtiofsd on +// it, then becomes Cloud Hypervisor, so the unit ends with the VM. +func Monitor(args []string) error { + if len(args) != 1 || !filepath.IsAbs(args[0]) { + return errors.New("usage: aether-server vm-monitor ") + } + dir := args[0] + l, err := readLaunch(dir) + if err != nil { + return err + } + for _, name := range []string{apiSocket, vsockSocket, fsSocket} { + if err = os.Remove(filepath.Join(dir, name)); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("vm monitor: remove stale %s: %w", name, err) + } + } + if len(l.Shares) > 0 { + root := filepath.Join(dir, shareDir) + if err = composeShare(root, l.Shares); err != nil { + return err + } + if err = startVirtiofsd(l.Virtiofsd, filepath.Join(dir, fsSocket), root); err != nil { + return err + } + } + argv := append([]string{l.CloudHypervisor}, l.Args...) + path, err := exec.LookPath(l.CloudHypervisor) + if err != nil { + return fmt.Errorf("vm monitor: %w", err) + } + return syscall.Exec(path, argv, os.Environ()) +} + +// composeShare mounts a small tmpfs at root and binds every share at +// root/: nosuid and nodev always, read-only where asked, and +// recursively, like Docker's rbind. The tmpfs itself ends read-only. +func composeShare(root string, shares []share) error { + if err := os.MkdirAll(root, 0o755); err != nil { + return fmt.Errorf("vm monitor: %w", err) + } + if err := unix.Mount("aether-share", root, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, "mode=0755,size=1m"); err != nil { + return fmt.Errorf("vm monitor: mount share root: %w", err) + } + for i, s := range shares { + if err := bindShare(filepath.Join(root, strconv.Itoa(i)), s); err != nil { + return err + } + } + if err := unix.MountSetattr(unix.AT_FDCWD, root, 0, &unix.MountAttr{Attr_set: unix.MOUNT_ATTR_RDONLY}); err != nil { + return fmt.Errorf("vm monitor: make share root read-only: %w", err) + } + return nil +} + +// bindShare resolves a subpath beneath its base without following a symlink +// out of it, as Docker's volume subpath does, because an agent may control +// the base's content. +func bindShare(target string, s share) error { + fd, err := unix.Open(s.Source, unix.O_PATH|unix.O_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("vm monitor: open %s: %w", s.Source, err) + } + defer func() { _ = unix.Close(fd) }() + if s.Subpath != "" { + sub, openErr := unix.Openat2(fd, s.Subpath, &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS, + }) + if openErr != nil { + return fmt.Errorf("vm monitor: resolve %s beneath %s: %w", s.Subpath, s.Source, openErr) + } + _ = unix.Close(fd) + fd = sub + } + var st unix.Stat_t + if err = unix.Fstat(fd, &st); err != nil { + return fmt.Errorf("vm monitor: stat %s: %w", s.Source, err) + } + if st.Mode&unix.S_IFMT == unix.S_IFDIR { + err = os.Mkdir(target, 0o755) + } else { + err = os.WriteFile(target, nil, 0o644) + } + if err != nil { + return fmt.Errorf("vm monitor: %w", err) + } + tree, err := unix.OpenTree(fd, "", unix.OPEN_TREE_CLONE|unix.OPEN_TREE_CLOEXEC|unix.AT_EMPTY_PATH|unix.AT_RECURSIVE) + if err != nil { + return fmt.Errorf("vm monitor: clone %s: %w", s.Source, err) + } + defer func() { _ = unix.Close(tree) }() + attr := unix.MountAttr{Attr_set: unix.MOUNT_ATTR_NOSUID | unix.MOUNT_ATTR_NODEV, Propagation: unix.MS_PRIVATE} + if s.ReadOnly { + attr.Attr_set |= unix.MOUNT_ATTR_RDONLY + } + if err := unix.MountSetattr(tree, "", unix.AT_EMPTY_PATH|unix.AT_RECURSIVE, &attr); err != nil { + return fmt.Errorf("vm monitor: set mount flags on %s: %w", s.Source, err) + } + if err := unix.MoveMount(tree, "", unix.AT_FDCWD, target, unix.MOVE_MOUNT_F_EMPTY_PATH); err != nil { + return fmt.Errorf("vm monitor: bind %s: %w", s.Source, err) + } + return nil +} + +// startVirtiofsd leaves virtiofsd as a child that outlives the exec into +// Cloud Hypervisor; the unit stops it with the VM. +func startVirtiofsd(binary, socket, root string) error { + cmd := exec.Command(binary, "--socket-path="+socket, "--shared-dir="+root, + "--announce-submounts", "--xattr", "--inode-file-handles=prefer", "--sandbox=namespace") + cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr + if err := cmd.Start(); err != nil { + return fmt.Errorf("vm monitor: start virtiofsd: %w", err) + } + exited := make(chan error, 1) + go func() { exited <- cmd.Wait() }() + deadline := time.After(virtiofsdReadyTimeout) + for { + if _, err := os.Stat(socket); err == nil { + return nil + } + select { + case err := <-exited: + return fmt.Errorf("vm monitor: virtiofsd exited before it listened: %v", err) + case <-deadline: + return fmt.Errorf("vm monitor: virtiofsd did not listen on %s within %s", socket, virtiofsdReadyTimeout) + case <-time.After(10 * time.Millisecond): + } + } +} diff --git a/internal/runtime/vm/monitor_other.go b/internal/runtime/vm/monitor_other.go new file mode 100644 index 000000000..e1916f2f4 --- /dev/null +++ b/internal/runtime/vm/monitor_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package vm + +import "errors" + +func Monitor([]string) error { + return errors.New("vm monitor: session VMs run only on Linux") +} diff --git a/internal/runtime/vm/network.go b/internal/runtime/vm/network.go new file mode 100644 index 000000000..7256bbebc --- /dev/null +++ b/internal/runtime/vm/network.go @@ -0,0 +1,150 @@ +package vm + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net/netip" + "os" + "strings" + + "github.com/3xDevOps/Aether/internal/guest" +) + +// vmSubnet holds one /30 per VM: .1 is the host end of its tap, .2 the +// guest. It sits outside Docker's default address pools. +var vmSubnet = netip.MustParsePrefix("10.200.0.0/16") + +const ( + tapPrefix = "aethervm" + maxNets = 1 << 14 +) + +// dockerDNS is what Docker gives a container when the host resolver is +// loopback only. +var dockerDNS = []netip.Addr{netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("8.8.4.4")} + +func tapName(index int) string { return fmt.Sprintf("%s%d", tapPrefix, index) } + +func netAddrs(index int) (host, guestAddr netip.Addr) { + base := vmSubnet.Addr().As4() + n := index * 4 + base[2], base[3] = byte(n>>8), byte(n) + host = netip.AddrFrom4(base).Next() + return host, host.Next() +} + +// chNetArg lets Cloud Hypervisor create the tap and give its host end an +// address; the tap goes away with the VM. +func chNetArg(index int) string { + host, _ := netAddrs(index) + return fmt.Sprintf("tap=%s,ip=%s,mask=255.255.255.252,mac=02:ae:00:00:%02x:%02x", tapName(index), host, byte(index>>8), byte(index)) +} + +func guestNetwork(index int, dns []netip.Addr) *guest.Network { + host, guestAddr := netAddrs(index) + return &guest.Network{Address: netip.PrefixFrom(guestAddr, 30), Gateway: host, DNS: dns} +} + +// allocateNet returns the lowest index no recorded or reserved VM holds. +// The caller releases the reservation once the record is stored. +func (e *engine) allocateNet() (int, error) { + e.netMu.Lock() + defer e.netMu.Unlock() + used := make(map[int]bool, len(e.reserved)) + for index := range e.reserved { + used[index] = true + } + keys, err := e.keys() + if err != nil { + return 0, err + } + for _, key := range keys { + rec, err := e.load(key) + if err != nil { + return 0, err + } + used[rec.Net] = true + } + for index := range maxNets { + if !used[index] { + e.reserved[index] = true + return index, nil + } + } + return 0, fmt.Errorf("vm runtime: all %d VM networks in %s are in use", maxNets, vmSubnet) +} + +func (e *engine) releaseNet(index int) { + e.netMu.Lock() + delete(e.reserved, index) + e.netMu.Unlock() +} + +// ensureNAT runs once per process; a failure is retried at the next start. +func (e *engine) ensureNAT(ctx context.Context) error { + e.netMu.Lock() + defer e.netMu.Unlock() + if e.natReady { + return nil + } + if err := e.run(ctx, "sysctl", "-q", "-w", "net.ipv4.ip_forward=1"); err != nil { + return err + } + subnet, taps := vmSubnet.String(), tapPrefix+"+" + for _, rule := range [][]string{ + {"-t", "nat", "POSTROUTING", "-s", subnet, "!", "-d", subnet, "-j", "MASQUERADE"}, + {"-t", "filter", "FORWARD", "-i", taps, "-j", "ACCEPT"}, + {"-t", "filter", "FORWARD", "-o", taps, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}, + } { + table, chain, match := rule[:2], rule[2], rule[3:] + check := append(append(append([]string{}, table...), "-C", chain), match...) + if e.run(ctx, "iptables", check...) == nil { + continue + } + insert := append(append(append([]string{}, table...), "-I", chain), match...) + if err := e.run(ctx, "iptables", insert...); err != nil { + return err + } + } + e.natReady = true + return nil +} + +// hostDNS prefers systemd-resolved's upstream servers, since its stub at +// 127.0.0.53 is unreachable from a guest. +func hostDNS() []netip.Addr { + for _, path := range []string{"/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"} { + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + break + } + servers := nameservers(f) + _ = f.Close() + if len(servers) > 0 { + return servers + } + } + return dockerDNS +} + +func nameservers(r io.Reader) []netip.Addr { + var out []netip.Addr + sc := bufio.NewScanner(r) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) < 2 || fields[0] != "nameserver" { + continue + } + addr, err := netip.ParseAddr(fields[1]) + if err == nil && addr.Is4() && !addr.IsLoopback() { + out = append(out, addr) + } + } + return out +} diff --git a/internal/runtime/vm/network_test.go b/internal/runtime/vm/network_test.go new file mode 100644 index 000000000..e10b3bf6a --- /dev/null +++ b/internal/runtime/vm/network_test.go @@ -0,0 +1,84 @@ +package vm + +import ( + "context" + "errors" + "net/netip" + "slices" + "strings" + "testing" +) + +func TestNetworkAddressing(t *testing.T) { + host, guestAddr := netAddrs(65) + if host.String() != "10.200.1.5" || guestAddr.String() != "10.200.1.6" { + t.Errorf("addresses of 65 = %s, %s", host, guestAddr) + } + if got := chNetArg(65); got != "tap=aethervm65,ip=10.200.1.5,mask=255.255.255.252,mac=02:ae:00:00:00:41" { + t.Errorf("net arg = %s", got) + } + last, _ := netAddrs(maxNets - 1) + if !vmSubnet.Contains(last) { + t.Errorf("last network %s leaves %s", last, vmSubnet) + } + n := guestNetwork(0, dockerDNS) + if n.Address != netip.MustParsePrefix("10.200.0.2/30") || n.Gateway.String() != "10.200.0.1" { + t.Errorf("guest network = %+v", n) + } + servers := nameservers(strings.NewReader("# x\nnameserver 127.0.0.53\nnameserver 10.0.0.2\nnameserver fe80::1\nsearch ec2.internal\n")) + if !slices.Equal(servers, []netip.Addr{netip.MustParseAddr("10.0.0.2")}) { + t.Errorf("nameservers = %v", servers) + } +} + +func TestAllocateNetSkipsRecordedAndReserved(t *testing.T) { + h := newHarness(t) + h.createStarted("run-a", sessionSpec("run-a")) + first, err := h.engine.allocateNet() + if err != nil || first != 1 { + t.Fatalf("allocate = %d, %v", first, err) + } + second, err := h.engine.allocateNet() + if err != nil || second != 2 { + t.Fatalf("allocate while 1 is reserved = %d, %v", second, err) + } + h.engine.releaseNet(first) + if again, _ := h.engine.allocateNet(); again != 1 { + t.Errorf("allocate after release = %d", again) + } +} + +func TestEnsureNATAddsOnlyMissingRules(t *testing.T) { + h := newHarness(t) + e := h.engine + e.natReady = false + var calls []string + e.run = func(_ context.Context, name string, args ...string) error { + calls = append(calls, strings.Join(append([]string{name}, args...), " ")) + if slices.Contains(args, "-C") && slices.Contains(args, "MASQUERADE") { + return nil + } + if slices.Contains(args, "-C") { + return errors.New("no such rule") + } + return nil + } + if err := e.ensureNAT(t.Context()); err != nil { + t.Fatal(err) + } + want := []string{ + "sysctl -q -w net.ipv4.ip_forward=1", + "iptables -t nat -C POSTROUTING -s 10.200.0.0/16 ! -d 10.200.0.0/16 -j MASQUERADE", + "iptables -t filter -C FORWARD -i aethervm+ -j ACCEPT", + "iptables -t filter -I FORWARD -i aethervm+ -j ACCEPT", + "iptables -t filter -C FORWARD -o aethervm+ -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT", + "iptables -t filter -I FORWARD -o aethervm+ -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT", + } + if !slices.Equal(calls, want) { + t.Errorf("calls:\n%s", strings.Join(calls, "\n")) + } + calls = nil + if err := e.ensureNAT(t.Context()); err != nil || calls != nil { + t.Errorf("second ensureNAT ran %v, %v", calls, err) + } +} diff --git a/internal/runtime/vm/record.go b/internal/runtime/vm/record.go new file mode 100644 index 000000000..0ad092a11 --- /dev/null +++ b/internal/runtime/vm/record.go @@ -0,0 +1,152 @@ +package vm + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +// Files in a VM's directory, /vm//. +const ( + recordFile = "vm.json" + launchFile = "launch.json" + apiSocket = "api.sock" + vsockSocket = "vsock.sock" + fsSocket = "fs.sock" + shareDir = "share" + helperImage = "helper.img" + rootDisk = "root.img" + dataDisk = "data.img" + consoleLog = "console.log" + monitorLog = "monitor.log" + snapshotDir = "snapshot" +) + +// keyPattern keeps a creation key usable as a directory, a systemd unit name +// and a guest hostname. +var keyPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +// record is everything a VM needs to boot again and to answer Inspect +// without reaching the guest. Env is the image's environment overlaid with +// the spec's, which is what a container reports. +type record struct { + Key string `json:"key"` + Fingerprint string `json:"fingerprint,omitempty"` + Image string `json:"image"` + Base vmimage.Base `json:"base"` + Root vmimage.Disk `json:"root"` + Data vmimage.Disk `json:"data"` + User string `json:"user,omitempty"` + Env []string `json:"env"` + WorkingDir string `json:"working_dir,omitempty"` + Command []string `json:"command"` + TTY bool `json:"tty"` + SetupScript string `json:"setup_script,omitempty"` + SecretEnv []string `json:"secret_env,omitempty"` + NestedDocker bool `json:"nested_docker,omitempty"` + Sudo bool `json:"sudo,omitempty"` + CPUs int `json:"cpus"` + CPULimit float64 `json:"cpu_limit,omitempty"` + MemoryMiB int64 `json:"memory_mib"` + PidsLimit int64 `json:"pids_limit,omitempty"` + Mounts []runtime.Mount `json:"mounts,omitempty"` + Net int `json:"net"` + // Booted is set once a boot was launched: the next cold start builds a + // fresh root disk. Boots counts them, so a wait on one boot can tell it + // from the next. + Booted bool `json:"booted,omitempty"` + Boots int `json:"boots,omitempty"` + // Saved means the snapshot directory holds the VM's state and the VM is + // stopped until Restore. + Saved bool `json:"saved,omitempty"` + // ExitCode is the last boot's until the next boot exits. + Exited bool `json:"exited,omitempty"` + ExitCode int `json:"exit_code,omitempty"` +} + +func (e *engine) dir(key string) string { return filepath.Join(e.root, key) } + +func (e *engine) path(key, name string) string { return filepath.Join(e.root, key, name) } + +// imagesDir is the image store's directory beside the VMs. +const imagesDir = "images" + +func checkKey(key string) error { + if !keyPattern.MatchString(key) || key == imagesDir { + return fmt.Errorf("vm runtime: creation key %q is not a usable VM name (lowercase letters, digits and dashes)", key) + } + return nil +} + +// load returns runtime.ErrNotFound for a key with no VM. +func (e *engine) load(key string) (*record, error) { + if err := checkKey(key); err != nil { + return nil, fmt.Errorf("%w: %w", runtime.ErrNotFound, err) + } + data, err := os.ReadFile(e.path(key, recordFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("vm runtime: VM %q: %w", key, runtime.ErrNotFound) + } + if err != nil { + return nil, fmt.Errorf("vm runtime: read VM %q: %w", key, err) + } + var rec record + if err := json.Unmarshal(data, &rec); err != nil { + return nil, fmt.Errorf("vm runtime: decode VM %q: %w", key, err) + } + return &rec, nil +} + +func (e *engine) store(rec *record) error { + data, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return fmt.Errorf("vm runtime: encode VM %q: %w", rec.Key, err) + } + return writeFileAtomic(e.path(rec.Key, recordFile), data) +} + +func writeFileAtomic(path string, data []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*") + if err != nil { + return fmt.Errorf("vm runtime: write %s: %w", path, err) + } + _, werr := tmp.Write(data) + serr := tmp.Sync() + cerr := tmp.Close() + if err := errors.Join(werr, serr, cerr); err != nil { + _ = os.Remove(tmp.Name()) + return fmt.Errorf("vm runtime: write %s: %w", path, err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + _ = os.Remove(tmp.Name()) + return fmt.Errorf("vm runtime: write %s: %w", path, err) + } + return nil +} + +func (e *engine) keys() ([]string, error) { + entries, err := os.ReadDir(e.root) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("vm runtime: list VMs: %w", err) + } + var keys []string + for _, entry := range entries { + if !entry.IsDir() || checkKey(entry.Name()) != nil { + continue + } + if _, err := os.Stat(e.path(entry.Name(), recordFile)); err == nil { + keys = append(keys, entry.Name()) + } + } + return keys, nil +} diff --git a/internal/runtime/vm/save.go b/internal/runtime/vm/save.go new file mode 100644 index 000000000..16ba7a973 --- /dev/null +++ b/internal/runtime/vm/save.go @@ -0,0 +1,164 @@ +package vm + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" +) + +const restoreTimeout = 60 * time.Second + +// save pauses the VM, writes its memory and device state to the snapshot +// directory and ends the monitor. A VM with a virtio-fs share is refused: a +// fresh virtiofsd after restore knows none of the guest's open files, so +// every access to the share fails with EBADF. +func (e *engine) save(ctx context.Context, key string) error { + unlock := e.lock(key) + defer unlock() + rec, err := e.requireRunning(ctx, key, "save") + if err != nil { + return err + } + p, err := planMounts(rec.Mounts, false, e.path(key, vsockSocket)) + if err != nil { + return err + } + if len(p.shares) > 0 { + targets := make([]string, len(p.mounts)) + for i, m := range p.mounts { + targets[i] = m.Target + } + return fmt.Errorf("vm runtime: cannot save VM %s: its virtio-fs mounts (%s) would come back unusable after a restore; work items 6 and 7 move them off virtio-fs", key, strings.Join(targets, ", ")) + } + api := newCHAPI(e.path(key, apiSocket)) + if err := api.pause(ctx); err != nil { + return fmt.Errorf("vm runtime: save VM %s: %w", key, err) + } + dir := e.path(key, snapshotDir) + if err := os.RemoveAll(dir); err != nil { + return e.resumeAfter(ctx, api, key, err) + } + if err := os.Mkdir(dir, 0o700); err != nil { + return e.resumeAfter(ctx, api, key, err) + } + if err := api.snapshot(ctx, dir); err != nil { + _ = os.RemoveAll(dir) + return e.resumeAfter(ctx, api, key, err) + } + rec.Saved = true + if err := e.store(rec); err != nil { + return e.resumeAfter(ctx, api, key, err) + } + e.forgetHelper(key) + if err := api.shutdownVMM(ctx); err != nil { + return e.units.Stop(context.WithoutCancel(ctx), unitName(key)) + } + return e.awaitStopped(ctx, key) +} + +func (e *engine) resumeAfter(ctx context.Context, api chAPI, key string, cause error) error { + if err := api.resume(context.WithoutCancel(ctx)); err != nil { + return fmt.Errorf("vm runtime: save VM %s: %w; and it stays paused: %w", key, cause, err) + } + return fmt.Errorf("vm runtime: save VM %s: %w", key, cause) +} + +func (e *engine) awaitStopped(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), powerOffTimeout) + defer cancel() + for { + up, err := e.running(ctx, key) + if err == nil && !up { + return nil + } + select { + case <-ctx.Done(): + return e.units.Stop(context.WithoutCancel(ctx), unitName(key)) + case <-time.After(waitPoll): + } + } +} + +// restore starts the monitor on the saved state, resumes the guest and sets +// its clock. Once the guest runs, its disks move past the snapshot, so the +// snapshot is dropped durably before the guest resumes: a second restore of +// it would pair old memory with newer disks. Cloud Hypervisor has read all +// of it by then. +func (e *engine) restore(ctx context.Context, key string) (err error) { + unlock := e.lock(key) + defer unlock() + rec, err := e.load(key) + if err != nil { + return err + } + if !rec.Saved { + return fmt.Errorf("vm runtime: restore VM %s: it has no saved state", key) + } + if up, upErr := e.running(ctx, key); upErr != nil || up { + return errors.Join(upErr, fmt.Errorf("vm runtime: restore VM %s: it is already running", key)) + } + dir := e.path(key, snapshotDir) + l := launch{CloudHypervisor: e.cfg.CloudHypervisor, Args: []string{ + "--api-socket", "path=" + e.path(key, apiSocket), "--restore", "source_url=file://" + dir, + }} + if err = e.ensureNAT(ctx); err != nil { + return err + } + if err = e.launch(ctx, rec, l); err != nil { + return err + } + defer func() { + if err != nil { + e.kill(context.WithoutCancel(ctx), key) + } + }() + api := newCHAPI(e.path(key, apiSocket)) + if err = e.awaitRestored(ctx, key, api); err != nil { + return err + } + rec.Saved = false + if err = e.store(rec); err != nil { + return err + } + if err = os.RemoveAll(dir); err != nil { + return fmt.Errorf("vm runtime: remove the saved state of VM %s: %w", key, err) + } + if err = api.resume(ctx); err != nil { + return fmt.Errorf("vm runtime: resume restored VM %s: %w", key, err) + } + h, err := e.helper(key) + if err != nil { + return err + } + if err = e.waitHelper(ctx, key, h); err != nil { + return err + } + if err = h.SyncClock(ctx, time.Now()); err != nil { + return fmt.Errorf("vm runtime: set the clock of restored VM %s: %w", key, err) + } + return nil +} + +// awaitRestored waits until Cloud Hypervisor holds the restored, still +// paused VM. +func (e *engine) awaitRestored(ctx context.Context, key string, api chAPI) error { + ctx, cancel := context.WithTimeout(ctx, restoreTimeout) + defer cancel() + for { + state, err := api.state(ctx) + if err == nil && state == "Paused" { + return nil + } + if up, upErr := e.running(ctx, key); upErr == nil && !up { + return fmt.Errorf("vm runtime: VM %s stopped while restoring%s", key, e.logTail(key)) + } + select { + case <-ctx.Done(): + return fmt.Errorf("vm runtime: restore VM %s: %w (last state %q, %v)%s", key, ctx.Err(), state, err, e.logTail(key)) + case <-time.After(waitPoll): + } + } +} diff --git a/internal/runtime/vm/share.go b/internal/runtime/vm/share.go new file mode 100644 index 000000000..9c6767f3f --- /dev/null +++ b/internal/runtime/vm/share.go @@ -0,0 +1,120 @@ +package vm + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +// share is one host path the monitor binds into the VM's share directory, +// at share/. +type share struct { + Source string `json:"source"` + Subpath string `json:"subpath,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` +} + +// firstSocketPort leaves 1024 to the helper's own API. +const firstSocketPort = 1025 + +type plan struct { + shares []share + mounts []guest.Mount + sockets []guest.Socket + // links maps each vsock.sock_ to the host socket Cloud Hypervisor + // connects to for that port. + links map[string]string +} + +// specMounts lists a spec's mounts in Docker's order: the checkout, extra +// worktrees, then the rest. +func specMounts(spec runtime.Spec) []runtime.Mount { + var out []runtime.Mount + if spec.WorktreeHostPath != "" { + out = append(out, runtime.Mount{HostPath: spec.WorktreeHostPath, ContainerPath: spec.WorktreeMountPath}) + } + out = append(out, spec.ExtraWorktrees...) + return append(out, spec.Mounts...) +} + +// planMounts reads the host once per boot. A unix socket cannot cross +// virtio-fs, so sockets at the top of a shared directory are forwarded, and +// a directory holding nothing but sockets is not shared at all: the helper +// makes it on the guest's /run. +func planMounts(mounts []runtime.Mount, nestedDocker bool, vsock string) (plan, error) { + p := plan{links: map[string]string{}} + forward := func(host, target string) { + port := firstSocketPort + len(p.sockets) + p.sockets = append(p.sockets, guest.Socket{Port: uint32(port), Path: target}) + p.links[vsock+"_"+strconv.Itoa(port)] = host + } + for _, m := range mounts { + if m.Subpath == "" { + info, err := os.Stat(m.HostPath) + if err != nil { + return plan{}, fmt.Errorf("vm runtime: mount %s at %s: %w", m.HostPath, m.ContainerPath, err) + } + if info.Mode()&fs.ModeSocket != 0 { + forward(m.HostPath, m.ContainerPath) + continue + } + if info.IsDir() { + sockets, others, err := topSockets(m.HostPath) + if err != nil { + return plan{}, err + } + for _, name := range sockets { + forward(filepath.Join(m.HostPath, name), filepath.Join(m.ContainerPath, name)) + } + if len(sockets) > 0 && !others { + continue + } + } + } + p.mounts = append(p.mounts, guest.Mount{Volume: guest.VolumeShare, Source: strconv.Itoa(len(p.shares)), Target: m.ContainerPath, ReadOnly: m.ReadOnly}) + p.shares = append(p.shares, share{Source: m.HostPath, Subpath: m.Subpath, ReadOnly: m.ReadOnly}) + } + if nestedDocker { + p.mounts = append(p.mounts, guest.Mount{Volume: guest.VolumeData, Source: "docker", Target: "/var/lib/docker"}) + } + return p, nil +} + +func topSockets(dir string) (sockets []string, others bool, err error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, false, fmt.Errorf("vm runtime: read mount %s: %w", dir, err) + } + for _, entry := range entries { + if entry.Type()&fs.ModeSocket != 0 { + sockets = append(sockets, entry.Name()) + } else { + others = true + } + } + return sockets, others, nil +} + +func linkSockets(dir string, links map[string]string) error { + old, err := filepath.Glob(filepath.Join(dir, vsockSocket+"_*")) + if err != nil { + return err + } + for _, path := range old { + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("vm runtime: remove socket link: %w", err) + } + } + for link, target := range links { + if err := os.Symlink(target, link); err != nil { + return fmt.Errorf("vm runtime: link socket %s: %w", target, err) + } + } + return nil +} diff --git a/internal/runtime/vm/share_test.go b/internal/runtime/vm/share_test.go new file mode 100644 index 000000000..3ab928897 --- /dev/null +++ b/internal/runtime/vm/share_test.go @@ -0,0 +1,90 @@ +package vm + +import ( + "net" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/3xDevOps/Aether/internal/guest" + "github.com/3xDevOps/Aether/internal/runtime" +) + +func listen(t *testing.T, path string) { + t.Helper() + l, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = l.Close() }) +} + +func TestPlanMountsSharesFilesAndForwardsSockets(t *testing.T) { + dir, err := os.MkdirTemp("", "vmp") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + home, coord, signing := filepath.Join(dir, "home"), filepath.Join(dir, "coord"), filepath.Join(dir, "sign") + for _, d := range []string{home, coord, signing} { + if err = os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + bin := filepath.Join(dir, "aether-server") + if err = os.WriteFile(bin, nil, 0o755); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(coord, "next-command"), nil, 0o644); err != nil { + t.Fatal(err) + } + listen(t, filepath.Join(coord, "coord3.sock")) + listen(t, filepath.Join(signing, "agent.sock")) + + mounts := []runtime.Mount{ + {HostPath: home, ContainerPath: "/root"}, + {HostPath: bin, ContainerPath: "/opt/aether/aether-server", ReadOnly: true}, + {HostPath: coord, ContainerPath: "/run/aether", ReadOnly: true}, + {HostPath: signing, ContainerPath: "/run/aether-signing", ReadOnly: true}, + {HostPath: home, Subpath: ".claude", ContainerPath: "/root/.claude", ReadOnly: true}, + } + vsock := filepath.Join(dir, vsockSocket) + p, err := planMounts(mounts, true, vsock) + if err != nil { + t.Fatal(err) + } + wantShares := []share{{Source: home}, {Source: bin, ReadOnly: true}, {Source: coord, ReadOnly: true}, {Source: home, Subpath: ".claude", ReadOnly: true}} + if !slices.Equal(p.shares, wantShares) { + t.Errorf("shares = %+v", p.shares) + } + wantMounts := []guest.Mount{ + {Volume: guest.VolumeShare, Source: "0", Target: "/root"}, + {Volume: guest.VolumeShare, Source: "1", Target: "/opt/aether/aether-server", ReadOnly: true}, + {Volume: guest.VolumeShare, Source: "2", Target: "/run/aether", ReadOnly: true}, + {Volume: guest.VolumeShare, Source: "3", Target: "/root/.claude", ReadOnly: true}, + {Volume: guest.VolumeData, Source: "docker", Target: "/var/lib/docker"}, + } + if !slices.Equal(p.mounts, wantMounts) { + t.Errorf("mounts = %+v", p.mounts) + } + wantSockets := []guest.Socket{{Port: 1025, Path: "/run/aether/coord3.sock"}, {Port: 1026, Path: "/run/aether-signing/agent.sock"}} + if !slices.Equal(p.sockets, wantSockets) { + t.Errorf("sockets = %+v", p.sockets) + } + if err := linkSockets(dir, p.links); err != nil { + t.Fatal(err) + } + if target, err := os.Readlink(vsock + "_1026"); err != nil || target != filepath.Join(signing, "agent.sock") { + t.Errorf("link = %q, %v", target, err) + } + if err := linkSockets(dir, map[string]string{}); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(vsock + "_1026"); !os.IsNotExist(err) { + t.Errorf("stale link kept: %v", err) + } + if _, err := planMounts([]runtime.Mount{{HostPath: filepath.Join(dir, "gone"), ContainerPath: "/x"}}, false, vsock); err == nil { + t.Error("a missing mount source was planned") + } +} diff --git a/internal/runtime/vm/stats.go b/internal/runtime/vm/stats.go new file mode 100644 index 000000000..dcf79c73e --- /dev/null +++ b/internal/runtime/vm/stats.go @@ -0,0 +1,133 @@ +package vm + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/3xDevOps/Aether/internal/disk" + "github.com/3xDevOps/Aether/internal/runtime" +) + +const ( + statsConcurrency = 4 + statsTimeout = 3 * time.Second +) + +// memoryEvents reports a stopped VM as fs.ErrNotExist, as the OOM watcher +// treats a stopped container's missing cgroup. A paused guest cannot answer +// and kills nothing, so it reports what it last did. +func (e *engine) memoryEvents(ctx context.Context, key string) (runtime.MemoryEvents, error) { + if _, err := e.load(key); err != nil { + return runtime.MemoryEvents{}, err + } + if up, err := e.running(ctx, key); err != nil || !up { + return runtime.MemoryEvents{}, errors.Join(err, fmt.Errorf("vm runtime: VM %s is not running: %w", key, fs.ErrNotExist)) + } + if e.paused(ctx, key) { + e.mu.Lock() + events, known := e.events[key] + e.mu.Unlock() + if !known { + return events, fmt.Errorf("vm runtime: VM %s is paused: %w", key, fs.ErrNotExist) + } + return events, nil + } + h, err := e.helper(key) + if err != nil { + return runtime.MemoryEvents{}, err + } + st, err := h.Stats(ctx) + if err != nil { + return runtime.MemoryEvents{}, fmt.Errorf("vm runtime: memory events of VM %s: %w", key, err) + } + events := runtime.MemoryEvents{OOMKills: st.OOMKills, LimitBytes: st.MemoryLimitBytes} + e.mu.Lock() + e.events[key] = events + e.mu.Unlock() + return events, nil +} + +func (e *engine) paused(ctx context.Context, key string) bool { + state, err := newCHAPI(e.path(key, apiSocket)).state(ctx) + return err == nil && state == "Paused" +} + +// CPU, memory and processes come from the helper, network and disk bytes +// from Cloud Hypervisor's device counters. +func (e *engine) stats(ctx context.Context) ([]runtime.ContainerStat, error) { + ctx, cancel := context.WithTimeout(ctx, statsTimeout) + defer cancel() + keys, err := e.keys() + if err != nil { + return nil, err + } + results := make([]*runtime.ContainerStat, len(keys)) + errs := make([]error, len(keys)) + var g errgroup.Group + g.SetLimit(statsConcurrency) + for i, key := range keys { + g.Go(func() error { + results[i], errs[i] = e.sample(ctx, key) + return nil + }) + } + _ = g.Wait() + var out []runtime.ContainerStat + for _, st := range results { + if st != nil { + out = append(out, *st) + } + } + return out, errors.Join(errs...) +} + +func (e *engine) sample(ctx context.Context, key string) (*runtime.ContainerStat, error) { + rec, err := e.load(key) + if err != nil { + return nil, err + } + if up, upErr := e.running(ctx, key); upErr != nil || !up || e.paused(ctx, key) { + return nil, upErr + } + h, err := e.helper(key) + if err != nil { + return nil, err + } + g, err := h.Stats(ctx) + if err != nil { + return nil, fmt.Errorf("vm runtime: stats of VM %s: %w", key, err) + } + counters, err := newCHAPI(e.path(key, apiSocket)).counters(ctx) + if err != nil { + return nil, fmt.Errorf("vm runtime: device counters of VM %s: %w", key, err) + } + cores := rec.CPULimit + if cores == 0 { + cores = float64(rec.CPUs) + } + return &runtime.ContainerStat{ + ID: idPrefix + key, CreationKey: key, Read: time.Now(), + CPUUsageNanos: g.CPUUsageNanos, CPULimitCores: cores, + MemoryBytes: g.MemoryBytes, MemoryLimitBytes: g.MemoryLimitBytes, Pids: g.Pids, PidsLimit: g.PidsLimit, + NetRxBytes: counters["net0"]["rx_bytes"], NetTxBytes: counters["net0"]["tx_bytes"], + DiskReadBytes: counters["root"]["read_bytes"] + counters["data"]["read_bytes"], + DiskWriteBytes: counters["root"]["write_bytes"] + counters["data"]["write_bytes"], + }, nil +} + +func (e *engine) filesystem() (runtime.FilesystemCapacity, error) { + if err := os.MkdirAll(e.root, 0o700); err != nil { + return runtime.FilesystemCapacity{}, err + } + usage, err := disk.Filesystem(e.root) + if err != nil { + return runtime.FilesystemCapacity{}, err + } + return runtime.FilesystemCapacity{Name: "Session VM disks", TotalBytes: usage.TotalBytes, FreeBytes: usage.FreeBytes}, nil +} diff --git a/internal/runtime/vm/unit.go b/internal/runtime/vm/unit.go new file mode 100644 index 000000000..5226bdbc1 --- /dev/null +++ b/internal/runtime/vm/unit.go @@ -0,0 +1,81 @@ +package vm + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "strconv" + "strings" + + "github.com/3xDevOps/Aether/internal/runtime" +) + +type unitState struct { + Active bool + MainPID int +} + +// units runs each VM's monitor in its own transient systemd unit, outside +// the server's process tree, so a VM outlives a server restart or update. +type units interface { + Start(ctx context.Context, name string, argv, properties []string) error + Stop(ctx context.Context, name string) error + State(ctx context.Context, name string) (unitState, error) +} + +func unitName(key string) string { return "aether-vm-" + key + ".service" } + +type systemdUnits struct{} + +// Start uses --collect so a unit that failed does not keep its name from +// the next Start. +func (systemdUnits) Start(ctx context.Context, name string, argv, properties []string) error { + args := []string{"--unit=" + name, "--slice=" + runtime.SessionSlice, "--collect", "--quiet"} + for _, p := range properties { + args = append(args, "--property="+p) + } + args = append(append(args, "--"), argv...) + return systemctl(ctx, "systemd-run", args...) +} + +// Stop treats a unit systemd no longer knows as stopped. +func (systemdUnits) Stop(ctx context.Context, name string) error { + err := systemctl(ctx, "systemctl", "stop", name) + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 5 { + return nil + } + return err +} + +// State counts a unit as active while it starts, runs or reloads, so a VM +// whose monitor is still composing its share is not mistaken for gone. +func (systemdUnits) State(ctx context.Context, name string) (unitState, error) { + out, err := exec.CommandContext(ctx, "systemctl", "show", "--property=ActiveState,MainPID", name).Output() + if err != nil { + return unitState{}, fmt.Errorf("vm runtime: systemctl show %s: %w", name, err) + } + var st unitState + for line := range strings.SplitSeq(string(out), "\n") { + key, value, _ := strings.Cut(line, "=") + switch key { + case "ActiveState": + st.Active = value == "active" || value == "activating" || value == "reloading" + case "MainPID": + st.MainPID, _ = strconv.Atoi(value) + } + } + return st, nil +} + +func systemctl(ctx context.Context, name string, args ...string) error { + var stderr bytes.Buffer + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("vm runtime: %s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return nil +} diff --git a/internal/runtime/vm/vm.go b/internal/runtime/vm/vm.go new file mode 100644 index 000000000..c583f5ad7 --- /dev/null +++ b/internal/runtime/vm/vm.go @@ -0,0 +1,270 @@ +// Package vm runs each session in its own Cloud Hypervisor virtual machine +// behind runtime.Runtime. Every other container kind, and every image +// operation, still goes to Docker until it moves into a VM as well. +package vm + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/3xDevOps/Aether/internal/coordtransport" + "github.com/3xDevOps/Aether/internal/runtime" +) + +// idPrefix marks a VM's runtime.ID; Docker IDs are hex and never carry it. +const idPrefix = "vm-" + +// Runtime sends session containers to VMs and everything else to the +// embedded Docker runtime, whose optional methods it keeps. +type Runtime struct { + *runtime.Docker + vms *engine +} + +var ( + _ runtime.Runtime = (*Runtime)(nil) + _ runtime.ManagedExecRuntime = (*Runtime)(nil) + _ runtime.SudoRuntime = (*Runtime)(nil) + _ runtime.CapacityRuntime = (*Runtime)(nil) + _ runtime.InitPIDRuntime = (*Runtime)(nil) + _ runtime.MemoryEventsRuntime = (*Runtime)(nil) + _ runtime.BrowserRuntime = (*Runtime)(nil) + _ runtime.SaveRuntime = (*Runtime)(nil) +) + +// Config names what a VM is built from. Monitor is the command that runs +// Monitor inside each VM's systemd unit: the server binary and its +// vm-monitor subcommand. +type Config struct { + Docker *runtime.Docker + DataDir string + Images Images + Monitor []string + Helper string + CloudHypervisor string + Virtiofsd string +} + +// New returns a Runtime that keeps its VMs under DataDir/vm. +func New(cfg Config) (*Runtime, error) { + if cfg.Docker == nil || cfg.Images == nil || len(cfg.Monitor) == 0 || cfg.Helper == "" { + return nil, errors.New("vm runtime: Docker, the image store, the monitor command and the helper binary are required") + } + if cfg.CloudHypervisor == "" { + cfg.CloudHypervisor = "cloud-hypervisor" + } + if cfg.Virtiofsd == "" { + cfg.Virtiofsd = "/usr/libexec/virtiofsd" + } + return &Runtime{Docker: cfg.Docker, vms: newEngine(cfg, systemdUnits{}, dialHelper)}, nil +} + +// isSession is the one routing rule: a session container carries its run ID +// both as creation key and in AETHER_RUN_ID. Environment terminals, the +// assistant, the agent updater and the cleanup helper carry no run ID. +func isSession(spec runtime.Spec) bool { + return spec.CreationKey != "" && spec.Env[coordtransport.RunIDEnv] == spec.CreationKey +} + +func vmKey(id runtime.ID) (string, bool) { + return strings.CutPrefix(string(id), idPrefix) +} + +func (r *Runtime) Create(ctx context.Context, spec runtime.Spec) (runtime.ID, error) { + if !isSession(spec) { + return r.Docker.Create(ctx, spec) + } + return r.vms.create(ctx, spec) +} + +func (r *Runtime) Start(ctx context.Context, id runtime.ID) error { + if key, ok := vmKey(id); ok { + return r.vms.start(ctx, key) + } + return r.Docker.Start(ctx, id) +} + +func (r *Runtime) Pause(ctx context.Context, id runtime.ID) error { + if key, ok := vmKey(id); ok { + return r.vms.pause(ctx, key) + } + return r.Docker.Pause(ctx, id) +} + +func (r *Runtime) Resume(ctx context.Context, id runtime.ID) error { + if key, ok := vmKey(id); ok { + return r.vms.resume(ctx, key) + } + return r.Docker.Resume(ctx, id) +} + +func (r *Runtime) Stop(ctx context.Context, id runtime.ID, grace time.Duration) error { + if key, ok := vmKey(id); ok { + return r.vms.stop(ctx, key, grace) + } + return r.Docker.Stop(ctx, id, grace) +} + +func (r *Runtime) Destroy(ctx context.Context, id runtime.ID) error { + if key, ok := vmKey(id); ok { + return r.vms.destroy(ctx, key) + } + return r.Docker.Destroy(ctx, id) +} + +func (r *Runtime) Attach(ctx context.Context, id runtime.ID) (runtime.Attachment, error) { + if key, ok := vmKey(id); ok { + return r.vms.attach(ctx, key) + } + return r.Docker.Attach(ctx, id) +} + +func (r *Runtime) ContainerIP(ctx context.Context, id runtime.ID) (string, error) { + if key, ok := vmKey(id); ok { + return r.vms.containerIP(ctx, key) + } + return r.Docker.ContainerIP(ctx, id) +} + +func (r *Runtime) ExecTTY(ctx context.Context, id runtime.ID, argv []string, workDir string, cols, rows uint) (runtime.Attachment, error) { + if key, ok := vmKey(id); ok { + return r.vms.execTTY(ctx, key, argv, workDir, cols, rows) + } + return r.Docker.ExecTTY(ctx, id, argv, workDir, cols, rows) +} + +func (r *Runtime) Exec(ctx context.Context, id runtime.ID, argv []string, workDir string) (int, string, string, error) { + if key, ok := vmKey(id); ok { + return r.vms.exec(ctx, key, argv, workDir) + } + return r.Docker.Exec(ctx, id, argv, workDir) +} + +func (r *Runtime) Wait(ctx context.Context, id runtime.ID) (runtime.ExitStatus, error) { + if key, ok := vmKey(id); ok { + return r.vms.wait(ctx, key) + } + return r.Docker.Wait(ctx, id) +} + +func (r *Runtime) Inspect(ctx context.Context, id runtime.ID) (runtime.ContainerInfo, error) { + if key, ok := vmKey(id); ok { + return r.vms.inspect(ctx, key) + } + return r.Docker.Inspect(ctx, id) +} + +// FindByCreationKey looks among VMs first: a session's key names a VM only +// when it was created under this runtime. +func (r *Runtime) FindByCreationKey(ctx context.Context, key string) (runtime.ID, error) { + id, err := r.vms.find(key) + if !errors.Is(err, runtime.ErrNotFound) { + return id, err + } + return r.Docker.FindByCreationKey(ctx, key) +} + +func (r *Runtime) Commit(ctx context.Context, id runtime.ID, tag string, labels map[string]string) error { + if _, ok := vmKey(id); ok { + return errors.New("vm runtime: Commit (Save environment) of a session VM is not supported until work item 9") + } + return r.Docker.Commit(ctx, id, tag, labels) +} + +func (r *Runtime) InitPID(ctx context.Context, id runtime.ID) (int, error) { + if key, ok := vmKey(id); ok { + return r.vms.initPID(ctx, key) + } + return r.Docker.InitPID(ctx, id) +} + +func (r *Runtime) MemoryEvents(ctx context.Context, id runtime.ID) (runtime.MemoryEvents, bool, error) { + key, ok := vmKey(id) + if !ok { + return runtime.MemoryEvents{}, false, nil + } + events, err := r.vms.memoryEvents(ctx, key) + return events, true, err +} + +func (r *Runtime) StartExecTTY(ctx context.Context, id runtime.ID, spec runtime.ExecSpec) (runtime.ManagedExec, error) { + if key, ok := vmKey(id); ok { + return r.vms.startManagedExec(ctx, key, spec, true) + } + return r.Docker.StartExecTTY(ctx, id, spec) +} + +func (r *Runtime) StartExecPipe(ctx context.Context, id runtime.ID, spec runtime.ExecSpec) (runtime.ManagedExec, error) { + if key, ok := vmKey(id); ok { + return r.vms.startManagedExec(ctx, key, spec, false) + } + return r.Docker.StartExecPipe(ctx, id, spec) +} + +func (r *Runtime) RecoverExec(ctx context.Context, identity runtime.ExecIdentity) (runtime.ManagedExec, error) { + if key, ok := vmKey(identity.ContainerID); ok { + return r.vms.recoverExec(ctx, key, identity) + } + return r.Docker.RecoverExec(ctx, identity) +} + +func (r *Runtime) SettingUp(id runtime.ID) bool { + if key, ok := vmKey(id); ok { + return r.vms.settingUp(key) + } + return r.Docker.SettingUp(id) +} + +func (r *Runtime) ExecAttached(ctx context.Context, id runtime.ID, spec runtime.AttachedExec) (int, error) { + if key, ok := vmKey(id); ok { + return r.vms.execAttached(ctx, key, spec) + } + return r.Docker.ExecAttached(ctx, id, spec) +} + +func (r *Runtime) CreateBrowser(ctx context.Context, spec runtime.BrowserSpec) (runtime.ID, error) { + if _, ok := vmKey(spec.RunContainer); ok { + return "", errors.New("vm runtime: the development browser cannot run beside a session VM until work item 9 moves it into the VM") + } + return r.Docker.CreateBrowser(ctx, spec) +} + +func (r *Runtime) Capacity(ctx context.Context) (runtime.HostCapacity, error) { + capacity, err := r.Docker.Capacity(ctx) + if err != nil { + return capacity, err + } + fs, err := r.vms.filesystem() + if err != nil { + return runtime.HostCapacity{}, fmt.Errorf("vm runtime: VM disk capacity: %w", err) + } + capacity.Filesystems = append(capacity.Filesystems, fs) + return capacity, nil +} + +// ContainerStats adds every running VM to Docker's sample. A VM that cannot +// be sampled is left out and named in the error, like a failed container. +func (r *Runtime) ContainerStats(ctx context.Context) ([]runtime.ContainerStat, error) { + stats, dockerErr := r.Docker.ContainerStats(ctx) + vms, vmErr := r.vms.stats(ctx) + return append(stats, vms...), errors.Join(dockerErr, vmErr) +} + +func (r *Runtime) Save(ctx context.Context, id runtime.ID) error { + key, ok := vmKey(id) + if !ok { + return errors.New("vm runtime: only a session VM can be saved; Docker containers cannot") + } + return r.vms.save(ctx, key) +} + +func (r *Runtime) Restore(ctx context.Context, id runtime.ID) error { + key, ok := vmKey(id) + if !ok { + return errors.New("vm runtime: only a session VM can be restored; Docker containers cannot") + } + return r.vms.restore(ctx, key) +} diff --git a/internal/runtime/vm/vm_test.go b/internal/runtime/vm/vm_test.go new file mode 100644 index 000000000..29a84a58f --- /dev/null +++ b/internal/runtime/vm/vm_test.go @@ -0,0 +1,78 @@ +package vm + +import ( + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/3xDevOps/Aether/internal/runtime" +) + +func TestIsSessionNeedsTheRunIDTwice(t *testing.T) { + for _, tc := range []struct { + spec runtime.Spec + want bool + }{ + {runtime.Spec{CreationKey: "run-a", Env: map[string]string{"AETHER_RUN_ID": "run-a"}}, true}, + {runtime.Spec{CreationKey: "terminal:m1"}, false}, + {runtime.Spec{CreationKey: "aether-agent-update-x", Env: map[string]string{"AETHER_RUN_ID": "run-a"}}, false}, + {runtime.Spec{Env: map[string]string{"AETHER_RUN_ID": ""}}, false}, + } { + if got := isSession(tc.spec); got != tc.want { + t.Errorf("isSession(%+v) = %v", tc.spec, got) + } + } +} + +// dockerOnly takes IDs that only Docker ever hands out. +var dockerOnly = []string{"DictationState", "InspectBrowser"} + +// TestEveryContainerMethodIsRouted fails when Docker gains a method that +// takes a container, unless Runtime routes it or it is listed above: a +// promoted method would hand a VM's ID to Docker. +func TestEveryContainerMethodIsRouted(t *testing.T) { + declared := map[string]bool{} + names, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, name := range names { + if strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(token.NewFileSet(), name, nil, 0) + if err != nil { + t.Fatal(err) + } + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil { + continue + } + if star, ok := fn.Recv.List[0].Type.(*ast.StarExpr); ok && star.X.(*ast.Ident).Name == "Runtime" { + declared[fn.Name.Name] = true + } + } + } + container := map[reflect.Type]bool{ + reflect.TypeFor[runtime.ID](): true, reflect.TypeFor[runtime.ExecIdentity](): true, reflect.TypeFor[runtime.BrowserSpec](): true, + reflect.TypeFor[runtime.Spec](): true, + } + docker := reflect.TypeFor[*runtime.Docker]() + for i := range docker.NumMethod() { + m := docker.Method(i) + for j := 1; j < m.Type.NumIn(); j++ { + if container[m.Type.In(j)] && !declared[m.Name] && !slices.Contains(dockerOnly, m.Name) { + t.Errorf("Runtime does not route %s, which takes a %s", m.Name, m.Type.In(j)) + } + } + } + if !declared["FindByCreationKey"] || !declared["ContainerStats"] || !declared["Capacity"] { + t.Error("Runtime must also route FindByCreationKey, ContainerStats and Capacity") + } +} diff --git a/internal/scheduler/oomwatch.go b/internal/scheduler/oomwatch.go index d022518d3..e9fe8ea8f 100644 --- a/internal/scheduler/oomwatch.go +++ b/internal/scheduler/oomwatch.go @@ -102,6 +102,12 @@ func (w *oomWatcher) forget(id runtime.ID) { } func (w *oomWatcher) read(ctx context.Context, pids runtime.InitPIDRuntime, id runtime.ID) (memoryStats, error) { + if guest, ok := pids.(runtime.MemoryEventsRuntime); ok { + events, own, err := guest.MemoryEvents(ctx, id) + if own { + return memoryStats{oomKills: events.OOMKills, max: events.LimitBytes}, err + } + } path, ok := w.paths[id] if !ok { pid, err := pids.InitPID(ctx, id) diff --git a/internal/scheduler/oomwatch_test.go b/internal/scheduler/oomwatch_test.go index 3a7ae8f2a..e6091ace2 100644 --- a/internal/scheduler/oomwatch_test.go +++ b/internal/scheduler/oomwatch_test.go @@ -1,9 +1,12 @@ package scheduler import ( + "context" "os" "path/filepath" "testing" + + "github.com/3xDevOps/Aether/internal/runtime" ) func TestReadMemoryStatsAndMessage(t *testing.T) { @@ -54,3 +57,43 @@ func TestReadMemoryStatsAndMessage(t *testing.T) { t.Fatalf("missing memory.events error = %v, want not-exist", err) } } + +type guestMemory struct { + pid int + own bool + events runtime.MemoryEvents +} + +func (g guestMemory) InitPID(context.Context, runtime.ID) (int, error) { return g.pid, nil } + +func (g guestMemory) MemoryEvents(context.Context, runtime.ID) (runtime.MemoryEvents, bool, error) { + return g.events, g.own, nil +} + +func TestOOMWatcherReadsGuestKillsFromTheRuntime(t *testing.T) { + proc, cg := t.TempDir(), t.TempDir() + if err := os.MkdirAll(filepath.Join(proc, "7"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, "7", "cgroup"), []byte("0::/aether.slice/run.scope\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(cg, "aether.slice", "run.scope"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cg, "aether.slice", "run.scope", "memory.events"), []byte("oom_kill 2\n"), 0o644); err != nil { + t.Fatal(err) + } + w := &oomWatcher{checked: true, enabled: true, cgroup: cg, proc: proc, paths: map[runtime.ID]string{}, kills: map[runtime.ID]uint64{}} + + guest := guestMemory{pid: 7, own: true, events: runtime.MemoryEvents{OOMKills: 5, LimitBytes: 4 << 30}} + st, err := w.read(t.Context(), guest, "vm-run-a") + if err != nil || st.oomKills != 5 || st.max != 4<<30 { + t.Fatalf("guest read = %+v, %v", st, err) + } + host := guestMemory{pid: 7} + st, err = w.read(t.Context(), host, "container-b") + if err != nil || st.oomKills != 2 { + t.Fatalf("cgroup fallback = %+v, %v", st, err) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index eb17dc649..d7231733b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -4,6 +4,7 @@ package server import ( + "cmp" "context" "errors" "fmt" @@ -185,6 +186,14 @@ type Config struct { // it registered, empty means the Docker daemon default. ContainerRuntime string + // SessionRuntime is where sessions run: SessionRuntimeDocker (empty + // means the same) or SessionRuntimeVM, one Cloud Hypervisor VM each. + // Every other container stays on Docker either way. + SessionRuntime string + // VMKernel is the guest kernel session VMs boot; empty means + // DefaultVMKernel. + VMKernel string + // SysboxInstaller lets admins install Sysbox on the host through the // control channel. Only cmd/aether-server sets it; an embedded server // never runs host installers. @@ -193,6 +202,11 @@ type Config struct { const DefaultContainerRuntime = runtime.RuntimeAuto +const ( + SessionRuntimeDocker = "docker" + SessionRuntimeVM = "vm" +) + // Server is the assembled Aether server. type Server struct { db *store.DB @@ -311,6 +325,15 @@ func New(ctx context.Context, cfg Config) (srv *Server, err error) { return nil, err } s.rt = s.docker + switch cfg.SessionRuntime { + case "", SessionRuntimeDocker: + case SessionRuntimeVM: + if s.rt, err = newVMRuntime(cfg.DataDir, cmp.Or(cfg.VMKernel, DefaultVMKernel), s.docker); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("server: session-runtime %q: use %s or %s", cfg.SessionRuntime, SessionRuntimeDocker, SessionRuntimeVM) + } } removeAgentFiles := runtime.Remover(s.rt, cfg.StandardImage) diff --git a/internal/server/vmruntime.go b/internal/server/vmruntime.go new file mode 100644 index 000000000..73a7b1e3e --- /dev/null +++ b/internal/server/vmruntime.go @@ -0,0 +1,31 @@ +package server + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/3xDevOps/Aether/internal/runtime" + "github.com/3xDevOps/Aether/internal/runtime/vm" + "github.com/3xDevOps/Aether/internal/vmimage" +) + +// DefaultVMKernel is where the guest kernel a release ships is installed. +const DefaultVMKernel = "/usr/local/lib/aether/vmlinux" + +// newVMRuntime runs sessions in VMs whose monitor is this binary's +// vm-monitor subcommand and whose in-VM helper is this binary too. +func newVMRuntime(dataDir, kernel string, docker *runtime.Docker) (*vm.Runtime, error) { + exe, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("server: find the server binary for session VMs: %w", err) + } + images, err := vmimage.Open(filepath.Join(dataDir, "vm", "images"), kernel, docker) + if err != nil { + return nil, fmt.Errorf("server: open the VM image store: %w", err) + } + return vm.New(vm.Config{ + Docker: docker, DataDir: dataDir, Images: images, + Monitor: []string{exe, vm.MonitorCommand}, Helper: exe, + }) +}