diff --git a/lib/hyper.ex b/lib/hyper.ex index a46fed5a..2d212f5f 100644 --- a/lib/hyper.ex +++ b/lib/hyper.ex @@ -56,11 +56,12 @@ defmodule Hyper do Returns `{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}`, or `{:error, reason}` — `:agent_unavailable` if the relay/agent is not yet - reachable, `:timeout` if the gRPC deadline expired, `:not_found` if the VM - cannot be resolved (unknown vm_id, or a pid whose id/owning node is gone), - `:node_unreachable` if the owning node cannot be reached over the cluster. + reachable, `:timeout` if the call did not finish within `:timeout`, + `:not_found` if the VM cannot be resolved (unknown vm_id, or a pid whose + id/owning node is gone), `:node_unreachable` if the owning node cannot be + reached over the cluster. `opts`: `:env` (map `%{String.t() => String.t()}`), `:cwd` (string), `:timeout` - (ms, gRPC deadline forwarded to the relay; default 30 000). + (ms, deadline for the whole call, also sent to the guest; default 30 000). `argv` is executed directly, not through a shell. Use an **absolute** `argv` head (`["/bin/echo", "hi"]`): the guest runs as PID 1 with a near-empty diff --git a/lib/hyper/node/fire_vmm/agent.ex b/lib/hyper/node/fire_vmm/agent.ex index be0ff3e0..0d0181e6 100644 --- a/lib/hyper/node/fire_vmm/agent.ex +++ b/lib/hyper/node/fire_vmm/agent.ex @@ -7,17 +7,23 @@ defmodule Hyper.Node.FireVMM.Agent do path from it, so both sides agree without a process-registry lookup. """ - alias Hyper.Agent.V1.{ExecRequest, GuestAgent.Stub} - - @typep channel :: %GRPC.Channel{} + alias Hyper.Agent.V1.{ExecRequest, ExecResponse, GuestAgent} # GRPC.Status codes stored as module attributes so they can be used in # guard-equivalent pattern matches without calling functions at match time. @unavailable GRPC.Status.unavailable() @deadline_exceeded GRPC.Status.deadline_exceeded() + @cancelled GRPC.Status.cancelled() + + @exec_path "/#{GuestAgent.Service.__meta__(:name)}/Exec" + @wire_max 99_999_999 + @wire_units [{"m", 1}, {"S", 1_000}, {"M", 60_000}, {"H", 3_600_000}] @default_timeout 30_000 + @type result :: + {:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}} | {:error, term()} + @doc """ Deterministic host-side path for the per-VM relay Unix socket. @@ -35,10 +41,15 @@ defmodule Hyper.Node.FireVMM.Agent do Run `argv` in the guest VM identified by `vm_id` via gRPC through the relay. Returns `{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}`, - or `{:error, term()}`. Error values: `:agent_unavailable` (relay unreachable — - gRPC UNAVAILABLE), `:timeout` (the call did not finish within `:timeout`), - `{:bad_agent_reply, reason}` (the reply did not decode), or raw transport - errors for unexpected failures, including a reply over the relay's 16 MiB cap. + or `{:error, term()}`. Error values: `:agent_unavailable` (relay missing, + refused or closed before replying, or gRPC UNAVAILABLE), `:timeout` (the call + did not finish within `:timeout`), `{:bad_agent_reply, reason}` (the reply is + not a well-formed gRPC response), `%GRPC.RPCError{}` for any other gRPC + status, or raw transport errors for unexpected failures, including a reply + over the relay's 16 MiB cap. + + The guest gets `:timeout` as its `grpc-timeout`, rounded up, so it never gives + up before the host does. ## Options @@ -46,12 +57,10 @@ defmodule Hyper.Node.FireVMM.Agent do - `:cwd` — working directory inside the guest (`nil` → not set) - `:timeout` — deadline for the whole call in milliseconds (default: #{@default_timeout}) """ - @spec exec(Hyper.Vm.Id.t(), [String.t()], keyword()) :: - {:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}} - | {:error, term()} + @spec exec(Hyper.Vm.Id.t(), [String.t()], keyword()) :: result() def exec(vm_id, argv, opts \\ []) do - path = relay_socket_path(vm_id) timeout = Keyword.get(opts, :timeout, @default_timeout) + deadline = System.monotonic_time(:millisecond) + timeout req = %ExecRequest{ argv: argv, @@ -59,53 +68,146 @@ defmodule Hyper.Node.FireVMM.Agent do cwd: opts[:cwd] } - case GRPC.Stub.connect("unix://" <> path, adapter: GRPC.Client.Adapters.Gun) do - {:error, _} = err -> - map_grpc_error(err) + path = vm_id |> relay_socket_path() |> String.to_charlist() + + # The caller owns the connection: gun monitors its owner, so a killed + # caller takes the socket, and the relay's vsock leg, down with it, without + # draining the hung stream for gun's default 15 s closing_timeout. With + # retry: 0 an unreachable relay is a prompt :down instead of reconnects. + {:ok, conn} = + :gun.open_unix(path, %{ + protocols: [:http2], + retry: 0, + http2_opts: %{closing_timeout: 1} + }) + + try do + call(conn, req, timeout, deadline) + after + _ = :gun.close(conn) + :gun.flush(conn) + end + end - {:ok, ch} -> - try do - ch |> call(req, timeout) |> await(timeout) - after - _ = GRPC.Stub.disconnect(ch) - end + @spec call(pid(), ExecRequest.t(), non_neg_integer(), integer()) :: result() + defp call(conn, req, timeout, deadline) do + case :gun.await_up(conn, remaining(deadline)) do + {:ok, :http2} -> + msg = ExecRequest.encode(req) + body = <<0, byte_size(msg)::32, msg::binary>> + ref = :gun.post(conn, @exec_path, headers(timeout), body) + await_response(conn, ref, deadline) + + {:error, :timeout} -> + {:error, :timeout} + + {:error, _} -> + {:error, :agent_unavailable} end end - @spec call(channel(), ExecRequest.t(), timeout()) :: Task.t() - defp call(ch, req, timeout) do - Task.async(fn -> - try do - Stub.exec(ch, req, timeout: timeout) - rescue - e -> {:error, {:bad_agent_reply, e.__struct__}} - catch - :exit, reason -> {:error, {:bad_agent_reply, reason}} - end + @spec headers(non_neg_integer()) :: [{String.t(), String.t()}] + defp headers(timeout) do + [ + {"content-type", "application/grpc"}, + {"te", "trailers"}, + {"grpc-timeout", wire_timeout(timeout)} + ] + end + + # grpc-timeout carries at most 8 digits: take the finest unit that fits and + # round up, so the guest never gives up before the host's deadline. + @spec wire_timeout(non_neg_integer()) :: String.t() + defp wire_timeout(ms) do + Enum.find_value(@wire_units, "#{@wire_max}H", fn {unit, unit_ms} -> + n = div(ms + unit_ms - 1, unit_ms) + if n <= @wire_max, do: "#{n}#{unit}" end) end - @spec await(Task.t(), timeout()) :: - {:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}} | {:error, term()} - defp await(task, timeout) do - case Task.yield(task, timeout) || Task.shutdown(task, :brutal_kill) do - {:ok, {:ok, resp}} -> - {:ok, %{exit_code: resp.exit_code, stdout: resp.stdout, stderr: resp.stderr}} + @spec await_response(pid(), reference(), integer()) :: result() + defp await_response(conn, ref, deadline) do + case :gun.await(conn, ref, remaining(deadline)) do + {:response, fin, 200, headers} -> + cond do + not grpc?(headers) -> {:error, {:bad_agent_reply, {:content_type, ctype(headers)}}} + fin == :fin -> reply(headers, "") + true -> await_body(conn, ref, deadline, []) + end - {:ok, err} -> - map_grpc_error(err) + {:response, _fin, status, _headers} -> + {:error, {:bad_agent_reply, {:http_status, status}}} - _ -> + {:error, :timeout} -> {:error, :timeout} + + {:error, _} -> + {:error, :agent_unavailable} + + _ -> + await_response(conn, ref, deadline) end end - @spec map_grpc_error({:error, term()}) :: {:error, term()} - defp map_grpc_error({:error, %GRPC.RPCError{status: @unavailable}}), - do: {:error, :agent_unavailable} + @spec await_body(pid(), reference(), integer(), iodata()) :: result() + defp await_body(conn, ref, deadline, acc) do + case :gun.await(conn, ref, remaining(deadline)) do + {:data, :nofin, data} -> await_body(conn, ref, deadline, [acc, data]) + {:data, :fin, _data} -> {:error, {:bad_agent_reply, :no_trailers}} + {:trailers, trailers} -> reply(trailers, IO.iodata_to_binary(acc)) + {:error, _} = err -> err + _ -> await_body(conn, ref, deadline, acc) + end + end - defp map_grpc_error({:error, %GRPC.RPCError{status: @deadline_exceeded}}), - do: {:error, :timeout} + @spec reply([{binary(), binary()}], binary()) :: result() + defp reply(trailers, body) do + case {trailers |> header("grpc-status") |> Num.Int.parse_unsigned(), body} do + {{:ok, 0}, <<0, len::32, msg::binary-size(len)>>} -> decode(msg) + {{:ok, 0}, _} -> {:error, {:bad_agent_reply, :framing}} + {{:ok, code}, _} -> map_status(code, trailers |> header("grpc-message") |> percent_decode()) + {:error, _} -> {:error, {:bad_agent_reply, :grpc_status}} + end + end + + @spec decode(binary()) :: result() + defp decode(msg) do + resp = ExecResponse.decode(msg) + {:ok, %{exit_code: resp.exit_code, stdout: resp.stdout, stderr: resp.stderr}} + rescue + e -> {:error, {:bad_agent_reply, e.__struct__}} + end + + # tonic answers a lapsed grpc-timeout with CANCELLED "Timeout expired", not + # DEADLINE_EXCEEDED. + @spec map_status(pos_integer(), binary()) :: {:error, term()} + defp map_status(@unavailable, _message), do: {:error, :agent_unavailable} + defp map_status(@deadline_exceeded, _message), do: {:error, :timeout} + defp map_status(@cancelled, "Timeout expired"), do: {:error, :timeout} + defp map_status(code, message), do: {:error, GRPC.RPCError.exception(code, message)} + + @spec grpc?([{binary(), binary()}]) :: boolean() + defp grpc?(headers), do: String.starts_with?(ctype(headers), "application/grpc") + + @spec ctype([{binary(), binary()}]) :: binary() + defp ctype(headers), do: header(headers, "content-type") + + @spec header([{binary(), binary()}], String.t()) :: binary() + defp header(headers, name) do + case List.keyfind(headers, name, 0) do + {^name, value} -> value + nil -> "" + end + end + + @spec percent_decode(binary()) :: binary() + defp percent_decode(message) do + URI.decode(message) + rescue + ArgumentError -> message + end - defp map_grpc_error({:error, _} = err), do: err + @spec remaining(integer()) :: non_neg_integer() + defp remaining(deadline) when is_integer(deadline), + do: max(deadline - System.monotonic_time(:millisecond), 0) end diff --git a/test/e2e/agent_exec_bounds_test.exs b/test/e2e/agent_exec_bounds_test.exs new file mode 100644 index 00000000..af9c5225 --- /dev/null +++ b/test/e2e/agent_exec_bounds_test.exs @@ -0,0 +1,79 @@ +defmodule Hyper.E2e.AgentExecBoundsTest do + @moduledoc """ + Live contract of `Hyper.exec/3`'s bounds (#132) against a real guest: + + - an honest reply under the relay's 16 MiB cap arrives byte-for-byte; + - a reply over the cap is an error that is not `:timeout`, returned well + within the deadline instead of being buffered, and the VM stays usable: + the next exec succeeds; + - a command that outlives `:timeout` is `{:error, :timeout}` no later than + the deadline plus slack, and the VM stays usable. + + Runs only under `--only integration` on a host provisioned per + docs/cookbook/install.md. + """ + use ExUnit.Case, async: false + + import Hyper.E2e + + alias Unit.Information + + @moduletag :integration + @moduletag timeout: :timer.minutes(10) + + @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19") + @slack_ms 2_000 + + setup_all do + {:ok, img_id} = Hyper.Img.OciLoader.load(@image) + {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) + on_exit(fn -> Hyper.Node.stop_image_vm(vm) end) + {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"]) + %{vm: vm} + end + + defp zeros(bytes), do: ["/bin/sh", "-c", "head -c #{bytes} /dev/zero"] + + defp assert_still_usable(vm) do + assert {:ok, %{exit_code: 0, stdout: "alive\n"}} = + Hyper.exec(vm, ["/bin/echo", "alive"], timeout: 10_000) + end + + test "an honest reply under the cap arrives intact", %{vm: vm} do + bytes = Information.as_bytes(Information.mib(8)) + + assert {:ok, %{exit_code: 0, stdout: stdout}} = Hyper.exec(vm, zeros(bytes), timeout: 30_000) + assert byte_size(stdout) == bytes + assert stdout == :binary.copy(<<0>>, bytes) + end + + test "a reply over the cap is a prompt non-timeout error and the VM stays usable", %{vm: vm} do + deadline_ms = 30_000 + bytes = Information.as_bytes(Information.mib(24)) + + {elapsed_us, result} = + :timer.tc(fn -> Hyper.exec(vm, zeros(bytes), timeout: deadline_ms) end) + + assert {:error, reason} = result + assert reason != :timeout + assert elapsed_us < deadline_ms * 1_000 + assert_still_usable(vm) + end + + test "a command outliving the deadline is :timeout at the deadline and the VM stays usable", + %{vm: vm} do + observed = + for deadline_ms <- [700, 1_500, 4_000] do + {elapsed_us, result} = + :timer.tc(fn -> Hyper.exec(vm, ["/bin/sleep", "20"], timeout: deadline_ms) end) + + elapsed_ms = div(elapsed_us, 1_000) + + {deadline_ms, result, elapsed_ms >= deadline_ms - 50, + elapsed_ms < deadline_ms + @slack_ms} + end + + assert observed == for(d <- [700, 1_500, 4_000], do: {d, {:error, :timeout}, true, true}) + assert_still_usable(vm) + end +end diff --git a/test/hyper/node/fire_vmm/agent_deadline_test.exs b/test/hyper/node/fire_vmm/agent_deadline_test.exs new file mode 100644 index 00000000..10415ba4 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent_deadline_test.exs @@ -0,0 +1,150 @@ +defmodule Hyper.Node.FireVMM.AgentDeadlineTest do + @moduledoc """ + `Agent.exec/3`'s `:timeout` is the deadline for the *whole* call (#132): + + - an unreachable relay (no socket file, or a stale one nobody listens on) + is `{:error, :agent_unavailable}`, decided within the deadline rather than + after the transport's own connect retries; + - a relay that accepts and never answers is `{:error, :timeout}`, returned + no earlier than the deadline and no later than the deadline plus slack; + - whatever the outcome, the call leaves nothing behind in the caller: no + stray message in its mailbox (even when it traps exits) and no gRPC + connection process. + + `async: false`: `Hyper.Cfg.Toml.put_cache/1` is process-global. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Agent.V1.ExecResponse + alias Hyper.Node.FireVMM.Agent + + @slack_ms 1_000 + + defmodule Echo do + @moduledoc false + use GRPC.Server, service: Hyper.Agent.V1.GuestAgent.Service + + def exec(%{argv: ["/bin/echo" | rest]}, _stream), + do: %ExecResponse{exit_code: 0, stdout: Enum.join(rest, " ")} + + def health(_req, _stream), do: %Hyper.Agent.V1.HealthResponse{ok: true} + end + + setup_all do + {:ok, _} = Application.ensure_all_started(:grpc) + {:ok, _} = Application.ensure_all_started(:cowboy) + original = Hyper.Cfg.Toml.reload() + tmp = Path.join(System.tmp_dir!(), "agent-deadline-#{System.unique_integer([:positive])}") + :ok = Hyper.Cfg.Toml.put_cache(Map.put(original, "work_dir", tmp)) + File.mkdir_p!(Hyper.Cfg.Dirs.socket_dir()) + + {:ok, _pid, _port} = + GRPC.Server.start(Echo, 0, adapter_opts: [ip: {:local, Agent.relay_socket_path("vecho")}]) + + stale = Agent.relay_socket_path("vstale") + {:ok, s} = :socket.open(:local, :stream) + :ok = :socket.bind(s, %{family: :local, path: stale}) + :ok = :socket.close(s) + true = File.exists?(stale) + + {:ok, silent} = :socket.open(:local, :stream) + :ok = :socket.bind(silent, %{family: :local, path: Agent.relay_socket_path("vsilent")}) + :ok = :socket.listen(silent, 64) + acceptor = spawn(fn -> accept_and_hold(silent, []) end) + + on_exit(fn -> + Process.exit(acceptor, :kill) + GRPC.Server.stop(Echo) + Hyper.Cfg.Toml.put_cache(original) + File.rm_rf(tmp) + end) + + :ok + end + + defp accept_and_hold(listen, held) do + case :socket.accept(listen) do + {:ok, conn} -> accept_and_hold(listen, [conn | held]) + {:error, _} -> Process.sleep(:infinity) + end + end + + test "an unreachable relay is :agent_unavailable within the deadline" do + deadline_ms = 300 + + observed = + for id <- ["vmissing", "vstale"] do + {elapsed_us, result} = + :timer.tc(fn -> Agent.exec(id, ["/bin/echo"], timeout: deadline_ms) end) + + {id, result, div(elapsed_us, 1_000) < deadline_ms + @slack_ms, div(elapsed_us, 1_000)} + end + + assert Enum.map(observed, fn {id, result, in_time?, _ms} -> {id, result, in_time?} end) == + [ + {"vmissing", {:error, :agent_unavailable}, true}, + {"vstale", {:error, :agent_unavailable}, true} + ], + "observed (id, result, in_time?, ms): #{inspect(observed)}" + end + + property "a silent relay times out at the deadline, never early and never much later" do + check all(deadline_ms <- integer(50..600), max_runs: 6) do + {elapsed_us, result} = + :timer.tc(fn -> Agent.exec("vsilent", ["/bin/echo"], timeout: deadline_ms) end) + + assert result == {:error, :timeout} + assert elapsed_us >= deadline_ms * 1_000, "returned early: #{elapsed_us}us" + assert elapsed_us < (deadline_ms + @slack_ms) * 1_000, "late: #{elapsed_us}us" + end + end + + test "no outcome leaves a message in a caller that traps exits, nor a connection" do + rows = [ + {"vecho", &(&1 == {:ok, %{exit_code: 0, stdout: "hi", stderr: ""}})}, + {"vsilent", &(&1 == {:error, :timeout})}, + {"vmissing", &match?({:error, _}, &1)} + ] + + offenders = + for {id, expected?} <- rows, + {result, stray, closed?} = exec_in_trapping_caller(id), + not (expected?.(result) and stray == [] and closed?), + do: {id, result, stray, closed?} + + assert offenders == [] + end + + defp exec_in_trapping_caller(id) do + before = connections() + parent = self() + + spawn(fn -> + Process.flag(:trap_exit, true) + result = Agent.exec(id, ["/bin/echo", "hi"], timeout: 300) + Process.sleep(200) + {:messages, stray} = Process.info(self(), :messages) + send(parent, {:done, result, stray}) + end) + + assert_receive {:done, result, stray}, 10_000 + {result, stray, closed_back_to?(before)} + end + + defp closed_back_to?(count, tries \\ 100) + defp closed_back_to?(count, 0), do: connections() <= count + + defp closed_back_to?(count, tries) do + connections() <= count or + ( + Process.sleep(10) + closed_back_to?(count, tries - 1) + ) + end + + defp connections, + do: + length(DynamicSupervisor.which_children(GRPC.Client.Supervisor)) + + length(Supervisor.which_children(:gun_conns_sup)) +end diff --git a/test/hyper/node/fire_vmm/agent_hostile_test.exs b/test/hyper/node/fire_vmm/agent_hostile_test.exs index 59804fc3..54cb4565 100644 --- a/test/hyper/node/fire_vmm/agent_hostile_test.exs +++ b/test/hyper/node/fire_vmm/agent_hostile_test.exs @@ -168,5 +168,8 @@ defmodule Hyper.Node.FireVMM.AgentHostileTest do ) end - defp connections, do: length(DynamicSupervisor.which_children(GRPC.Client.Supervisor)) + defp connections, + do: + length(DynamicSupervisor.which_children(GRPC.Client.Supervisor)) + + length(Supervisor.which_children(:gun_conns_sup)) end diff --git a/test/hyper/node/fire_vmm/agent_security_test.exs b/test/hyper/node/fire_vmm/agent_security_test.exs new file mode 100644 index 00000000..10d159d5 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent_security_test.exs @@ -0,0 +1,332 @@ +defmodule Hyper.Node.FireVMM.AgentSecurityTest do + @moduledoc """ + `Agent.exec/3` against a hostile guest agent (#132). Whatever bytes the guest + answers with — wrong HTTP status, wrong content type, gRPC frames whose + declared length lies, a set compression flag nobody negotiated, non-numeric + or out-of-range `grpc-status`, a stream that ends without trailers, or one + that never ends — the call: + + - **is total** — it returns `{:ok, %{stdout: binary, stderr: binary, + exit_code: integer}}` or `{:error, _}`, and never raises, throws or exits + into the caller; + - **is bounded by its deadline** — it returns within `:timeout` plus slack; + - **leaves nothing behind** — no stray message in the caller, no gRPC + connection process. + + And independent of the reply: + + - a caller killed mid-call does not leak its gRPC connection, and the guest + sees its connection closed; + - a timed-out call through the real relay closes the guest-side (vsock) + connection, so a silent guest cannot accumulate host-held connections; + - a well-formed status-0 reply is decoded exactly, and a caller that traps + exits is left no stray `:EXIT` message by it; + - a relay whose guest is not (yet) listening, or answers the vsock CONNECT + with garbage, is `:agent_unavailable` within the deadline. + + `async: false`: `Hyper.Cfg.Toml.put_cache/1` is process-global. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Agent.V1.ExecResponse + alias Hyper.Node.FireVMM.Agent + alias Hyper.Node.FireVMM.Agent.Relay + + @slack_ms 1_000 + @fuzz_id "vhostilefuzz" + + defmodule Guest do + @moduledoc false + + def init(req, id) do + script = :persistent_term.get({__MODULE__, id}) + send(script.test, {:guest_request, self()}) + {:ok, _body, req} = :cowboy_req.read_body(req) + req = :cowboy_req.stream_reply(script.status, %{"content-type" => script.ctype}, req) + Enum.each(script.frames, &:cowboy_req.stream_body(&1, :nofin, req)) + finish(script.finish, req) + {:ok, req, id} + end + + defp finish({:trailers, trailers}, req), do: :cowboy_req.stream_trailers(trailers, req) + defp finish(:fin, req), do: :cowboy_req.stream_body("", :fin, req) + defp finish(:hang, _req), do: Process.sleep(:infinity) + end + + setup_all do + {:ok, _} = Application.ensure_all_started(:grpc) + {:ok, _} = Application.ensure_all_started(:cowboy) + original = Hyper.Cfg.Toml.reload() + tmp = Path.join(System.tmp_dir!(), "agent-sec-#{System.unique_integer([:positive])}") + :ok = Hyper.Cfg.Toml.put_cache(Map.put(original, "work_dir", tmp)) + File.mkdir_p!(Hyper.Cfg.Dirs.socket_dir()) + + dispatch = :cowboy_router.compile([{:_, [{:_, Guest, @fuzz_id}]}]) + + {:ok, _} = + :cowboy.start_clear( + {__MODULE__, @fuzz_id}, + [ip: {:local, Agent.relay_socket_path(@fuzz_id)}, port: 0], + %{env: %{dispatch: dispatch}, protocols: [:http2]} + ) + + on_exit(fn -> + :cowboy.stop_listener({__MODULE__, @fuzz_id}) + :persistent_term.erase({Guest, @fuzz_id}) + Hyper.Cfg.Toml.put_cache(original) + File.rm_rf(tmp) + end) + + :ok + end + + defp script(fields), do: :persistent_term.put({Guest, @fuzz_id}, Map.put(fields, :test, self())) + + defp grpc_frame do + gen all( + flag <- one_of([constant(0), constant(1), integer(2..255)]), + payload <- binary(max_length: 256), + declared <- + one_of([ + constant(byte_size(payload)), + integer(0..0xFFFF_FFFF), + constant(0xFFFF_FFFF) + ]) + ) do + <> + end + end + + defp grpc_status do + one_of([ + constant("0"), + member_of(["1", "4", "13", "14", "16"]), + member_of(["abc", "-1", "", " 0", "0x0", "99999999999999999999999", "1e3"]), + string(:printable, max_length: 16) + ]) + end + + defp finish do + one_of([ + gen all( + status <- grpc_status(), + message <- string(:printable, max_length: 64), + drop_status? <- boolean() + ) do + trailers = %{"grpc-message" => message} + + {:trailers, + if(drop_status?, do: trailers, else: Map.put(trailers, "grpc-status", status))} + end, + constant(:fin), + constant(:hang) + ]) + end + + defp hostile_reply do + gen all( + status <- member_of([200, 200, 200, 204, 400, 404, 500, 503]), + ctype <- + member_of(["application/grpc", "application/grpc+proto", "text/html", "", "x/y"]), + frames <- list_of(grpc_frame(), max_length: 4), + finish <- finish() + ) do + %{status: status, ctype: ctype, frames: frames, finish: finish} + end + end + + defp isolated_exec(id, opts, trap? \\ false) do + parent = self() + + {pid, ref} = + spawn_monitor(fn -> + Process.flag(:trap_exit, trap?) + {us, result} = :timer.tc(fn -> Agent.exec(id, ["/bin/true"], opts) end) + Process.sleep(100) + {:messages, stray} = Process.info(self(), :messages) + send(parent, {:exec_done, self(), us, result, stray}) + end) + + receive do + {:exec_done, ^pid, us, result, stray} -> + Process.demonitor(ref, [:flush]) + {:returned, us, result, stray} + + {:DOWN, ^ref, :process, ^pid, reason} -> + {:caller_died, reason} + after + Keyword.fetch!(opts, :timeout) + 10_000 -> {:caller_hung, pid} + end + end + + defp well_formed?({:ok, %{stdout: out, stderr: err, exit_code: code}}), + do: is_binary(out) and is_binary(err) and is_integer(code) + + defp well_formed?({:error, _}), do: true + defp well_formed?(_), do: false + + property "any hostile reply is a well-formed result within the deadline, leaving nothing behind" do + check all(reply <- hostile_reply(), deadline_ms <- integer(150..400), max_runs: 60) do + script(reply) + before = connections() + + outcome = isolated_exec(@fuzz_id, timeout: deadline_ms) + + assert {:returned, us, result, stray} = outcome, + "caller did not return: #{inspect(outcome)}" + + assert well_formed?(result), "ill-formed result #{inspect(result)} for #{inspect(reply)}" + assert us < (deadline_ms + @slack_ms) * 1_000, "took #{div(us, 1_000)}ms: #{inspect(reply)}" + assert stray == [], "left #{inspect(stray)} in the caller for #{inspect(reply)}" + assert closed_back_to?(before), "left a gRPC connection for #{inspect(reply)}" + end + end + + test "a well-formed status-0 reply decodes exactly" do + resp = %ExecResponse{exit_code: 42, stdout: <<0, 255, "out">>, stderr: "err\n"} + msg = ExecResponse.encode(resp) + + script(%{ + status: 200, + ctype: "application/grpc", + frames: [<<0, byte_size(msg)::32, msg::binary>>], + finish: {:trailers, %{"grpc-status" => "0"}} + }) + + assert {:returned, _us, {:ok, %{exit_code: 42, stdout: <<0, 255, "out">>, stderr: "err\n"}}, + []} = isolated_exec(@fuzz_id, timeout: 2_000) + end + + test "a caller that traps exits is left no stray message by a successful call" do + msg = ExecResponse.encode(%ExecResponse{exit_code: 0, stdout: "ok"}) + + script(%{ + status: 200, + ctype: "application/grpc", + frames: [<<0, byte_size(msg)::32, msg::binary>>], + finish: {:trailers, %{"grpc-status" => "0"}} + }) + + assert {:returned, _us, {:ok, %{stdout: "ok"}}, stray} = + isolated_exec(@fuzz_id, [timeout: 2_000], true) + + assert stray == [] + end + + test "a zero deadline times out at once without leaking a connection" do + script(%{status: 200, ctype: "application/grpc", frames: [], finish: :hang}) + before = connections() + + assert {:returned, us, {:error, :timeout}, []} = isolated_exec(@fuzz_id, timeout: 0) + assert us < @slack_ms * 1_000 + assert closed_back_to?(before) + end + + test "a caller killed mid-call does not leak its gRPC connection" do + script(%{status: 200, ctype: "application/grpc", frames: [], finish: :hang}) + before = connections() + + caller = spawn(fn -> Agent.exec(@fuzz_id, ["/bin/true"], timeout: 60_000) end) + assert_receive {:guest_request, handler}, 5_000 + handler_ref = Process.monitor(handler) + Process.exit(caller, :kill) + + assert closed_back_to?(before), + "killed caller left #{connections() - before} gRPC connection(s) behind" + + assert_receive {:DOWN, ^handler_ref, :process, ^handler, _}, 5_000 + end + + describe "through the real relay" do + setup do + id = "vsecrelay#{System.unique_integer([:positive])}" + vsock_path = Path.join(Hyper.Cfg.Dirs.socket_dir(), "vsock-#{id}.sock") + {:ok, fc} = :socket.open(:local, :stream) + :ok = :socket.bind(fc, %{family: :local, path: vsock_path}) + :ok = :socket.listen(fc, 16) + + start_supervised!( + {Relay, %{vm_id: id, vsock_uds: vsock_path, listen_path: Agent.relay_socket_path(id)}} + ) + + on_exit(fn -> + :socket.close(fc) + File.rm(vsock_path) + end) + + %{id: id, fc: fc} + end + + test "a timed-out call closes the guest-side connection", %{id: id, fc: fc} do + test = self() + + guests = + for _ <- 1..4 do + spawn(fn -> + {:ok, conn} = :socket.accept(fc) + {:ok, "CONNECT " <> _} = :socket.recv(conn, 0, 2_000) + :ok = :socket.send(conn, "OK 5\n") + send(test, {:guest_fate, self(), fate(conn)}) + end) + end + + for _ <- guests do + assert {:returned, _us, {:error, :timeout}, []} = isolated_exec(id, timeout: 200) + end + + for g <- guests do + assert_receive {:guest_fate, ^g, :closed}, @slack_ms * 3 + end + end + + test "a guest that refuses the vsock CONNECT is :agent_unavailable, not a full-deadline timeout", + %{id: id, fc: fc} do + rows = [ + {"garbage CONNECT reply", + fn -> + spawn(fn -> + {:ok, conn} = :socket.accept(fc) + {:ok, _} = :socket.recv(conn, 0, 2_000) + _ = :socket.send(conn, :binary.copy("A", 1_000_000)) + Process.sleep(:infinity) + end) + end}, + {"guest not listening yet", fn -> :ok = :socket.close(fc) end} + ] + + for {label, guest} <- rows do + guest.() + before = connections() + + assert {:returned, us, result, []} = isolated_exec(id, timeout: 2_000) + assert {label, result} == {label, {:error, :agent_unavailable}} + assert us < 2_000_000, "#{label}: took #{div(us, 1_000)}ms, the whole deadline" + assert closed_back_to?(before) + end + end + end + + defp fate(conn) do + case :socket.recv(conn, 0, 10_000) do + {:ok, _} -> fate(conn) + {:error, reason} -> reason + end + end + + defp closed_back_to?(count, tries \\ 200) + defp closed_back_to?(count, 0), do: connections() <= count + + defp closed_back_to?(count, tries) do + connections() <= count or + ( + Process.sleep(10) + closed_back_to?(count, tries - 1) + ) + end + + defp connections, + do: + length(DynamicSupervisor.which_children(GRPC.Client.Supervisor)) + + length(Supervisor.which_children(:gun_conns_sup)) +end diff --git a/test/hyper/node/fire_vmm/agent_wire_deadline_properties_test.exs b/test/hyper/node/fire_vmm/agent_wire_deadline_properties_test.exs new file mode 100644 index 00000000..0d758063 --- /dev/null +++ b/test/hyper/node/fire_vmm/agent_wire_deadline_properties_test.exs @@ -0,0 +1,107 @@ +defmodule Hyper.Node.FireVMM.AgentWireDeadlinePropertiesTest do + @moduledoc """ + The deadline `Agent.exec/3` puts on the wire must be the caller's own + (#132: `:timeout` is the deadline for the whole call). + + The guest agent (tonic) enforces the request's `grpc-timeout` header and + answers `CANCELLED "Timeout expired"` when it lapses. So the law is: for + every `:timeout`, the `grpc-timeout` the guest receives, decoded per the + gRPC HTTP/2 spec (`n u m S M H`), is never shorter than `:timeout` — or the + guest abandons the call before the host's deadline and the caller gets a + raw `CANCELLED` instead of `{:error, :timeout}`, early. + + The decoder below is an independent oracle written from the spec, not the + client's own codec. + + `async: false`: `Hyper.Cfg.Toml.put_cache/1` is process-global. + """ + use ExUnit.Case, async: false + use ExUnitProperties + + alias Hyper.Agent.V1.ExecResponse + alias Hyper.Node.FireVMM.Agent + + @vm_id "vwiredeadline" + + defmodule Reflect do + @moduledoc false + + def init(req, state) do + {:ok, _body, req} = :cowboy_req.read_body(req) + header = :cowboy_req.header("grpc-timeout", req, "") + msg = ExecResponse.encode(%ExecResponse{exit_code: 0, stdout: header}) + req = :cowboy_req.stream_reply(200, %{"content-type" => "application/grpc"}, req) + :cowboy_req.stream_body(<<0, byte_size(msg)::32, msg::binary>>, :nofin, req) + :cowboy_req.stream_trailers(%{"grpc-status" => "0"}, req) + {:ok, req, state} + end + end + + setup_all do + {:ok, _} = Application.ensure_all_started(:grpc) + {:ok, _} = Application.ensure_all_started(:cowboy) + original = Hyper.Cfg.Toml.reload() + tmp = Path.join(System.tmp_dir!(), "agent-wire-#{System.unique_integer([:positive])}") + :ok = Hyper.Cfg.Toml.put_cache(Map.put(original, "work_dir", tmp)) + File.mkdir_p!(Hyper.Cfg.Dirs.socket_dir()) + + dispatch = :cowboy_router.compile([{:_, [{:_, Reflect, nil}]}]) + + {:ok, _} = + :cowboy.start_clear( + {__MODULE__, @vm_id}, + [ip: {:local, Agent.relay_socket_path(@vm_id)}, port: 0], + %{env: %{dispatch: dispatch}, protocols: [:http2]} + ) + + on_exit(fn -> + :cowboy.stop_listener({__MODULE__, @vm_id}) + Hyper.Cfg.Toml.put_cache(original) + File.rm_rf(tmp) + end) + + :ok + end + + @unit_ns %{ + "n" => 1, + "u" => 1_000, + "m" => 1_000_000, + "S" => 1_000_000_000, + "M" => 60_000_000_000, + "H" => 3_600_000_000_000 + } + + defp wire_ns(header) do + {digits, unit} = String.split_at(header, -1) + assert digits =~ ~r/\A[0-9]{1,8}\z/, "malformed grpc-timeout #{inspect(header)}" + String.to_integer(digits) * Map.fetch!(@unit_ns, unit) + end + + defp wire_timeout_ms(timeout_ms) do + {:ok, %{stdout: header}} = Agent.exec(@vm_id, ["/bin/true"], timeout: timeout_ms) + {header, wire_ns(header) / 1_000_000} + end + + property "the guest is never told a shorter deadline than the caller asked for" do + check all( + timeout_ms <- + one_of([integer(1..999), integer(1_000..59_999), integer(60_000..7_200_000)]), + max_runs: 100 + ) do + {header, wire_ms} = wire_timeout_ms(timeout_ms) + + assert wire_ms >= timeout_ms, + "timeout: #{timeout_ms}ms went on the wire as #{header} (#{wire_ms}ms)" + end + end + + test "concrete deadlines the guest would cut short" do + rows = [1_500, 1_999, 30_000, 90_000, 119_000] + + shortened = + for t <- rows, {header, wire_ms} = wire_timeout_ms(t), wire_ms < t, do: {t, header} + + assert shortened == [] + end +end