Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions lib/hyper.ex
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,12 @@ defmodule Hyper do

Returns `{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}`, or
`{:error, reason}` — `:agent_unavailable` if the relay/agent is not yet
reachable, `:timeout` if the gRPC deadline expired, `:not_found` if the VM
cannot be resolved (unknown vm_id, or a pid whose id/owning node is gone),
`:node_unreachable` if the owning node cannot be reached over the cluster.
reachable, `:timeout` if the call did not finish within `:timeout`,
`:not_found` if the VM cannot be resolved (unknown vm_id, or a pid whose
id/owning node is gone), `:node_unreachable` if the owning node cannot be
reached over the cluster.
`opts`: `:env` (map `%{String.t() => String.t()}`), `:cwd` (string), `:timeout`
(ms, gRPC deadline forwarded to the relay; default 30 000).
(ms, deadline for the whole call, also sent to the guest; default 30 000).

`argv` is executed directly, not through a shell. Use an **absolute** `argv`
head (`["/bin/echo", "hi"]`): the guest runs as PID 1 with a near-empty
Expand Down
192 changes: 147 additions & 45 deletions lib/hyper/node/fire_vmm/agent.ex
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,23 @@ defmodule Hyper.Node.FireVMM.Agent do
path from it, so both sides agree without a process-registry lookup.
"""

alias Hyper.Agent.V1.{ExecRequest, GuestAgent.Stub}

@typep channel :: %GRPC.Channel{}
alias Hyper.Agent.V1.{ExecRequest, ExecResponse, GuestAgent}

# GRPC.Status codes stored as module attributes so they can be used in
# guard-equivalent pattern matches without calling functions at match time.
@unavailable GRPC.Status.unavailable()
@deadline_exceeded GRPC.Status.deadline_exceeded()
@cancelled GRPC.Status.cancelled()

@exec_path "/#{GuestAgent.Service.__meta__(:name)}/Exec"
@wire_max 99_999_999
@wire_units [{"m", 1}, {"S", 1_000}, {"M", 60_000}, {"H", 3_600_000}]

@default_timeout 30_000

@type result ::
{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}} | {:error, term()}

@doc """
Deterministic host-side path for the per-VM relay Unix socket.

Expand All @@ -35,77 +41,173 @@ defmodule Hyper.Node.FireVMM.Agent do
Run `argv` in the guest VM identified by `vm_id` via gRPC through the relay.

Returns `{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}`,
or `{:error, term()}`. Error values: `:agent_unavailable` (relay unreachable —
gRPC UNAVAILABLE), `:timeout` (the call did not finish within `:timeout`),
`{:bad_agent_reply, reason}` (the reply did not decode), or raw transport
errors for unexpected failures, including a reply over the relay's 16 MiB cap.
or `{:error, term()}`. Error values: `:agent_unavailable` (relay missing,
refused or closed before replying, or gRPC UNAVAILABLE), `:timeout` (the call
did not finish within `:timeout`), `{:bad_agent_reply, reason}` (the reply is
not a well-formed gRPC response), `%GRPC.RPCError{}` for any other gRPC
status, or raw transport errors for unexpected failures, including a reply
over the relay's 16 MiB cap.

The guest gets `:timeout` as its `grpc-timeout`, rounded up, so it never gives
up before the host does.

## Options

- `:env` — environment map `%{String.t() => String.t()}` for the guest process
- `:cwd` — working directory inside the guest (`nil` → not set)
- `:timeout` — deadline for the whole call in milliseconds (default: #{@default_timeout})
"""
@spec exec(Hyper.Vm.Id.t(), [String.t()], keyword()) ::
{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}
| {:error, term()}
@spec exec(Hyper.Vm.Id.t(), [String.t()], keyword()) :: result()
def exec(vm_id, argv, opts \\ []) do
path = relay_socket_path(vm_id)
timeout = Keyword.get(opts, :timeout, @default_timeout)
deadline = System.monotonic_time(:millisecond) + timeout

req = %ExecRequest{
argv: argv,
env: Map.new(Keyword.get(opts, :env, %{})),
cwd: opts[:cwd]
}

case GRPC.Stub.connect("unix://" <> path, adapter: GRPC.Client.Adapters.Gun) do
{:error, _} = err ->
map_grpc_error(err)
path = vm_id |> relay_socket_path() |> String.to_charlist()

# The caller owns the connection: gun monitors its owner, so a killed
# caller takes the socket, and the relay's vsock leg, down with it, without
# draining the hung stream for gun's default 15 s closing_timeout. With
# retry: 0 an unreachable relay is a prompt :down instead of reconnects.
{:ok, conn} =
:gun.open_unix(path, %{
protocols: [:http2],
retry: 0,
http2_opts: %{closing_timeout: 1}
})

try do
call(conn, req, timeout, deadline)
after
_ = :gun.close(conn)
:gun.flush(conn)
end
end

{:ok, ch} ->
try do
ch |> call(req, timeout) |> await(timeout)
after
_ = GRPC.Stub.disconnect(ch)
end
@spec call(pid(), ExecRequest.t(), non_neg_integer(), integer()) :: result()
defp call(conn, req, timeout, deadline) do
case :gun.await_up(conn, remaining(deadline)) do
{:ok, :http2} ->
msg = ExecRequest.encode(req)
body = <<0, byte_size(msg)::32, msg::binary>>
ref = :gun.post(conn, @exec_path, headers(timeout), body)
await_response(conn, ref, deadline)

{:error, :timeout} ->
{:error, :timeout}

{:error, _} ->
{:error, :agent_unavailable}
end
end

@spec call(channel(), ExecRequest.t(), timeout()) :: Task.t()
defp call(ch, req, timeout) do
Task.async(fn ->
try do
Stub.exec(ch, req, timeout: timeout)
rescue
e -> {:error, {:bad_agent_reply, e.__struct__}}
catch
:exit, reason -> {:error, {:bad_agent_reply, reason}}
end
@spec headers(non_neg_integer()) :: [{String.t(), String.t()}]
defp headers(timeout) do
[
{"content-type", "application/grpc"},
{"te", "trailers"},
{"grpc-timeout", wire_timeout(timeout)}
]
end

# grpc-timeout carries at most 8 digits: take the finest unit that fits and
# round up, so the guest never gives up before the host's deadline.
@spec wire_timeout(non_neg_integer()) :: String.t()
defp wire_timeout(ms) do
Enum.find_value(@wire_units, "#{@wire_max}H", fn {unit, unit_ms} ->
n = div(ms + unit_ms - 1, unit_ms)
if n <= @wire_max, do: "#{n}#{unit}"
end)
end

@spec await(Task.t(), timeout()) ::
{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}} | {:error, term()}
defp await(task, timeout) do
case Task.yield(task, timeout) || Task.shutdown(task, :brutal_kill) do
{:ok, {:ok, resp}} ->
{:ok, %{exit_code: resp.exit_code, stdout: resp.stdout, stderr: resp.stderr}}
@spec await_response(pid(), reference(), integer()) :: result()
defp await_response(conn, ref, deadline) do
case :gun.await(conn, ref, remaining(deadline)) do
{:response, fin, 200, headers} ->
cond do
not grpc?(headers) -> {:error, {:bad_agent_reply, {:content_type, ctype(headers)}}}
fin == :fin -> reply(headers, "")
true -> await_body(conn, ref, deadline, [])
end

{:ok, err} ->
map_grpc_error(err)
{:response, _fin, status, _headers} ->
{:error, {:bad_agent_reply, {:http_status, status}}}

_ ->
{:error, :timeout} ->
{:error, :timeout}

{:error, _} ->
{:error, :agent_unavailable}

_ ->
await_response(conn, ref, deadline)
end
end

@spec map_grpc_error({:error, term()}) :: {:error, term()}
defp map_grpc_error({:error, %GRPC.RPCError{status: @unavailable}}),
do: {:error, :agent_unavailable}
@spec await_body(pid(), reference(), integer(), iodata()) :: result()
defp await_body(conn, ref, deadline, acc) do
case :gun.await(conn, ref, remaining(deadline)) do
{:data, :nofin, data} -> await_body(conn, ref, deadline, [acc, data])
{:data, :fin, _data} -> {:error, {:bad_agent_reply, :no_trailers}}
{:trailers, trailers} -> reply(trailers, IO.iodata_to_binary(acc))
{:error, _} = err -> err
_ -> await_body(conn, ref, deadline, acc)
end
end

defp map_grpc_error({:error, %GRPC.RPCError{status: @deadline_exceeded}}),
do: {:error, :timeout}
@spec reply([{binary(), binary()}], binary()) :: result()
defp reply(trailers, body) do
case {trailers |> header("grpc-status") |> Num.Int.parse_unsigned(), body} do
{{:ok, 0}, <<0, len::32, msg::binary-size(len)>>} -> decode(msg)
{{:ok, 0}, _} -> {:error, {:bad_agent_reply, :framing}}
{{:ok, code}, _} -> map_status(code, trailers |> header("grpc-message") |> percent_decode())
{:error, _} -> {:error, {:bad_agent_reply, :grpc_status}}
end
end

@spec decode(binary()) :: result()
defp decode(msg) do
resp = ExecResponse.decode(msg)
{:ok, %{exit_code: resp.exit_code, stdout: resp.stdout, stderr: resp.stderr}}
rescue
e -> {:error, {:bad_agent_reply, e.__struct__}}
end

# tonic answers a lapsed grpc-timeout with CANCELLED "Timeout expired", not
# DEADLINE_EXCEEDED.
@spec map_status(pos_integer(), binary()) :: {:error, term()}
defp map_status(@unavailable, _message), do: {:error, :agent_unavailable}
defp map_status(@deadline_exceeded, _message), do: {:error, :timeout}
defp map_status(@cancelled, "Timeout expired"), do: {:error, :timeout}
defp map_status(code, message), do: {:error, GRPC.RPCError.exception(code, message)}

@spec grpc?([{binary(), binary()}]) :: boolean()
defp grpc?(headers), do: String.starts_with?(ctype(headers), "application/grpc")

@spec ctype([{binary(), binary()}]) :: binary()
defp ctype(headers), do: header(headers, "content-type")

@spec header([{binary(), binary()}], String.t()) :: binary()
defp header(headers, name) do
case List.keyfind(headers, name, 0) do
{^name, value} -> value
nil -> ""
end
end

@spec percent_decode(binary()) :: binary()
defp percent_decode(message) do
URI.decode(message)
rescue
ArgumentError -> message
end

defp map_grpc_error({:error, _} = err), do: err
@spec remaining(integer()) :: non_neg_integer()
defp remaining(deadline) when is_integer(deadline),
do: max(deadline - System.monotonic_time(:millisecond), 0)
end
79 changes: 79 additions & 0 deletions test/e2e/agent_exec_bounds_test.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
defmodule Hyper.E2e.AgentExecBoundsTest do
@moduledoc """
Live contract of `Hyper.exec/3`'s bounds (#132) against a real guest:

- an honest reply under the relay's 16 MiB cap arrives byte-for-byte;
- a reply over the cap is an error that is not `:timeout`, returned well
within the deadline instead of being buffered, and the VM stays usable:
the next exec succeeds;
- a command that outlives `:timeout` is `{:error, :timeout}` no later than
the deadline plus slack, and the VM stays usable.

Runs only under `--only integration` on a host provisioned per
docs/cookbook/install.md.
"""
use ExUnit.Case, async: false

import Hyper.E2e

alias Unit.Information

@moduletag :integration
@moduletag timeout: :timer.minutes(10)

@image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19")
@slack_ms 2_000

setup_all do
{:ok, img_id} = Hyper.Img.OciLoader.load(@image)
{:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro})
on_exit(fn -> Hyper.Node.stop_image_vm(vm) end)
{:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"])
%{vm: vm}
end

defp zeros(bytes), do: ["/bin/sh", "-c", "head -c #{bytes} /dev/zero"]

defp assert_still_usable(vm) do
assert {:ok, %{exit_code: 0, stdout: "alive\n"}} =
Hyper.exec(vm, ["/bin/echo", "alive"], timeout: 10_000)
end

test "an honest reply under the cap arrives intact", %{vm: vm} do
bytes = Information.as_bytes(Information.mib(8))

assert {:ok, %{exit_code: 0, stdout: stdout}} = Hyper.exec(vm, zeros(bytes), timeout: 30_000)
assert byte_size(stdout) == bytes
assert stdout == :binary.copy(<<0>>, bytes)
end

test "a reply over the cap is a prompt non-timeout error and the VM stays usable", %{vm: vm} do
deadline_ms = 30_000
bytes = Information.as_bytes(Information.mib(24))

{elapsed_us, result} =
:timer.tc(fn -> Hyper.exec(vm, zeros(bytes), timeout: deadline_ms) end)

assert {:error, reason} = result
assert reason != :timeout
assert elapsed_us < deadline_ms * 1_000
assert_still_usable(vm)
end

test "a command outliving the deadline is :timeout at the deadline and the VM stays usable",
%{vm: vm} do
observed =
for deadline_ms <- [700, 1_500, 4_000] do
{elapsed_us, result} =
:timer.tc(fn -> Hyper.exec(vm, ["/bin/sleep", "20"], timeout: deadline_ms) end)

elapsed_ms = div(elapsed_us, 1_000)

{deadline_ms, result, elapsed_ms >= deadline_ms - 50,
elapsed_ms < deadline_ms + @slack_ms}
end

assert observed == for(d <- [700, 1_500, 4_000], do: {d, {:error, :timeout}, true, true})
assert_still_usable(vm)
end
end
Loading
Loading