diff --git a/.nextchanges/cli/sandbox-ssh-port-forward.md b/.nextchanges/cli/sandbox-ssh-port-forward.md new file mode 100644 index 00000000000..e372b3e7f60 --- /dev/null +++ b/.nextchanges/cli/sandbox-ssh-port-forward.md @@ -0,0 +1 @@ +* Use local SSH keys and authenticated WebSocket port forwarding for `databricks sandbox ssh`, including sandboxes created through the public API. ([#7589](https://github.com/databricks/cli/pull/7589)) diff --git a/NOTICE b/NOTICE index fb51e5c0914..72fd47a044d 100644 --- a/NOTICE +++ b/NOTICE @@ -4,6 +4,10 @@ This Software includes software developed at Databricks (https://www.databricks. This Software contains code from the following open source projects, licensed under the Apache 2.0 license: +grpc/grpc-go - https://github.com/grpc/grpc-go +Copyright The gRPC Authors +License - https://github.com/grpc/grpc-go/blob/master/LICENSE + spf13/cobra - https://github.com/spf13/cobra Copyright cobra authors License - https://github.com/spf13/cobra/blob/main/LICENSE.txt @@ -33,6 +37,10 @@ License - https://github.com/gorilla/websocket/blob/main/LICENSE This Software contains code from the following open source projects, licensed under the BSD (3-clause) license: +protocolbuffers/protobuf-go - https://github.com/protocolbuffers/protobuf-go +Copyright The Go Authors +License - https://github.com/protocolbuffers/protobuf-go/blob/master/LICENSE + spf13/pflag - https://github.com/spf13/pflag Copyright (c) 2012 Alex Ogier. All rights reserved. Copyright (c) 2012 The Go Authors. All rights reserved. diff --git a/cmd/sandbox/sandbox.go b/cmd/sandbox/sandbox.go index 294968b3eac..8974225af09 100644 --- a/cmd/sandbox/sandbox.go +++ b/cmd/sandbox/sandbox.go @@ -18,7 +18,6 @@ tooling (Python, Node.js, Rust, Databricks CLI) and persistent storage. Getting started: databricks auth login --host https://... # authenticate to a Databricks workspace - databricks sandbox register # generate and register an SSH key databricks sandbox create # provision a sandbox (becomes your default) databricks sandbox ssh # SSH to your default sandbox diff --git a/cmd/sandbox/server.go b/cmd/sandbox/server.go index d6e1d1ce602..08f13f0c868 100644 --- a/cmd/sandbox/server.go +++ b/cmd/sandbox/server.go @@ -1,9 +1,13 @@ package sandbox import ( + "bytes" "errors" "fmt" + "io" "net" + "os" + "runtime" "time" sshtunnel "github.com/databricks/cli/experimental/ssh/sandbox" @@ -28,9 +32,15 @@ func newServerCommand() *cobra.Command { if err != nil { return fmt.Errorf("listen for SSH WebSockets: %w", err) } + output, err := openServerOutput(cmd.OutOrStdout()) + if err != nil { + listener.Close() + return fmt.Errorf("open command output: %w", err) + } + defer output.Close() return sshtunnel.Serve(cmd.Context(), listener, sshtunnel.ServerOptions{ - SSHD: sshd, Config: config, IdleTimeout: idleTimeout, - }, cmd.OutOrStdout()) + SSHD: sshd, Config: config, IdleTimeout: idleTimeout, OutputHeartbeat: 30 * time.Second, + }, output) }, } cmd.Flags().StringVar(&config, "sshd-config", "", "Prepared sshd configuration file") @@ -40,3 +50,27 @@ func newServerCommand() *cobra.Command { cmd.Flags().DurationVar(&idleTimeout, "idle-timeout", time.Minute, "Stop after this duration without an SSH connection") return cmd } + +func openServerOutput(output io.Writer) (io.WriteCloser, error) { + if _, ok := output.(*bytes.Buffer); ok || output == io.Discard { + return serverOutputWriter{output}, nil + } + file, ok := output.(*os.File) + if !ok || runtime.GOOS != "linux" { + return nil, errors.New("server output must be a Linux command-execution pipe") + } + info, err := file.Stat() + if err != nil { + return nil, err + } + if info.Mode()&os.ModeNamedPipe == 0 { + return nil, errors.New("server output must be a Linux command-execution pipe") + } + // Reopening inherited stdout registers its pipe with Go's poller, so + // Close interrupts blocked writes without changing the inherited fd. + return os.OpenFile(fmt.Sprintf("/proc/self/fd/%d", file.Fd()), os.O_WRONLY, 0) +} + +type serverOutputWriter struct{ io.Writer } + +func (serverOutputWriter) Close() error { return nil } diff --git a/cmd/sandbox/server_test.go b/cmd/sandbox/server_test.go index 95431b05b1d..dd1b3b6fdc9 100644 --- a/cmd/sandbox/server_test.go +++ b/cmd/sandbox/server_test.go @@ -1,19 +1,65 @@ package sandbox import ( + "bufio" "bytes" + "context" "encoding/json" + "fmt" + "io" "net" "os" + "os/exec" "path/filepath" "runtime" "strconv" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestServerOutputInterruptsInheritedStdout(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Linux sandbox command output uses /proc/self/fd") + } + const marker = "server output write interrupted\n" + if os.Getenv("DATABRICKS_TEST_SERVER_OUTPUT") == "1" { + output, err := openServerOutput(os.Stdout) + require.NoError(t, err) + defer output.Close() + file := output.(*os.File) + require.NoError(t, file.SetWriteDeadline(time.Now().Add(100*time.Millisecond))) + _, err = file.Write(make([]byte, 16<<20)) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + fmt.Fprint(os.Stderr, marker) + return + } + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, executable, "-test.run=^TestServerOutputInterruptsInheritedStdout$") + cmd.Env = append(os.Environ(), "DATABRICKS_TEST_SERVER_OUTPUT=1") + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + stderr, err := cmd.StderrPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + // Keep stdout full until the child's write has been interrupted. + line, err := bufio.NewReader(stderr).ReadString('\n') + require.NoError(t, err) + require.Equal(t, marker, line) + _, err = io.Copy(io.Discard, stdout) + require.NoError(t, err) + require.NoError(t, cmd.Wait()) +} + func TestServerCommandValidatesArgumentsWithoutAuthentication(t *testing.T) { for _, args := range [][]string{ {"--sshd-config", "config", "--port", "22"}, diff --git a/cmd/sandbox/ssh.go b/cmd/sandbox/ssh.go index 7fbe7df1c20..5c4cdaf705a 100644 --- a/cmd/sandbox/ssh.go +++ b/cmd/sandbox/ssh.go @@ -4,312 +4,268 @@ import ( "context" "errors" "fmt" - "io/fs" "os" + "os/signal" + "runtime" "strings" + "syscall" + "unicode" "github.com/databricks/cli/cmd/root" + sshtunnel "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/execv" "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/service/iam" + sdk "github.com/databricks/databricks-sdk-go/service/sandbox" "github.com/spf13/cobra" ) -const defaultGatewayPort = "2222" - func newSSHCommand() *cobra.Command { - var gatewayPort string - + var proxy bool + var releasesDir string + var expectedIdentity string + var configFile string cmd := &cobra.Command{ Use: "ssh [sandbox-id] [-- ...]", Short: "SSH into a Sandbox environment", - Long: `SSH into a Sandbox environment. + Long: `Connect to your default or a named sandbox through authenticated WebSocket port forwarding. -Connect to your default or a named sandbox via SSH. Extra arguments -after -- are passed directly to the ssh process. This lets you run -remote commands, set up port forwarding, or pass any other ssh flags. +The CLI creates a local SSH key automatically. No Gateway SSH key registration +or persistent SSH Host configuration is required. The sandbox must have Python 3 +and openssh-server installed, and the local machine must have OpenSSH. Examples: - databricks sandbox ssh # interactive shell on default sandbox - databricks sandbox ssh happy-panda-1234 # interactive shell on specific sandbox - databricks sandbox ssh -- ls -la /home # run command on default sandbox - databricks sandbox ssh happy-panda-1234 -- cat /etc/os-release # run command on specific sandbox - databricks sandbox ssh -- -L 8080:localhost:8080 # port forwarding on default sandbox`, - Args: cobra.ArbitraryArgs, - PreRunE: root.MustWorkspaceClient, - ValidArgsFunction: completeSandboxIDs, - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - profile := w.Config.Profile - if profile == "" { - profile = w.Config.Host + databricks sandbox ssh + databricks sandbox ssh happy-panda-1234 + databricks sandbox ssh happy-panda-1234 -- cat /etc/os-release + databricks sandbox ssh -- -L 8080:localhost:8080`, + Args: cobra.ArbitraryArgs, ValidArgsFunction: completeSandboxIDs, + PreRunE: func(cmd *cobra.Command, args []string) error { + if configFile == "" { + return root.MustWorkspaceClient(cmd, args) } - - keyPath, err := sandboxKeyPath(ctx) - if err != nil { - return fmt.Errorf("failed to determine sandbox key path: %w", err) + if !proxy { + return errors.New("--ssh-config-file requires --proxy") } - if _, err := os.Stat(keyPath); errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("sandbox SSH key not found at %s — run 'databricks sandbox register' first", keyPath) + // The SDK reads this setting from process environment during initialization. + previous, present := env.Lookup(cmd.Context(), "DATABRICKS_CONFIG_FILE") + if err := os.Setenv("DATABRICKS_CONFIG_FILE", configFile); err != nil { + return err } - - var sandboxID string - var extraArgs []string - - switch dashAt := cmd.ArgsLenAtDash(); dashAt { - case -1: - if len(args) > 0 { - sandboxID = args[0] + defer func() { + if present { + _ = os.Setenv("DATABRICKS_CONFIG_FILE", previous) + } else { + _ = os.Unsetenv("DATABRICKS_CONFIG_FILE") } - case 0: - extraArgs = args[dashAt:] - default: - sandboxID = args[0] - extraArgs = args[dashAt:] + }() + return root.MustWorkspaceClient(cmd, args) + }, + RunE: func(cmd *cobra.Command, args []string) error { + var requested string + var extra []string + dash := cmd.ArgsLenAtDash() + if dash == -1 { + dash = len(args) } - - if sandboxID != "" { - resolved, err := resolveLocalID(ctx, profile, sandboxID) - if err != nil { - return err - } - sandboxID = resolved + if dash > 1 { + return errors.New("specify at most one sandbox before --") } - - // Captured from any Sandbox response we touch below; "" when - // we never hit the API in this invocation. - var ( - sandboxGatewayHost string - sandboxStatus string - ) - - api, err := newSandboxAPI(w) + if dash == 1 { + requested = args[0] + } + extra = args[dash:] + if proxy && len(extra) != 0 { + return errors.New("--proxy does not accept SSH arguments") + } + ctx := cmd.Context() + if proxy { + var stop context.CancelFunc + ctx, stop = signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP) + defer stop() + } + w := cmdctx.WorkspaceClient(ctx) + name, err := resolveSSHName(ctx, w, requested) if err != nil { return err } - - // Surface deleted-key errors before ssh's opaque "Permission denied". - if err := verifyKeyRegistered(ctx, api, keyPath); err != nil { + identity, err := prepareSSHIdentity(ctx, w, name) + if err != nil { return err } - - if sandboxID == "" { - def := getDefault(ctx, profile) - if def == "" { - return errors.New("no default sandbox configured — run `databricks sandbox create` to provision one, or `databricks sandbox default ` to point at an existing sandbox") - } - // Confirm the saved default still exists. A stale entry is - // almost always user-correctable (deleted from another - // machine, reaped by an admin); fail fast with an actionable - // message rather than silently provisioning a fresh sandbox - // and surprising the user with a bill. - sb, err := api.get(ctx, def) - switch { - case err == nil: - sandboxID = def - sandboxGatewayHost = sb.GatewayHost - sandboxStatus = sb.Status - case errors.Is(err, apierr.ErrNotFound): - _ = clearDefault(ctx, profile) - return fmt.Errorf("saved default %q no longer exists (cleared) — run `databricks sandbox create` to provision a new one, or `databricks sandbox default ` to point at an existing sandbox", def) - default: - warn(ctx, fmt.Sprintf("Could not validate default %s: %v", def, err)) - sandboxID = def - } - } else { - // Validate the explicit ID against the server. Two reasons: - // 1. Surface `sandbox ssh fake-id` as a clear 404 instead of - // letting the user wade through `Permission denied` from - // ssh when the gateway can't route an unknown sandbox. - // 2. Capture `gateway_host` to drive the resolution below. - // Non-NotFound errors fall through so transient API hiccups - // don't block a connection the gateway can still route. - sb, err := api.get(ctx, sandboxID) - switch { - case err == nil: - sandboxGatewayHost = sb.GatewayHost - sandboxStatus = sb.Status - case errors.Is(err, apierr.ErrNotFound): - return fmt.Errorf("no sandbox named %q — `databricks sandbox list` shows available IDs", sandboxID) - default: - warn(ctx, fmt.Sprintf("Could not validate sandbox %s: %v", sandboxID, err)) - } - } - - // Drive start ourselves so the user sees a spinner instead - // of an opaque multi-minute hang inside ssh's connect path. - if sandboxStatus != "" && !strings.EqualFold(sandboxStatus, "running") { - final, err := ensureRunning(ctx, api, sandboxID, sandboxStatus) - if err != nil { - return err - } - if final.GatewayHost != "" { - sandboxGatewayHost = final.GatewayHost - } + if expectedIdentity != "" && identity.HostKeyAlias != expectedIdentity { + return errors.New("SSH workspace or caller changed; rerun ssh-setup with the intended Databricks profile") } - - // Resolution precedence: fresh API response → cached value - // for this profile. The server stamps the workspace's - // gateway on every Sandbox / SshKey response, so the cache - // is populated after the first call against the workspace - // (sandbox register / list / create / etc.). - host := sandboxGatewayHost - if host == "" { - host = getGatewayHost(ctx, profile) + if proxy { + return sshtunnel.ConnectProxy(ctx, w, name, identity, releasesDir, os.Stdin, cmd.OutOrStdout(), cmd.ErrOrStderr()) } - if host == "" { - return errors.New("could not resolve the sandbox SSH gateway — run `databricks sandbox register` first") - } - - // Persist whatever the server just told us, so the next invocation - // can short-circuit the explicit-id `get` above. - if sandboxGatewayHost != "" { - _ = setGatewayHost(ctx, profile, sandboxGatewayHost) + options, err := sshClientOptions(cmd, w.Config, name, identity, releasesDir, extra) + if err != nil { + return err } - - // No "Connecting…" spinner here. execSSHDirect replaces - // the CLI process via execv, so a deferred Close() would - // never run — and Bubble Tea hides the terminal cursor at - // init and only restores it on clean shutdown, leaving the - // cursor invisible for the entire ssh session and after it - // exits. The spinner couldn't animate during the handshake - // anyway (the CLI process is gone by then). - return execSSHDirect(sandboxID, host, gatewayPort, keyPath, extraArgs) + return execv.Execv(options) }, } - - cmd.Flags().StringVar(&gatewayPort, "port", defaultGatewayPort, "Sandbox gateway SSH port") - + cmd.Flags().BoolVar(&proxy, "proxy", false, "Relay SSH bytes over stdin/stdout without launching ssh") + cmd.Flags().MarkHidden("proxy") + cmd.Flags().StringVar(&expectedIdentity, "ssh-identity", "", "Expected local SSH identity scope") + cmd.Flags().MarkHidden("ssh-identity") + cmd.Flags().StringVar(&configFile, "ssh-config-file", "", "Databricks configuration file saved by ssh-setup") + cmd.Flags().MarkHidden("ssh-config-file") + cmd.Flags().StringVar(&releasesDir, "releases-dir", "", "Directory containing version-matched Linux CLI archives") return cmd } -// verifyKeyRegistered confirms the local sandbox public key is still -// registered with the workspace before we open the SSH socket. The -// gateway itself already does this check during userauth, but the -// SSH protocol's reply surface (USERAUTH_FAILURE has no free-form -// reason; USERAUTH_BANNER is widely swallowed) flattens "unknown -// key", "key registered but not authorized for this sandbox", and -// "upstream service is down" into the same "Permission denied -// (publickey)". This out-of-band HTTP check surfaces the specific -// case in language the user can act on. listKeys errors fall -// through with a warning so a transient API hiccup doesn't block a -// connection the gateway could still route. -func verifyKeyRegistered(ctx context.Context, api *sandboxAPI, keyPath string) error { - pub, err := os.ReadFile(keyPath + ".pub") +func sshClientOptions(cmd *cobra.Command, cfg *config.Config, name string, identity *sshtunnel.Identity, releasesDir string, extra []string) (execv.Options, error) { + executable, err := os.Executable() if err != nil { - return fmt.Errorf("reading public key %s.pub: %w", keyPath, err) + return execv.Options{}, err } - want := keyHash(string(pub)) - - keys, err := api.listKeys(ctx) - if err != nil { - warn(ctx, fmt.Sprintf("Could not verify SSH key registration: %v", err)) - return nil + profile := "" + if flag := cmd.Flag("profile"); flag != nil && flag.Changed { + profile = flag.Value.String() } - for _, k := range keys { - if k.KeyHash == want { - return nil + target := "" + for _, flagName := range []string{"target", "environment"} { + if flag := cmd.Flag(flagName); flag != nil && flag.Value.String() != "" { + target = flag.Value.String() + break } } - return fmt.Errorf("your sandbox SSH key (%s) is not registered with this workspace — run `databricks sandbox register` to re-register it", want) + transport, err := sshProxyCommand(executable, name, sshProxyOptions{Profile: profile, ReleasesDir: releasesDir, IdentityScope: identity.HostKeyAlias, Target: target}) + if err != nil { + return execv.Options{}, err + } + return execv.Options{Args: buildSSHArgs(identity, transport, extra), Env: sshProcessEnvironment(cmd.Context(), cfg)}, nil } -// ensureRunning owns the lifecycle transition so SSH does not hang inside -// connection setup. The caller must not have a spinner open. -func ensureRunning(ctx context.Context, api *sandboxAPI, id, currentStatus string) (*sandboxEntry, error) { - if strings.EqualFold(currentStatus, "running") { - return &sandboxEntry{SandboxID: id, Status: currentStatus}, nil +func sshProcessEnvironment(ctx context.Context, cfg *config.Config) []string { + variables := env.All(ctx) + variables["DATABRICKS_HOST"] = cfg.Host + variables["DATABRICKS_ACCOUNT_ID"] = cfg.AccountID + variables["DATABRICKS_WORKSPACE_ID"] = cfg.WorkspaceID + if cfg.Profile != "" { + // An implicit profile must retain environment-first auth precedence. + // Converting it to --profile would give the profile precedence instead. + variables["DATABRICKS_CONFIG_PROFILE"] = cfg.Profile } - - s := spin(ctx, "Starting "+cmdio.Bold(ctx, id)+"…") - defer s.Close() - - var sb *sandboxEntry - if strings.EqualFold(currentStatus, "stopping") { - stopped, err := waitForStopped(ctx, api, s, id) - if err != nil { - s.fail("Failed to start " + id) - return nil, err - } - currentStatus = stopped.Status - s.Update("Starting " + id + "…") + if cfg.ConfigFile != "" { + variables["DATABRICKS_CONFIG_FILE"] = cfg.ConfigFile + } + result := make([]string, 0, len(variables)) + for key, value := range variables { + result = append(result, key+"="+value) } + return result +} - if strings.EqualFold(currentStatus, "stopped") { - updated, err := api.start(ctx, id) - if err != nil { - s.fail("Failed to start " + id) - return nil, fmt.Errorf("failed to start sandbox %s: %w", id, err) +func resolveSSHName(ctx context.Context, w *databricks.WorkspaceClient, requested string) (string, error) { + profile := w.Config.Profile + if profile == "" { + profile = w.Config.Host + } + savedDefault := requested == "" + if savedDefault { + requested = getDefault(ctx, profile) + } + if requested == "" { + return "", errors.New("no default sandbox configured — specify a sandbox ID or run 'databricks sandbox default '") + } + id, err := resolveLocalID(ctx, profile, strings.TrimPrefix(requested, "sandboxes/")) + if err != nil { + return "", err + } + if id == "" || strings.ContainsAny(id, "/\\?#") || strings.ContainsFunc(id, unicode.IsSpace) { + return "", fmt.Errorf("invalid sandbox ID %q", id) + } + name := "sandboxes/" + id + if _, err := w.Sandbox.GetSandbox(ctx, sdk.GetSandboxRequest{Name: name}); err != nil { + if savedDefault && errors.Is(err, apierr.ErrNotFound) { + return "", fmt.Errorf("saved default %q is unavailable through the Sandbox API; verify the profile or choose another sandbox: %w", id, err) } - sb = updated + return "", err } + return name, nil +} - if sb == nil || !strings.EqualFold(sb.Status, "running") { - final, err := waitForRunning(ctx, api, s, id) - if err != nil { - s.fail("Failed to start " + id) - return nil, err - } - sb = final +func prepareSSHIdentity(ctx context.Context, w *databricks.WorkspaceClient, name string) (*sshtunnel.Identity, error) { + caller, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) + if err != nil { + return nil, fmt.Errorf("resolve SSH caller: %w", err) + } + workspace, err := auth.ResolveWorkspaceID(ctx, w) + if err != nil { + return nil, fmt.Errorf("resolve SSH workspace: %w", err) } + return sshtunnel.LocalIdentity(ctx, w.Config.Host+"/"+workspace, caller.Id, name) +} - s.ok("Started " + cmdio.Bold(ctx, id)) - return sb, nil +type sshProxyOptions struct { + Profile, ReleasesDir, IdentityScope, Target, ConfigFile string } -// execSSHDirect replaces the CLI process with ssh (or simulates that on -// Windows via execv). All options are passed on the command line, so no -// ~/.ssh/config entry is required. -func execSSHDirect(sandboxID, host, port, keyPath string, extraArgs []string) error { - return execv.Execv(execv.Options{ - Args: buildSSHArgs(sandboxID, host, port, keyPath, extraArgs), - Env: os.Environ(), - }) +func sshProxyCommand(executable, name string, options sshProxyOptions) (string, error) { + args := []string{executable, "sandbox", "ssh", name, "--proxy"} + if options.Profile != "" { + args = append(args, "--profile", options.Profile) + } + if options.ReleasesDir != "" { + args = append(args, "--releases-dir", options.ReleasesDir) + } + if options.IdentityScope != "" { + args = append(args, "--ssh-identity", options.IdentityScope) + } + if options.Target != "" { + args = append(args, "--target", options.Target) + } + if options.ConfigFile != "" { + args = append(args, "--ssh-config-file", options.ConfigFile) + } + for i, arg := range args { + // OpenSSH expands percent tokens before invoking the shell, even inside quotes. + if strings.ContainsFunc(arg, unicode.IsControl) { + return "", errors.New("control characters are not supported in SSH proxy arguments") + } + arg = strings.ReplaceAll(arg, "%", "%%") + if runtime.GOOS == "windows" { + quoted, ok := shellquote.WindowsArg(arg) + if !ok { + return "", fmt.Errorf("unsupported Windows SSH proxy argument %q", arg) + } + args[i] = quoted + } else { + args[i] = shellquote.BashArg(arg) + } + } + return strings.Join(args, " "), nil } -// buildSSHArgs assembles the argv we hand to the `ssh` binary. -// -// `ssh` concatenates remote-command words with spaces and the remote -// shell re-parses them. That makes the two natural user shapes -// incompatible by default: -// -// - Single arg that's already a complete shell command: -// `sandbox ssh -- 'echo hi | head -3'` -// The user expects the remote shell to split and execute this -// string. ssh's "concat with spaces" is a no-op here, so we hand -// the arg through untouched. -// -// - Multi-arg exec-style invocation: -// `sandbox ssh -- bash -c 'echo hi'` -// Cobra splits this into `["bash", "-c", "echo hi"]`. ssh's join -// produces `bash -c echo hi` on the wire, which bash re-splits into -// `-c=echo` and `$0=hi` — losing the second word. We fix that by shell-quoting -// each arg before append, so the remote sees `bash -c 'echo hi'`. -// -// The heuristic: if there's exactly one extra arg, pass it untouched; -// otherwise quote every arg. shellquote.BashArg leaves safe args alone, -// so `ls -la /tmp` round-trips unchanged in the multi-arg path. -func buildSSHArgs(sandboxID, host, port, keyPath string, extraArgs []string) []string { +func buildSSHArgs(identity *sshtunnel.Identity, proxyCommand string, extraArgs []string) []string { args := []string{ - "ssh", - "-i", keyPath, - "-p", port, - "-o", "IdentitiesOnly=yes", - "-o", "PreferredAuthentications=publickey", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "LogLevel=ERROR", - fmt.Sprintf("%s@%s", sandboxID, host), + "ssh", "-i", identity.KeyPath, "-o", "IdentitiesOnly=yes", "-o", "PreferredAuthentications=publickey", + // Include authentication and the proxy's ten-minute remote setup budget. + "-o", "ConnectTimeout=900", + "-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile=" + quoteSSHOption(identity.KnownHostsPath), + "-o", "HostKeyAlias=" + identity.HostKeyAlias, "-o", "ServerAliveInterval=30", "-o", "ProxyCommand=" + proxyCommand, + "sandbox-agent@" + identity.HostKeyAlias, } if len(extraArgs) == 1 { return append(args, extraArgs[0]) } - for _, a := range extraArgs { - args = append(args, shellquote.BashArg(a)) + for _, arg := range extraArgs { + args = append(args, shellquote.BashArg(arg)) } return args } + +func quoteSSHOption(value string) string { + return `"` + strings.ReplaceAll(filepathToSSH(value), `"`, `\"`) + `"` +} + +func filepathToSSH(value string) string { return strings.ReplaceAll(value, `\`, "/") } diff --git a/cmd/sandbox/ssh_test.go b/cmd/sandbox/ssh_test.go index 31f53eee6b4..24b98d00ea6 100644 --- a/cmd/sandbox/ssh_test.go +++ b/cmd/sandbox/ssh_test.go @@ -1,134 +1,127 @@ package sandbox import ( - "encoding/json" + "io" "net/http" "net/http/httptest" - "sync/atomic" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" "testing" - "github.com/databricks/cli/libs/cmdio" + sshtunnel "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/env" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestEnsureRunningLifecycle(t *testing.T) { - type requestStep struct { - method string - path string - status string - } +func TestBuildSSHArgsBaseFlags(t *testing.T) { + identity := &sshtunnel.Identity{KeyPath: "/keys/id", KnownHostsPath: "/keys/known hosts", HostKeyAlias: "sandbox-host"} + got := buildSSHArgs(identity, "databricks sandbox ssh id --proxy", nil) + assert.Equal(t, "sandbox-agent@sandbox-host", got[len(got)-1]) + assert.Contains(t, got, "StrictHostKeyChecking=yes") + assert.Contains(t, got, `UserKnownHostsFile="/keys/known hosts"`) + assert.Contains(t, got, "ProxyCommand=databricks sandbox ssh id --proxy") +} - for _, tc := range []struct { - name string - initialStatus string - steps []requestStep - wantStatus string - wantErr string - wantStarts int32 +func TestSSHClientOptionsPreserveProfileAndTarget(t *testing.T) { + for _, test := range []struct { + name string + flags []string + profile string + target string }{ - { - name: "stopping waits for stopped before starting", - initialStatus: "Stopping", - steps: []requestStep{ - {method: http.MethodGet, path: sandboxPath("test-id"), status: "Stopped"}, - {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, - {method: http.MethodGet, path: sandboxPath("test-id"), status: "Running"}, - }, - wantStatus: "Running", - wantStarts: 1, - }, - { - name: "stopped starts normally", - initialStatus: "Stopped", - steps: []requestStep{ - {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, - {method: http.MethodGet, path: sandboxPath("test-id"), status: "Running"}, - }, - wantStatus: "Running", - wantStarts: 1, - }, - { - name: "running is a no-op", - initialStatus: "Running", - wantStatus: "Running", - }, - { - name: "stopped after start remains an error", - initialStatus: "Stopped", - steps: []requestStep{ - {method: http.MethodPost, path: sandboxPath("test-id") + "/start", status: "Pending"}, - {method: http.MethodGet, path: sandboxPath("test-id"), status: "Stopped"}, - }, - wantErr: `sandbox test-id reached unexpected state "Stopped" while starting`, - wantStarts: 1, - }, + {name: "explicit profile and target", flags: []string{"--profile", "work", "--target", "dev"}, profile: "work", target: "dev"}, + {name: "implicit profile"}, + {name: "deprecated environment flag", flags: []string{"--environment", "dev"}, target: "dev"}, + {name: "target takes precedence", flags: []string{"--target", "dev", "--environment", "prod"}, target: "dev"}, } { - t.Run(tc.name, func(t *testing.T) { - var requestCount atomic.Int32 - var startCount atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/.well-known/databricks-config" { - assert.NoError(t, json.NewEncoder(w).Encode(map[string]string{ - "oidc_endpoint": "https://workspace.example.test/oidc", - })) - return - } - - index := int(requestCount.Add(1) - 1) - if index >= len(tc.steps) { - t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) - http.Error(w, "unexpected request", http.StatusInternalServerError) - return - } - - step := tc.steps[index] - assert.Equal(t, step.method, r.Method) - assert.Equal(t, step.path, r.URL.Path) - if r.Method == http.MethodPost && r.URL.Path == sandboxPath("test-id")+"/start" { - startCount.Add(1) - } - assert.NoError(t, json.NewEncoder(w).Encode(sandboxEntry{ - SandboxID: "test-id", - Status: step.status, - })) - })) - t.Cleanup(server.Close) - - workspaceClient, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "test-token", - }) + t.Run(test.name, func(t *testing.T) { + cmd := newSSHCommand() + cmd.SetContext(env.Set(t.Context(), "DATABRICKS_TOKEN", "environment-token")) + cmd.Flags().String("profile", "", "Databricks profile") + cmd.Flags().String("target", "", "Bundle target") + cmd.Flags().String("environment", "", "Deprecated bundle target") + require.NoError(t, cmd.ParseFlags(test.flags)) + cfg := &config.Config{Host: "https://workspace.test", WorkspaceID: "123", Profile: "work"} + identity := &sshtunnel.Identity{KeyPath: "/keys/id", KnownHostsPath: "/keys/known-hosts", HostKeyAlias: "sandbox-scope"} + options, err := sshClientOptions(cmd, cfg, "sandboxes/test", identity, "/releases", []string{"uname", "-a"}) require.NoError(t, err) - api, err := newSandboxAPI(workspaceClient) - require.NoError(t, err) - - ctx := cmdio.MockDiscard(t.Context()) - got, err := ensureRunning(ctx, api, "test-id", tc.initialStatus) - if tc.wantErr != "" { - require.EqualError(t, err, tc.wantErr) - assert.Nil(t, got) + var proxyCommand string + for _, arg := range options.Args { + if strings.HasPrefix(arg, "ProxyCommand=") { + proxyCommand = arg + } + } + assert.Contains(t, proxyCommand, "sandbox ssh sandboxes/test --proxy") + assert.Contains(t, proxyCommand, "--ssh-identity sandbox-scope") + assert.Contains(t, proxyCommand, "--releases-dir /releases") + if test.profile == "" { + assert.NotContains(t, proxyCommand, "--profile") + } else { + assert.Contains(t, proxyCommand, "--profile "+test.profile) + } + if test.target == "" { + assert.NotContains(t, proxyCommand, "--target") } else { - require.NoError(t, err) - require.NotNil(t, got) - assert.Equal(t, tc.wantStatus, got.Status) + assert.Contains(t, proxyCommand, "--target "+test.target) } - assert.Equal(t, int32(len(tc.steps)), requestCount.Load()) - assert.Equal(t, tc.wantStarts, startCount.Load()) + assert.Equal(t, []string{"sandbox-agent@sandbox-scope", "uname", "-a"}, options.Args[len(options.Args)-3:]) + assert.Contains(t, options.Env, "DATABRICKS_HOST=https://workspace.test") + assert.Contains(t, options.Env, "DATABRICKS_WORKSPACE_ID=123") + assert.Contains(t, options.Env, "DATABRICKS_CONFIG_PROFILE=work") + assert.Contains(t, options.Env, "DATABRICKS_TOKEN=environment-token") + assert.Contains(t, options.Args, "ConnectTimeout=900") }) } } -func TestBuildSSHArgsBaseFlags(t *testing.T) { - got := buildSSHArgs("happy-panda-1234", "gw.example.test", "2222", "/keys/id", nil) - // Sanity: the destination is the last arg when no extras. - assert.Equal(t, "happy-panda-1234@gw.example.test", got[len(got)-1]) - // The base flag set is fixed; spot-check a couple. - assert.Contains(t, got, "-i") - assert.Contains(t, got, "/keys/id") - assert.Contains(t, got, "-p") - assert.Contains(t, got, "2222") +func TestSSHProcessEnvironmentPreservesResolvedWorkspaceAndCredentials(t *testing.T) { + ctx := env.Set(t.Context(), "DATABRICKS_HOST", "https://old-workspace.test") + ctx = env.Set(ctx, "DATABRICKS_TOKEN", "environment-token") + ctx = env.Set(ctx, "DATABRICKS_WORKSPACE_ID", "old-workspace") + configuration := &config.Config{Host: "https://selected-workspace.test", WorkspaceID: "123", AccountID: "selected-account", Profile: "work", ConfigFile: "/profiles/databrickscfg"} + actual := map[string]string{} + for _, variable := range sshProcessEnvironment(ctx, configuration) { + key, value, _ := strings.Cut(variable, "=") + actual[key] = value + } + assert.Equal(t, configuration.Host, actual["DATABRICKS_HOST"]) + assert.Equal(t, "123", actual["DATABRICKS_WORKSPACE_ID"]) + assert.Equal(t, "selected-account", actual["DATABRICKS_ACCOUNT_ID"]) + assert.Equal(t, "work", actual["DATABRICKS_CONFIG_PROFILE"]) + assert.Equal(t, configuration.ConfigFile, actual["DATABRICKS_CONFIG_FILE"]) + assert.Equal(t, "environment-token", actual["DATABRICKS_TOKEN"]) +} + +func TestSSHProxyLoadsSavedConfigFile(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, `{}`) + })) + defer server.Close() + filename := filepath.Join(t.TempDir(), "selected config") + require.NoError(t, os.WriteFile(filename, []byte("[work]\nhost = "+server.URL+"\ntoken = saved-token\n"), 0o600)) + t.Setenv("DATABRICKS_CONFIG_FILE", filepath.Join(t.TempDir(), "missing")) + previous := os.Getenv("DATABRICKS_CONFIG_FILE") + cmd := newSSHCommand() + cmd.SetContext(t.Context()) + cmd.Flags().String("profile", "", "Databricks profile") + require.NoError(t, cmd.ParseFlags([]string{"--proxy", "--profile", "work", "--ssh-config-file", filename})) + require.NoError(t, cmd.PreRunE(cmd, nil)) + cfg := cmdctx.WorkspaceClient(cmd.Context()).Config + assert.Equal(t, filename, cfg.ConfigFile) + assert.Equal(t, server.URL, cfg.Host) + assert.Equal(t, "saved-token", cfg.Token) + assert.Equal(t, previous, os.Getenv("DATABRICKS_CONFIG_FILE")) + + cmd = newSSHCommand() + require.NoError(t, cmd.ParseFlags([]string{"--ssh-config-file", filename})) + require.ErrorContains(t, cmd.PreRunE(cmd, nil), "requires --proxy") } func TestBuildSSHArgsQuoting(t *testing.T) { @@ -192,10 +185,10 @@ func TestBuildSSHArgsQuoting(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - got := buildSSHArgs("id", "host", "2222", "/k", tc.extraArgs) + got := buildSSHArgs(&sshtunnel.Identity{HostKeyAlias: "host"}, "proxy", tc.extraArgs) // Trim everything up to and including the destination so we // can assert just on the appended extras. - dest := "id@host" + dest := "sandbox-agent@host" cut := -1 for i, a := range got { if a == dest { @@ -213,3 +206,143 @@ func TestBuildSSHArgsQuoting(t *testing.T) { }) } } + +func TestSSHProxyCommandQuotesArguments(t *testing.T) { + command, err := sshProxyCommand("/path with spaces/databricks", "sandboxes/test", sshProxyOptions{Profile: "work profile", ReleasesDir: "/release dir"}) + require.NoError(t, err) + if runtime.GOOS == "windows" { + assert.Equal(t, `"/path with spaces/databricks" sandbox ssh sandboxes/test --proxy --profile "work profile" --releases-dir "/release dir"`, command) + } else { + assert.Equal(t, "'/path with spaces/databricks' sandbox ssh sandboxes/test --proxy --profile 'work profile' --releases-dir '/release dir'", command) + } + percent, percentErr := sshProxyCommand("databricks", "sandboxes/test", sshProxyOptions{Profile: "a%h"}) + if runtime.GOOS == "windows" { + require.ErrorContains(t, percentErr, "unsupported Windows SSH proxy argument") + } else { + require.NoError(t, percentErr) + assert.Equal(t, "databricks sandbox ssh sandboxes/test --proxy --profile 'a%%h'", percent) + } + _, err = sshProxyCommand("databricks", "sandboxes/test", sshProxyOptions{Profile: "bad\nprofile"}) + require.ErrorContains(t, err, "control characters") + scoped, err := sshProxyCommand("databricks", "sandboxes/test", sshProxyOptions{Profile: "work", IdentityScope: "sandbox-scope", Target: "dev"}) + require.NoError(t, err) + assert.Equal(t, "databricks sandbox ssh sandboxes/test --proxy --profile work --ssh-identity sandbox-scope --target dev", scoped) + saved, err := sshProxyCommand("databricks", "sandboxes/test", sshProxyOptions{Profile: "work", ConfigFile: "/profiles/custom config"}) + require.NoError(t, err) + if runtime.GOOS == "windows" { + assert.Equal(t, `databricks sandbox ssh sandboxes/test --proxy --profile work --ssh-config-file "/profiles/custom config"`, saved) + } else { + assert.Equal(t, "databricks sandbox ssh sandboxes/test --proxy --profile work --ssh-config-file '/profiles/custom config'", saved) + } +} + +func TestSSHProxyCommandDispatchAndIdentityMismatch(t *testing.T) { + for _, mismatch := range []bool{false, true} { + t.Run(strconv.FormatBool(mismatch), func(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + var paths []string + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + switch r.URL.Path { + case "/.well-known/databricks-config": + _, _ = io.WriteString(w, `{}`) + case "/api/2.0/preview/scim/v2/Me": + _, _ = io.WriteString(w, `{"id":"caller"}`) + default: + _, _ = io.WriteString(w, `{"status":{"state":"SANDBOX_STATE_UNSPECIFIED"}}`) + } + })) + defer endpoint.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: endpoint.URL, Token: "test-token", WorkspaceID: "123"}) + require.NoError(t, err) + cmd := newSSHCommand() + cmd.PreRunE = nil + args := []string{"test", "--proxy"} + if mismatch { + args = append(args, "--ssh-identity", "different-workspace") + } + cmd.SetArgs(args) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + err = cmd.ExecuteContext(cmdctx.SetWorkspaceClient(ctx, w)) + if mismatch { + require.ErrorContains(t, err, "workspace or caller changed") + } else { + require.ErrorContains(t, err, "unsupported state") + } + assert.Contains(t, paths, "/api/2.0/sandboxes/test") + assert.Contains(t, paths, "/api/2.0/preview/scim/v2/Me") + }) + } +} + +func TestSSHCommandRejectsExtraSandboxArguments(t *testing.T) { + cmd := newSSHCommand() + // Argument validation runs before credentials are resolved by RunE. + cmd.PreRunE = nil + cmd.SetArgs([]string{"first", "second"}) + require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "at most one sandbox") + cmd = newSSHCommand() + cmd.PreRunE = nil + cmd.SetArgs([]string{"first", "--proxy", "--", "echo"}) + require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "--proxy does not accept SSH arguments") +} + +func TestResolveSSHName(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + calls++ + if strings.HasSuffix(r.URL.Path, "missing") { + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"error_code":"NOT_FOUND","message":"missing"}`) + return + } + _, _ = io.WriteString(w, `{"name":"sandboxes/test","status":{"state":"SANDBOX_STATE_RUNNING"}}`) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token"}) + require.NoError(t, err) + require.NoError(t, setDefault(ctx, server.URL, "missing")) + _, err = resolveSSHName(ctx, w, "") + require.ErrorContains(t, err, "unavailable through the Sandbox API") + assert.Equal(t, "missing", getDefault(ctx, server.URL)) + _, err = resolveSSHName(ctx, w, "a/b") + require.ErrorContains(t, err, "invalid sandbox ID") + name, err := resolveSSHName(ctx, w, "sandboxes/test") + require.NoError(t, err) + assert.Equal(t, "sandboxes/test", name) + assert.Equal(t, 2, calls) +} + +func TestPrepareSSHIdentityUsesWorkspaceAndCaller(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + failure := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + assert.Equal(t, "/api/2.0/preview/scim/v2/Me", r.URL.Path) + if failure { + http.Error(w, `{"error_code":"PERMISSION_DENIED","message":"denied"}`, http.StatusForbidden) + return + } + _, _ = io.WriteString(w, `{"id":"caller"}`) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token", WorkspaceID: "123"}) + require.NoError(t, err) + got, err := prepareSSHIdentity(ctx, w, "sandboxes/test") + require.NoError(t, err) + expected, err := sshtunnel.LocalIdentity(ctx, server.URL+"/123", "caller", "sandboxes/test") + require.NoError(t, err) + assert.Equal(t, expected.KeyPath, got.KeyPath) + failure = true + _, err = prepareSSHIdentity(ctx, w, "sandboxes/test") + require.ErrorContains(t, err, "resolve SSH caller") +} diff --git a/cmd/sandbox/sshconfig.go b/cmd/sandbox/sshconfig.go index d4c54986aed..ac98d0ed70a 100644 --- a/cmd/sandbox/sshconfig.go +++ b/cmd/sandbox/sshconfig.go @@ -17,6 +17,7 @@ import ( ) const ( + defaultGatewayPort = "2222" // sshIncludeFileName is the sandbox-managed file referenced by the // user's ~/.ssh/config. The file is fully owned: we rewrite it on // every `sandbox register`, so manual edits to it will not survive. diff --git a/cmd/sandbox/start.go b/cmd/sandbox/start.go index ba48a42efd9..603527f5df0 100644 --- a/cmd/sandbox/start.go +++ b/cmd/sandbox/start.go @@ -97,14 +97,8 @@ func waitForRunning(ctx context.Context, api *sandboxAPI, s *spinner, id string) return waitForState(ctx, api, s, id, "Running", "Starting", "Stopped", "Terminated", "Failed") } -// The API rejects start requests until teardown finishes, so STOPPING needs -// its own polling phase. -func waitForStopped(ctx context.Context, api *sandboxAPI, s *spinner, id string) (*sandboxEntry, error) { - return waitForState(ctx, api, s, id, "Stopped", "Stopping", "Terminated", "Failed") -} - // Centralizing lifecycle polling keeps timeout, cancellation, and terminal -// state handling consistent across start and SSH. +// state handling consistent across start requests. func waitForState(ctx context.Context, api *sandboxAPI, s *spinner, id, targetStatus, operation string, unexpectedStatuses ...string) (*sandboxEntry, error) { start := time.Now() deadline := start.Add(startWaitTimeout) diff --git a/experimental/ssh/internal/client/releases.go b/experimental/ssh/internal/client/releases.go index def34ce363c..a38d6bff29b 100644 --- a/experimental/ssh/internal/client/releases.go +++ b/experimental/ssh/internal/client/releases.go @@ -9,12 +9,12 @@ import ( "io/fs" "net" "net/http" - "os" "path/filepath" "strings" "syscall" "time" + "github.com/databricks/cli/experimental/ssh/internal/releases" "github.com/databricks/cli/experimental/ssh/internal/workspace" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/filer" @@ -176,39 +176,12 @@ func isProxyUploadError(err error) bool { return strings.Contains(msg, "stream error") && strings.Contains(msg, "stream ID") } -func getReleaseName(architecture, version string) string { - if strings.Contains(version, "dev") { - return fmt.Sprintf("databricks_cli_linux_%s.zip", architecture) - } - return fmt.Sprintf("databricks_cli_%s_linux_%s.zip", version, architecture) -} +func getReleaseName(architecture, version string) string { return releases.Name(architecture, version) } func getLocalRelease(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { - log.Infof(ctx, "Looking for CLI releases in directory: %s", releasesDir) - releaseName := getReleaseName(architecture, version) - releasePath := filepath.Join(releasesDir, releaseName) - file, err := os.Open(releasePath) - if err != nil { - return nil, fmt.Errorf("failed to open %s: %w", releasePath, err) - } - return file, nil + return releases.OpenLocal(ctx, architecture, version, releasesDir) } func getGithubRelease(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { - // TODO: download and check databricks_cli__SHA256SUMS - fileName := getReleaseName(architecture, version) - downloadURL := fmt.Sprintf("https://github.com/databricks/cli/releases/download/v%s/%s", version, fileName) - log.Infof(ctx, "Downloading %s from %s", fileName, downloadURL) - - resp, err := http.Get(downloadURL) - if err != nil { - return nil, fmt.Errorf("failed to download %s: %w", downloadURL, err) - } - - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("failed to download %s: HTTP %d", downloadURL, resp.StatusCode) - } - - return resp.Body, nil + return releases.OpenPublished(ctx, architecture, version, releasesDir) } diff --git a/experimental/ssh/internal/releases/releases.go b/experimental/ssh/internal/releases/releases.go new file mode 100644 index 00000000000..c90cdf705c5 --- /dev/null +++ b/experimental/ssh/internal/releases/releases.go @@ -0,0 +1,54 @@ +package releases + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/log" +) + +func Name(architecture, version string) string { + if strings.Contains(version, "dev") { + return fmt.Sprintf("databricks_cli_linux_%s.zip", architecture) + } + return fmt.Sprintf("databricks_cli_%s_linux_%s.zip", version, architecture) +} + +func OpenLocal(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + log.Infof(ctx, "Looking for CLI releases in directory: %s", releasesDir) + releaseName := Name(architecture, version) + releasePath := filepath.Join(releasesDir, releaseName) + file, err := os.Open(releasePath) + if err != nil { + return nil, fmt.Errorf("failed to open %s: %w", releasePath, err) + } + return file, nil +} + +func OpenPublished(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + // TODO: download and check databricks_cli__SHA256SUMS + fileName := Name(architecture, version) + downloadURL := fmt.Sprintf("https://github.com/databricks/cli/releases/download/v%s/%s", version, fileName) + log.Infof(ctx, "Downloading %s from %s", fileName, downloadURL) + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(request) + if err != nil { + return nil, fmt.Errorf("failed to download %s: %w", downloadURL, err) + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("failed to download %s: HTTP %d", downloadURL, resp.StatusCode) + } + + return resp.Body, nil +} diff --git a/experimental/ssh/internal/releases/releases_test.go b/experimental/ssh/internal/releases/releases_test.go new file mode 100644 index 00000000000..b5c986762d1 --- /dev/null +++ b/experimental/ssh/internal/releases/releases_test.go @@ -0,0 +1,41 @@ +package releases_test + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/releases" + "github.com/stretchr/testify/require" +) + +type stalledDownload struct{ started chan struct{} } + +func (download stalledDownload) RoundTrip(request *http.Request) (*http.Response, error) { + close(download.started) + <-request.Context().Done() + return nil, request.Context().Err() +} + +func TestPublishedDownloadHonorsCancellation(t *testing.T) { + started := make(chan struct{}) + previous := http.DefaultTransport + http.DefaultTransport = stalledDownload{started: started} + t.Cleanup(func() { http.DefaultTransport = previous }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + finished := make(chan error, 1) + go func() { + _, err := releases.OpenPublished(ctx, "amd64", "1.0.0", "") + finished <- err + }() + <-started + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("release download ignored cancellation") + } +} diff --git a/experimental/ssh/sandbox/export_test.go b/experimental/ssh/sandbox/export_test.go new file mode 100644 index 00000000000..36bec946788 --- /dev/null +++ b/experimental/ssh/sandbox/export_test.go @@ -0,0 +1,14 @@ +package sandbox + +var ( + ProxyStreamForTest = proxyStream + LinuxExecutableForTest = linuxExecutable + ExecutableFromArchiveForTest = executableFromArchive + ReadServerOutputForTest = readServerOutput + ForwardingURLForTest = forwardingURL + PrepareRemoteForTest = prepareRemote + UploadExecutableForTest = uploadExecutable + ConnectProxyForTest = connectProxy +) + +type RemoteSetupForTest = remoteSetup diff --git a/experimental/ssh/sandbox/identity.go b/experimental/ssh/sandbox/identity.go new file mode 100644 index 00000000000..0ec651219ee --- /dev/null +++ b/experimental/ssh/sandbox/identity.go @@ -0,0 +1,97 @@ +package sandbox + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/pem" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/databricks/cli/experimental/ssh/internal/keys" + "github.com/databricks/cli/experimental/ssh/internal/sshconfig" + "golang.org/x/crypto/ssh" +) + +// Identity contains only paths and public material; private keys remain local. +type Identity struct { + KeyPath string + KnownHostsPath string + HostKeyAlias string + PublicKey []byte +} + +// LocalIdentity creates a reusable key scoped to the workspace, caller and sandbox. +func LocalIdentity(ctx context.Context, workspace, caller, name string) (*Identity, error) { + scope := fmt.Sprintf("sandbox-%x", sha256.Sum256([]byte(workspace+"\x00"+caller+"\x00"+name))) + directory, err := keys.GetLocalSSHKeyPath(ctx, scope, "") + if err != nil { + return nil, err + } + keyPath := filepath.Join(directory, "id_ed25519") + if _, err := os.Stat(directory); errors.Is(err, fs.ErrNotExist) { + if err := createLocalKey(directory); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + private, err := os.ReadFile(keyPath) + if err != nil { + return nil, fmt.Errorf("read local SSH key: %w", err) + } + signer, err := ssh.ParsePrivateKey(private) + if err != nil { + return nil, fmt.Errorf("parse local SSH key: %w", err) + } + if err := os.Chmod(keyPath, 0o600); err != nil { + return nil, err + } + knownHosts, err := sshconfig.GetKnownHostsPath(ctx, scope, "") + if err != nil { + return nil, err + } + return &Identity{KeyPath: keyPath, KnownHostsPath: knownHosts, HostKeyAlias: scope, PublicKey: ssh.MarshalAuthorizedKey(signer.PublicKey())}, nil +} + +func createLocalKey(directory string) error { + if err := os.MkdirAll(filepath.Dir(directory), 0o700); err != nil { + return err + } + temporary, err := os.MkdirTemp(filepath.Dir(directory), ".sandbox-key-*") + if err != nil { + return err + } + defer os.RemoveAll(temporary) + _, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + encoded, err := ssh.MarshalPrivateKey(private, "") + if err != nil { + return err + } + public, err := ssh.NewPublicKey(private.Public()) + if err != nil { + return err + } + if err := keys.SaveSSHKeyPair(filepath.Join(temporary, "id_ed25519"), pem.EncodeToMemory(encoded), ssh.MarshalAuthorizedKey(public)); err != nil { + return err + } + // Rename a complete directory: concurrent first connections either install + // their key or read the winner, without ever replacing an in-use private key. + if err := os.Rename(temporary, directory); err != nil { + if _, readErr := os.Stat(filepath.Join(directory, "id_ed25519")); readErr != nil { + return fmt.Errorf("install local SSH key: %w", err) + } + } + return nil +} + +func (identity *Identity) PinHostKey(key []byte) error { + return sshconfig.PinHostKey(identity.KnownHostsPath, identity.HostKeyAlias, key) +} diff --git a/experimental/ssh/sandbox/identity_test.go b/experimental/ssh/sandbox/identity_test.go new file mode 100644 index 00000000000..14a7be47d9f --- /dev/null +++ b/experimental/ssh/sandbox/identity_test.go @@ -0,0 +1,59 @@ +package sandbox_test + +import ( + "os" + "path/filepath" + "runtime" + "sync" + "testing" + + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLocalIdentityConcurrentCreationAndIsolation(t *testing.T) { + ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) + const count = 12 + identities := make([]*sandbox.Identity, count) + failures := make([]error, count) + var group sync.WaitGroup + for i := range count { + group.Go(func() { + identities[i], failures[i] = sandbox.LocalIdentity(ctx, "https://workspace.test/123", "caller", "sandboxes/test") + }) + } + group.Wait() + for i := range count { + require.NoError(t, failures[i]) + require.NotNil(t, identities[i]) + assert.Equal(t, identities[0], identities[i]) + } + first := identities[0] + for _, scope := range []struct{ workspace, caller, name string }{ + {"https://other.test/123", "caller", "sandboxes/test"}, + {"https://workspace.test/123", "other", "sandboxes/test"}, + {"https://workspace.test/123", "caller", "sandboxes/other"}, + } { + other, err := sandbox.LocalIdentity(ctx, scope.workspace, scope.caller, scope.name) + require.NoError(t, err) + assert.NotEqual(t, first.KeyPath, other.KeyPath) + assert.NotEqual(t, first.PublicKey, other.PublicKey) + } + if runtime.GOOS != "windows" { + stat, err := os.Stat(first.KeyPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), stat.Mode().Perm()) + stat, err = os.Stat(filepath.Dir(first.KeyPath)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), stat.Mode().Perm()) + } + require.NoError(t, first.PinHostKey(first.PublicKey)) + content, err := os.ReadFile(first.KnownHostsPath) + require.NoError(t, err) + assert.Equal(t, first.HostKeyAlias+" "+string(first.PublicKey), string(content)) + require.NoError(t, os.WriteFile(first.KeyPath, []byte("invalid key"), 0o600)) + _, err = sandbox.LocalIdentity(ctx, "https://workspace.test/123", "caller", "sandboxes/test") + require.ErrorContains(t, err, "parse local SSH key") +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/client.go b/experimental/ssh/sandbox/internal/sandboxexec/client.go new file mode 100644 index 00000000000..d3d328b2efd --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/client.go @@ -0,0 +1,89 @@ +package sandboxexec + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" +) + +const servicePath = "/databricks.sandbox.SandboxService/" + +type workspaceCredentials struct{ config *config.Config } + +func (c workspaceCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.config.Host, nil) + if err != nil { + return nil, err + } + if err := c.config.Authenticate(request); err != nil { + return nil, err + } + result := make(map[string]string) + for key, value := range auth.WorkspaceIDHeaders(c.config) { + result[strings.ToLower(key)] = value + } + for key, values := range request.Header { + result[strings.ToLower(key)] = strings.Join(values, ",") + } + return result, nil +} +func (workspaceCredentials) RequireTransportSecurity() bool { return true } + +// Client owns the authenticated command-execution channel for one sandbox. +type Client struct { + conn *grpc.ClientConn + name string +} + +func New(w *databricks.WorkspaceClient, name string) (*Client, error) { + host, err := url.Parse(w.Config.Host) + if err != nil || host.Scheme != "https" || host.Hostname() == "" { + return nil, errors.New("sandbox command execution requires an HTTPS workspace URL") + } + address := host.Host + if host.Port() == "" { + address += ":443" + } + conn, err := grpc.NewClient(address, + grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: w.Config.InsecureSkipVerify})), + grpc.WithPerRPCCredentials(workspaceCredentials{config: w.Config})) + if err != nil { + return nil, fmt.Errorf("create sandbox command channel: %w", err) + } + return &Client{conn: conn, name: name}, nil +} + +func (c *Client) Close() error { return c.conn.Close() } + +// Execute leaves the process running when its output stream is closed. Terminate +// must be called separately to stop the process and its descendants. +func (c *Client) Execute(ctx context.Context, commandID, executable string, args []string) (grpc.ServerStreamingClient[ExecuteCommandResponse], error) { + ctx = metadata.AppendToOutgoingContext(ctx, "x-databricks-sandbox-name", c.name) + stream, err := c.conn.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, servicePath+"ExecuteCommand") + if err != nil { + return nil, err + } + if err := stream.SendMsg(&ExecuteCommandRequest{Cmd: new(executable), Args: args, Envs: map[string]string{"TZ": "UTC"}, CommandId: new(commandID)}); err != nil { + return nil, err + } + if err := stream.CloseSend(); err != nil { + return nil, err + } + return &grpc.GenericClientStream[ExecuteCommandRequest, ExecuteCommandResponse]{ClientStream: stream}, nil +} + +func (c *Client) Terminate(ctx context.Context, commandID string) error { + ctx = metadata.AppendToOutgoingContext(ctx, "x-databricks-sandbox-name", c.name) + return c.conn.Invoke(ctx, servicePath+"TerminateCommand", &TerminateCommandRequest{CommandId: new(commandID)}, &TerminateCommandResponse{}) +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/client_test.go b/experimental/ssh/sandbox/internal/sandboxexec/client_test.go new file mode 100644 index 00000000000..b87704b62a2 --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/client_test.go @@ -0,0 +1,109 @@ +package sandboxexec_test + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +type ( + service any + recordedCall struct { + metadata metadata.MD + command string + } +) + +type commandService struct { + mu sync.Mutex + calls []recordedCall +} + +func (service *commandService) record(ctx context.Context, id string) { + md, _ := metadata.FromIncomingContext(ctx) + service.mu.Lock() + defer service.mu.Unlock() + service.calls = append(service.calls, recordedCall{md.Copy(), id}) +} + +func TestAuthenticatedStreamingExecutionAndTermination(t *testing.T) { + implementation := &commandService{} + server := grpc.NewServer() + server.RegisterService(&grpc.ServiceDesc{ + ServiceName: "databricks.sandbox.SandboxService", HandlerType: (*service)(nil), + Streams: []grpc.StreamDesc{{StreamName: "ExecuteCommand", ServerStreams: true, Handler: func(service any, stream grpc.ServerStream) error { + request := &sandboxexec.ExecuteCommandRequest{} + if err := stream.RecvMsg(request); err != nil { + return err + } + implementation.record(stream.Context(), request.GetCommandId()) + if request.Envs["TZ"] != "UTC" { + return errors.New("expected UTC expiry interpretation") + } + if request.GetCmd() != "/remote/databricks" { + return errors.New("unexpected executable") + } + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Start{Start: &sandboxexec.StartEvent{CommandId: request.CommandId}}}}) + }}}, + Methods: []grpc.MethodDesc{{MethodName: "TerminateCommand", Handler: func(service any, ctx context.Context, decode func(any) error, _ grpc.UnaryServerInterceptor) (any, error) { + request := &sandboxexec.TerminateCommandRequest{} + if err := decode(request); err != nil { + return nil, err + } + implementation.record(ctx, request.GetCommandId()) + return &sandboxexec.TerminateCommandResponse{}, nil + }}}, + }, implementation) + defer server.Stop() + endpoint := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") { + server.ServeHTTP(w, r) + return + } + _, _ = io.WriteString(w, `{}`) + })) + endpoint.EnableHTTP2 = true + endpoint.StartTLS() + defer endpoint.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: endpoint.URL, Token: "test-token", WorkspaceID: "123", InsecureSkipVerify: true, HTTPTransport: endpoint.Client().Transport}) + require.NoError(t, err) + client, err := sandboxexec.New(w, "sandboxes/test") + require.NoError(t, err) + defer client.Close() + stream, err := client.Execute(t.Context(), "command-id", "/remote/databricks", []string{"sandbox", "server"}) + require.NoError(t, err) + event, err := stream.Recv() + require.NoError(t, err) + assert.Equal(t, "command-id", event.GetProcessEvent().GetStart().GetCommandId()) + _, err = stream.Recv() + require.ErrorIs(t, err, io.EOF) + require.NoError(t, client.Terminate(t.Context(), "command-id")) + implementation.mu.Lock() + defer implementation.mu.Unlock() + require.Len(t, implementation.calls, 2) + for _, call := range implementation.calls { + assert.Equal(t, []string{"Bearer test-token"}, call.metadata.Get("authorization")) + assert.Equal(t, []string{"123"}, call.metadata.Get("x-databricks-workspace-id")) + assert.Equal(t, []string{"sandboxes/test"}, call.metadata.Get("x-databricks-sandbox-name")) + assert.Equal(t, "command-id", call.command) + } +} + +func TestClientRejectsPlaintext(t *testing.T) { + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: "http://workspace.test", Token: "test-token"}) + require.NoError(t, err) + _, err = sandboxexec.New(w, "sandboxes/test") + require.ErrorContains(t, err, "HTTPS") +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/command.pb.go b/experimental/ssh/sandbox/internal/sandboxexec/command.pb.go new file mode 100644 index 00000000000..ee12c3341a4 --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/command.pb.go @@ -0,0 +1,625 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v6.33.0 +// source: experimental/ssh/sandbox/internal/sandboxexec/command.proto + +package sandboxexec + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Streaming command messages used by the sandbox SSH client. +type ExecuteCommandRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Cmd *string `protobuf:"bytes,1,opt,name=cmd" json:"cmd,omitempty"` + Args []string `protobuf:"bytes,2,rep,name=args" json:"args,omitempty"` + Envs map[string]string `protobuf:"bytes,4,rep,name=envs" json:"envs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + CommandId *string `protobuf:"bytes,5,opt,name=command_id,json=commandId" json:"command_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecuteCommandRequest) Reset() { + *x = ExecuteCommandRequest{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecuteCommandRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecuteCommandRequest) ProtoMessage() {} + +func (x *ExecuteCommandRequest) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecuteCommandRequest.ProtoReflect.Descriptor instead. +func (*ExecuteCommandRequest) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{0} +} + +func (x *ExecuteCommandRequest) GetCmd() string { + if x != nil && x.Cmd != nil { + return *x.Cmd + } + return "" +} + +func (x *ExecuteCommandRequest) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *ExecuteCommandRequest) GetEnvs() map[string]string { + if x != nil { + return x.Envs + } + return nil +} + +func (x *ExecuteCommandRequest) GetCommandId() string { + if x != nil && x.CommandId != nil { + return *x.CommandId + } + return "" +} + +type ExecuteCommandResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProcessEvent *ProcessEvent `protobuf:"bytes,1,opt,name=process_event,json=processEvent" json:"process_event,omitempty"` + SequenceNumber *int64 `protobuf:"varint,2,opt,name=sequence_number,json=sequenceNumber" json:"sequence_number,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecuteCommandResponse) Reset() { + *x = ExecuteCommandResponse{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecuteCommandResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecuteCommandResponse) ProtoMessage() {} + +func (x *ExecuteCommandResponse) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecuteCommandResponse.ProtoReflect.Descriptor instead. +func (*ExecuteCommandResponse) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{1} +} + +func (x *ExecuteCommandResponse) GetProcessEvent() *ProcessEvent { + if x != nil { + return x.ProcessEvent + } + return nil +} + +func (x *ExecuteCommandResponse) GetSequenceNumber() int64 { + if x != nil && x.SequenceNumber != nil { + return *x.SequenceNumber + } + return 0 +} + +type ProcessEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Event: + // + // *ProcessEvent_Start + // *ProcessEvent_Data + // *ProcessEvent_End + Event isProcessEvent_Event `protobuf_oneof:"event"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProcessEvent) Reset() { + *x = ProcessEvent{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProcessEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProcessEvent) ProtoMessage() {} + +func (x *ProcessEvent) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProcessEvent.ProtoReflect.Descriptor instead. +func (*ProcessEvent) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{2} +} + +func (x *ProcessEvent) GetEvent() isProcessEvent_Event { + if x != nil { + return x.Event + } + return nil +} + +func (x *ProcessEvent) GetStart() *StartEvent { + if x != nil { + if x, ok := x.Event.(*ProcessEvent_Start); ok { + return x.Start + } + } + return nil +} + +func (x *ProcessEvent) GetData() *DataEvent { + if x != nil { + if x, ok := x.Event.(*ProcessEvent_Data); ok { + return x.Data + } + } + return nil +} + +func (x *ProcessEvent) GetEnd() *EndEvent { + if x != nil { + if x, ok := x.Event.(*ProcessEvent_End); ok { + return x.End + } + } + return nil +} + +type isProcessEvent_Event interface { + isProcessEvent_Event() +} + +type ProcessEvent_Start struct { + Start *StartEvent `protobuf:"bytes,1,opt,name=start,oneof"` +} + +type ProcessEvent_Data struct { + Data *DataEvent `protobuf:"bytes,2,opt,name=data,oneof"` +} + +type ProcessEvent_End struct { + End *EndEvent `protobuf:"bytes,3,opt,name=end,oneof"` +} + +func (*ProcessEvent_Start) isProcessEvent_Event() {} + +func (*ProcessEvent_Data) isProcessEvent_Event() {} + +func (*ProcessEvent_End) isProcessEvent_Event() {} + +type StartEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Pid *int64 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"` + CommandId *string `protobuf:"bytes,2,opt,name=command_id,json=commandId" json:"command_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartEvent) Reset() { + *x = StartEvent{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartEvent) ProtoMessage() {} + +func (x *StartEvent) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartEvent.ProtoReflect.Descriptor instead. +func (*StartEvent) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{3} +} + +func (x *StartEvent) GetPid() int64 { + if x != nil && x.Pid != nil { + return *x.Pid + } + return 0 +} + +func (x *StartEvent) GetCommandId() string { + if x != nil && x.CommandId != nil { + return *x.CommandId + } + return "" +} + +type DataEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Output: + // + // *DataEvent_Stdout + // *DataEvent_Stderr + // *DataEvent_Pty + Output isDataEvent_Output `protobuf_oneof:"output"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DataEvent) Reset() { + *x = DataEvent{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DataEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DataEvent) ProtoMessage() {} + +func (x *DataEvent) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DataEvent.ProtoReflect.Descriptor instead. +func (*DataEvent) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{4} +} + +func (x *DataEvent) GetOutput() isDataEvent_Output { + if x != nil { + return x.Output + } + return nil +} + +func (x *DataEvent) GetStdout() []byte { + if x != nil { + if x, ok := x.Output.(*DataEvent_Stdout); ok { + return x.Stdout + } + } + return nil +} + +func (x *DataEvent) GetStderr() []byte { + if x != nil { + if x, ok := x.Output.(*DataEvent_Stderr); ok { + return x.Stderr + } + } + return nil +} + +func (x *DataEvent) GetPty() []byte { + if x != nil { + if x, ok := x.Output.(*DataEvent_Pty); ok { + return x.Pty + } + } + return nil +} + +type isDataEvent_Output interface { + isDataEvent_Output() +} + +type DataEvent_Stdout struct { + Stdout []byte `protobuf:"bytes,1,opt,name=stdout,oneof"` +} + +type DataEvent_Stderr struct { + Stderr []byte `protobuf:"bytes,2,opt,name=stderr,oneof"` +} + +type DataEvent_Pty struct { + Pty []byte `protobuf:"bytes,3,opt,name=pty,oneof"` +} + +func (*DataEvent_Stdout) isDataEvent_Output() {} + +func (*DataEvent_Stderr) isDataEvent_Output() {} + +func (*DataEvent_Pty) isDataEvent_Output() {} + +type EndEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + ExitCode *int32 `protobuf:"varint,1,opt,name=exit_code,json=exitCode" json:"exit_code,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EndEvent) Reset() { + *x = EndEvent{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EndEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EndEvent) ProtoMessage() {} + +func (x *EndEvent) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EndEvent.ProtoReflect.Descriptor instead. +func (*EndEvent) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{5} +} + +func (x *EndEvent) GetExitCode() int32 { + if x != nil && x.ExitCode != nil { + return *x.ExitCode + } + return 0 +} + +type TerminateCommandRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId *string `protobuf:"bytes,1,opt,name=command_id,json=commandId" json:"command_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateCommandRequest) Reset() { + *x = TerminateCommandRequest{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateCommandRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateCommandRequest) ProtoMessage() {} + +func (x *TerminateCommandRequest) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminateCommandRequest.ProtoReflect.Descriptor instead. +func (*TerminateCommandRequest) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{6} +} + +func (x *TerminateCommandRequest) GetCommandId() string { + if x != nil && x.CommandId != nil { + return *x.CommandId + } + return "" +} + +type TerminateCommandResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateCommandResponse) Reset() { + *x = TerminateCommandResponse{} + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateCommandResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateCommandResponse) ProtoMessage() {} + +func (x *TerminateCommandResponse) ProtoReflect() protoreflect.Message { + mi := &file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminateCommandResponse.ProtoReflect.Descriptor instead. +func (*TerminateCommandResponse) Descriptor() ([]byte, []int) { + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP(), []int{7} +} + +var File_experimental_ssh_sandbox_internal_sandboxexec_command_proto protoreflect.FileDescriptor + +const file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc = "" + + "\n" + + ";experimental/ssh/sandbox/internal/sandboxexec/command.proto\x12\x12databricks.sandbox\"\xde\x01\n" + + "\x15ExecuteCommandRequest\x12\x10\n" + + "\x03cmd\x18\x01 \x01(\tR\x03cmd\x12\x12\n" + + "\x04args\x18\x02 \x03(\tR\x04args\x12G\n" + + "\x04envs\x18\x04 \x03(\v23.databricks.sandbox.ExecuteCommandRequest.EnvsEntryR\x04envs\x12\x1d\n" + + "\n" + + "command_id\x18\x05 \x01(\tR\tcommandId\x1a7\n" + + "\tEnvsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x88\x01\n" + + "\x16ExecuteCommandResponse\x12E\n" + + "\rprocess_event\x18\x01 \x01(\v2 .databricks.sandbox.ProcessEventR\fprocessEvent\x12'\n" + + "\x0fsequence_number\x18\x02 \x01(\x03R\x0esequenceNumber\"\xb6\x01\n" + + "\fProcessEvent\x126\n" + + "\x05start\x18\x01 \x01(\v2\x1e.databricks.sandbox.StartEventH\x00R\x05start\x123\n" + + "\x04data\x18\x02 \x01(\v2\x1d.databricks.sandbox.DataEventH\x00R\x04data\x120\n" + + "\x03end\x18\x03 \x01(\v2\x1c.databricks.sandbox.EndEventH\x00R\x03endB\a\n" + + "\x05event\"=\n" + + "\n" + + "StartEvent\x12\x10\n" + + "\x03pid\x18\x01 \x01(\x03R\x03pid\x12\x1d\n" + + "\n" + + "command_id\x18\x02 \x01(\tR\tcommandId\"]\n" + + "\tDataEvent\x12\x18\n" + + "\x06stdout\x18\x01 \x01(\fH\x00R\x06stdout\x12\x18\n" + + "\x06stderr\x18\x02 \x01(\fH\x00R\x06stderr\x12\x12\n" + + "\x03pty\x18\x03 \x01(\fH\x00R\x03ptyB\b\n" + + "\x06output\"'\n" + + "\bEndEvent\x12\x1b\n" + + "\texit_code\x18\x01 \x01(\x05R\bexitCode\"8\n" + + "\x17TerminateCommandRequest\x12\x1d\n" + + "\n" + + "command_id\x18\x01 \x01(\tR\tcommandId\"\x1a\n" + + "\x18TerminateCommandResponseBIZGgithub.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + +var ( + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescOnce sync.Once + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescData []byte +) + +func file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescGZIP() []byte { + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescOnce.Do(func() { + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc), len(file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc))) + }) + return file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDescData +} + +var file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_goTypes = []any{ + (*ExecuteCommandRequest)(nil), // 0: databricks.sandbox.ExecuteCommandRequest + (*ExecuteCommandResponse)(nil), // 1: databricks.sandbox.ExecuteCommandResponse + (*ProcessEvent)(nil), // 2: databricks.sandbox.ProcessEvent + (*StartEvent)(nil), // 3: databricks.sandbox.StartEvent + (*DataEvent)(nil), // 4: databricks.sandbox.DataEvent + (*EndEvent)(nil), // 5: databricks.sandbox.EndEvent + (*TerminateCommandRequest)(nil), // 6: databricks.sandbox.TerminateCommandRequest + (*TerminateCommandResponse)(nil), // 7: databricks.sandbox.TerminateCommandResponse + nil, // 8: databricks.sandbox.ExecuteCommandRequest.EnvsEntry +} +var file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_depIdxs = []int32{ + 8, // 0: databricks.sandbox.ExecuteCommandRequest.envs:type_name -> databricks.sandbox.ExecuteCommandRequest.EnvsEntry + 2, // 1: databricks.sandbox.ExecuteCommandResponse.process_event:type_name -> databricks.sandbox.ProcessEvent + 3, // 2: databricks.sandbox.ProcessEvent.start:type_name -> databricks.sandbox.StartEvent + 4, // 3: databricks.sandbox.ProcessEvent.data:type_name -> databricks.sandbox.DataEvent + 5, // 4: databricks.sandbox.ProcessEvent.end:type_name -> databricks.sandbox.EndEvent + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_init() } +func file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_init() { + if File_experimental_ssh_sandbox_internal_sandboxexec_command_proto != nil { + return + } + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[2].OneofWrappers = []any{ + (*ProcessEvent_Start)(nil), + (*ProcessEvent_Data)(nil), + (*ProcessEvent_End)(nil), + } + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes[4].OneofWrappers = []any{ + (*DataEvent_Stdout)(nil), + (*DataEvent_Stderr)(nil), + (*DataEvent_Pty)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc), len(file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_rawDesc)), + NumEnums: 0, + NumMessages: 9, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_goTypes, + DependencyIndexes: file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_depIdxs, + MessageInfos: file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_msgTypes, + }.Build() + File_experimental_ssh_sandbox_internal_sandboxexec_command_proto = out.File + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_goTypes = nil + file_experimental_ssh_sandbox_internal_sandboxexec_command_proto_depIdxs = nil +} diff --git a/experimental/ssh/sandbox/internal/sandboxexec/command.proto b/experimental/ssh/sandbox/internal/sandboxexec/command.proto new file mode 100644 index 00000000000..82ed3ff8c07 --- /dev/null +++ b/experimental/ssh/sandbox/internal/sandboxexec/command.proto @@ -0,0 +1,36 @@ +syntax = "proto2"; +package databricks.sandbox; +option go_package = "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec"; + +// Streaming command messages used by the sandbox SSH client. +message ExecuteCommandRequest { + optional string cmd = 1; + repeated string args = 2; + map envs = 4; + optional string command_id = 5; +} +message ExecuteCommandResponse { + optional ProcessEvent process_event = 1; + optional int64 sequence_number = 2; +} +message ProcessEvent { + oneof event { + StartEvent start = 1; + DataEvent data = 2; + EndEvent end = 3; + } +} +message StartEvent { + optional int64 pid = 1; + optional string command_id = 2; +} +message DataEvent { + oneof output { + bytes stdout = 1; + bytes stderr = 2; + bytes pty = 3; + } +} +message EndEvent { optional int32 exit_code = 1; } +message TerminateCommandRequest { optional string command_id = 1; } +message TerminateCommandResponse {} diff --git a/experimental/ssh/sandbox/native_test.go b/experimental/ssh/sandbox/native_test.go new file mode 100644 index 00000000000..090bd7aef35 --- /dev/null +++ b/experimental/ssh/sandbox/native_test.go @@ -0,0 +1,142 @@ +package sandbox_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net" + "os" + "os/exec" + "os/user" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/shellquote" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNativeSSHAndSCPThroughWebSocket(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("sandbox servers run on Linux") + } + for _, binary := range []string{"ssh", "scp", "sftp", "python3", "ssh-keygen"} { + if _, err := exec.LookPath(binary); err != nil { + t.Skipf("native interoperability test requires %s", binary) + } + } + if _, err := os.Stat("/usr/sbin/sshd"); err != nil { + t.Skip("native interoperability test requires openssh-server") + } + actualHome, err := env.UserHomeDir(t.Context()) + require.NoError(t, err) + home, err := os.MkdirTemp(actualHome, ".sandbox-ssh-test-") //nolint:usetesting // sshd StrictModes rejects authorized_keys under the shared temporary directory. + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(home) }) + identity, err := sandbox.LocalIdentity(env.WithUserHomeDir(t.Context(), home), "https://workspace.test", "caller", "sandboxes/native") + require.NoError(t, err) + script, err := os.ReadFile("setup.py") + require.NoError(t, err) + request, err := json.Marshal(map[string]string{"public_key": string(identity.PublicKey)}) + require.NoError(t, err) + managed := filepath.Join(home, ".databricks", "sandbox-ssh") + require.NoError(t, os.MkdirAll(managed, 0o700)) + for _, directory := range []string{filepath.Dir(managed), managed} { + require.NoError(t, os.Chmod(directory, 0o770)) + } + setup := exec.CommandContext(t.Context(), "python3", "-c", "import os; os.umask(0o002)\n"+string(script), string(request)) + setup.Env = append(os.Environ(), "HOME="+home) + var diagnostics bytes.Buffer + setup.Stderr = &diagnostics + output, err := setup.Output() + require.NoError(t, err, diagnostics.String()) + var remote struct { + Config, SSHD, User string + HostKey string `json:"host_key"` + } + require.NoError(t, json.Unmarshal(output, &remote)) + require.NoError(t, identity.PinHostKey([]byte(remote.HostKey))) + t.Setenv("TZ", "UTC") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + reader, writer := io.Pipe() + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: remote.SSHD, Config: remote.Config, IdleTimeout: time.Minute}, writer) + writer.Close() + }() + var ready sandbox.Readiness + require.NoError(t, json.NewDecoder(reader).Decode(&ready)) + executable, err := os.Executable() + require.NoError(t, err) + t.Setenv("DATABRICKS_TEST_SANDBOX_PROXY_URL", "ws://"+listener.Addr().String()+"/ssh") + options := []string{"-F", "/dev/null", "-i", identity.KeyPath, "-o", "IdentitiesOnly=yes", "-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile=" + identity.KnownHostsPath, "-o", "HostKeyAlias=" + identity.HostKeyAlias, "-o", "ProxyCommand=" + shellquote.BashArg(executable), "-o", "ConnectTimeout=10"} + account, err := user.Current() + require.NoError(t, err) + destination := account.Username + "@" + identity.HostKeyAlias + ssh := exec.CommandContext(t.Context(), "ssh", append(append([]string{}, options...), destination, "printf 'SSH-byte-stream'; exit 17")...) + output, err = ssh.CombinedOutput() + var exit *exec.ExitError + require.ErrorAs(t, err, &exit, string(output)) + assert.Equal(t, 17, exit.ExitCode(), string(output)) + assert.Contains(t, string(output), "SSH-byte-stream") + pty := exec.CommandContext(t.Context(), "ssh", append(append([]string{}, options...), "-tt", destination, "test -t 0 && printf 'PTY-ready'")...) + output, err = pty.CombinedOutput() + require.NoError(t, err, string(output)) + assert.Contains(t, string(output), "PTY-ready") + + source := filepath.Join(t.TempDir(), "source.bin") + payload := bytes.Repeat([]byte{0, 1, 128, 255}, 10000) + require.NoError(t, os.WriteFile(source, payload, 0o600)) + remoteFile := filepath.Join(t.TempDir(), "remote.bin") + upload := exec.CommandContext(t.Context(), "scp", append(append([]string{}, options...), source, destination+":"+remoteFile)...) + output, err = upload.CombinedOutput() + require.NoError(t, err, string(output)) + copied, err := os.ReadFile(remoteFile) + require.NoError(t, err) + assert.Equal(t, payload, copied) + downloadFile := filepath.Join(t.TempDir(), "download.bin") + download := exec.CommandContext(t.Context(), "scp", append(append([]string{}, options...), destination+":"+remoteFile, downloadFile)...) + output, err = download.CombinedOutput() + require.NoError(t, err, string(output)) + copied, err = os.ReadFile(downloadFile) + require.NoError(t, err) + assert.Equal(t, payload, copied) + // SFTP exercises the subsystem used by modern SCP, even on hosts whose scp + // executable still defaults to the legacy SCP protocol. + batchFile := filepath.Join(t.TempDir(), "batch") + sftpDestination := filepath.Join(t.TempDir(), "sftp.bin") + require.NoError(t, os.WriteFile(batchFile, []byte("put "+source+" "+sftpDestination+"\n"), 0o600)) + transfer := exec.CommandContext(t.Context(), "sftp", append(append([]string{"-b", batchFile}, options...), destination)...) + output, err = transfer.CombinedOutput() + require.NoError(t, err, string(output)) + copied, err = os.ReadFile(sftpDestination) + require.NoError(t, err) + assert.Equal(t, payload, copied) + config, err := os.ReadFile(remote.Config) + require.NoError(t, err) + assert.Contains(t, string(config), "Subsystem sftp internal-sftp") + entries, err := filepath.Glob(filepath.Join(home, ".databricks", "sandbox-ssh", "*", "authorized_keys")) + require.NoError(t, err) + require.Len(t, entries, 1) + contents, err := os.ReadFile(entries[0]) + require.NoError(t, err) + end := strings.Index(string(contents), "\"") + require.GreaterOrEqual(t, end, 0) + expired := `expiry-time="20000101000000" ` + string(identity.PublicKey) + require.NoError(t, os.WriteFile(entries[0], []byte(expired), 0o600)) + rejected := exec.CommandContext(t.Context(), "ssh", append(append([]string{}, options...), destination, "true")...) + output, err = rejected.CombinedOutput() + require.Error(t, err) + assert.Contains(t, string(output), "Permission denied") + cancel() + require.ErrorIs(t, <-finished, context.Canceled) +} diff --git a/experimental/ssh/sandbox/releases.go b/experimental/ssh/sandbox/releases.go new file mode 100644 index 00000000000..856e8699985 --- /dev/null +++ b/experimental/ssh/sandbox/releases.go @@ -0,0 +1,91 @@ +package sandbox + +import ( + "archive/zip" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "runtime" + + "github.com/databricks/cli/experimental/ssh/internal/releases" + "github.com/databricks/cli/internal/build" + "github.com/databricks/databricks-sdk-go" + sdk "github.com/databricks/databricks-sdk-go/service/sandbox" +) + +const maxReleaseSize = 256 << 20 + +func uploadExecutable(ctx context.Context, w *databricks.WorkspaceClient, name string, setup *remoteSetup, releasesDir string) (string, error) { + architecture := map[string]string{"x86_64": "amd64", "aarch64": "arm64"}[setup.Architecture] + if architecture == "" { + return "", fmt.Errorf("unsupported sandbox architecture %q", setup.Architecture) + } + executable, err := linuxExecutable(ctx, architecture, releasesDir) + if err != nil { + return "", err + } + defer executable.Close() + destination := path.Join(setup.SessionDir, "databricks") + if err := w.Sandbox.UploadFile(ctx, sdk.UploadFileRequest{Parent: name, Path: destination, Contents: executable}); err != nil { + return "", fmt.Errorf("upload sandbox SSH executable: %w", err) + } + result, err := w.Sandbox.ExecuteCommandSync(ctx, sdk.ExecuteCommandSyncRequest{Name: name, Cmd: "chmod", Args: []string{"0700", destination}}) + if err != nil { + return "", err + } + if result.Status != sdk.ExecuteCommandStatusExecuteCommandStatusCompleted || result.ExitCode != 0 { + return "", fmt.Errorf("make sandbox SSH executable runnable: %s", result.Stderr) + } + return destination, nil +} + +func linuxExecutable(ctx context.Context, architecture, releasesDir string) (io.ReadCloser, error) { + if releasesDir == "" && runtime.GOOS == "linux" && runtime.GOARCH == architecture { + executable, err := os.Executable() + if err != nil { + return nil, err + } + return os.Open(executable) + } + version := build.GetInfo().Version + if releasesDir == "" && build.GetInfo().IsDevelopment() { + return nil, errors.New("a development build requires --releases-dir with compatible Linux CLI archives") + } + openRelease := releases.OpenPublished + if releasesDir != "" { + openRelease = releases.OpenLocal + } + archive, err := openRelease(ctx, architecture, version, releasesDir) + if err != nil { + return nil, err + } + defer archive.Close() + return executableFromArchive(archive) +} + +func executableFromArchive(archive io.Reader) (io.ReadCloser, error) { + data, err := io.ReadAll(io.LimitReader(archive, maxReleaseSize+1)) + if err != nil { + return nil, err + } + if len(data) > maxReleaseSize { + return nil, errors.New("linux CLI archive exceeds size limit") + } + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, fmt.Errorf("open Linux CLI archive: %w", err) + } + for _, file := range reader.File { + if file.Name == "databricks" { + if file.UncompressedSize64 > maxReleaseSize { + return nil, errors.New("linux CLI executable exceeds size limit") + } + return file.Open() + } + } + return nil, errors.New("linux CLI archive does not contain databricks") +} diff --git a/experimental/ssh/sandbox/releases_test.go b/experimental/ssh/sandbox/releases_test.go new file mode 100644 index 00000000000..935d9f08afc --- /dev/null +++ b/experimental/ssh/sandbox/releases_test.go @@ -0,0 +1,135 @@ +package sandbox_test + +import ( + "archive/zip" + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/internal/build" + "github.com/databricks/databricks-sdk-go" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExecutableFromArchive(t *testing.T) { + for _, name := range []string{"databricks", "../databricks", "other"} { + t.Run(name, func(t *testing.T) { + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + file, err := writer.Create(name) + require.NoError(t, err) + _, err = file.Write([]byte("executable")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + executable, err := sandbox.ExecutableFromArchiveForTest(&buffer) + if name != "databricks" { + require.ErrorContains(t, err, "does not contain databricks") + return + } + require.NoError(t, err) + defer executable.Close() + data, err := io.ReadAll(executable) + require.NoError(t, err) + assert.Equal(t, "executable", string(data)) + }) + } + _, err := sandbox.ExecutableFromArchiveForTest(bytes.NewBufferString("invalid zip")) + require.ErrorContains(t, err, "open Linux CLI archive") +} + +func TestUploadExecutableValidatesArchitectureAndChmod(t *testing.T) { + for _, tc := range []struct { + architecture string + exit int + errorText string + }{ + {"riscv64", 0, "unsupported sandbox architecture"}, + {"x86_64", 2, "make sandbox SSH executable runnable"}, + {"aarch64", 0, ""}, + } { + t.Run(tc.architecture, func(t *testing.T) { + directory := t.TempDir() + for _, architecture := range []string{"amd64", "arm64"} { + filename := "databricks_cli_" + build.GetInfo().Version + "_linux_" + architecture + ".zip" + if build.GetInfo().IsDevelopment() { + filename = "databricks_cli_linux_" + architecture + ".zip" + } + file, err := os.Create(filepath.Join(directory, filename)) + require.NoError(t, err) + archive := zip.NewWriter(file) + entry, err := archive.Create("databricks") + require.NoError(t, err) + _, err = entry.Write([]byte("executable")) + require.NoError(t, err) + require.NoError(t, archive.Close()) + require.NoError(t, file.Close()) + } + uploads := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + if r.Method == http.MethodPut { + uploads++ + assert.Equal(t, "/api/2.0/sandbox-files/sandboxes/test/content", r.URL.Path) + assert.Equal(t, "/home/sandbox-agent/session/databricks", r.URL.Query().Get("path")) + data, err := io.ReadAll(r.Body) + assert.NoError(t, err) + assert.Equal(t, "executable", string(data)) + w.WriteHeader(http.StatusCreated) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"status": "EXECUTE_COMMAND_STATUS_COMPLETED", "exit_code": tc.exit, "stderr": "chmod failed"}) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token"}) + require.NoError(t, err) + _, err = sandbox.UploadExecutableForTest(t.Context(), w, "sandboxes/test", &sandbox.RemoteSetupForTest{Architecture: tc.architecture, SessionDir: "/home/sandbox-agent/session"}, directory) + if tc.errorText != "" { + require.ErrorContains(t, err, tc.errorText) + } else { + require.NoError(t, err) + } + if tc.architecture == "riscv64" { + assert.Zero(t, uploads) + } else { + assert.Equal(t, 1, uploads) + } + }) + } +} + +func TestLinuxExecutableUsesCurrentLinuxBuild(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires a Linux executable") + } + reader, err := sandbox.LinuxExecutableForTest(t.Context(), runtime.GOARCH, "") + require.NoError(t, err) + defer reader.Close() + got, err := io.ReadAll(reader) + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + expected, err := os.ReadFile(executable) + require.NoError(t, err) + assert.Equal(t, expected, got) +} + +func TestLinuxExecutableRequiresDevelopmentArchives(t *testing.T) { + if !build.GetInfo().IsDevelopment() { + t.Skip("requires a development build") + } + reader, err := sandbox.LinuxExecutableForTest(t.Context(), "different-architecture", "") + require.ErrorContains(t, err, "requires --releases-dir") + assert.Nil(t, reader) +} diff --git a/experimental/ssh/sandbox/remote.go b/experimental/ssh/sandbox/remote.go new file mode 100644 index 00000000000..55cd25d2caa --- /dev/null +++ b/experimental/ssh/sandbox/remote.go @@ -0,0 +1,304 @@ +package sandbox + +import ( + "context" + "crypto/tls" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + "uuid" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + sdk "github.com/databricks/databricks-sdk-go/service/sandbox" + "github.com/gorilla/websocket" +) + +//go:embed setup.py +var setupScript string + +type remoteSetup struct { + Home string `json:"home"` + Config string `json:"config"` + SSHD string `json:"sshd"` + User string `json:"user"` + HostKey string `json:"host_key"` + Architecture string `json:"architecture"` + SessionDir string `json:"session_dir"` +} + +// WaitForRunning preserves stopped-sandbox startup for SSH callers. +func WaitForRunning(ctx context.Context, w *databricks.WorkspaceClient, name string) error { + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + started := false + progressed := false + for { + sandbox, err := w.Sandbox.GetSandbox(ctx, sdk.GetSandboxRequest{Name: name}) + if err != nil { + return err + } + if sandbox.Status == nil { + return fmt.Errorf("sandbox %s has no runtime status", name) + } + switch sandbox.Status.State { + case sdk.SandboxStateSandboxStateRunning: + return nil + case sdk.SandboxStateSandboxStateStopped: + if started && progressed { + return fmt.Errorf("sandbox %s stopped before becoming ready", name) + } + // Start acknowledgement can precede the observable state transition. + if !started { + if _, err := w.Sandbox.StartSandbox(ctx, sdk.StartSandboxRequest{Name: name}); err != nil { + return err + } + started = true + } + case sdk.SandboxStateSandboxStateStopping, sdk.SandboxStateSandboxStatePending: + progressed = started + default: + return fmt.Errorf("sandbox %s has unsupported state %q", name, sandbox.Status.State) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } +} + +func prepareRemote(ctx context.Context, w *databricks.WorkspaceClient, name string, identity *Identity) (*remoteSetup, error) { + request, err := json.Marshal(map[string]string{"public_key": string(identity.PublicKey)}) + if err != nil { + return nil, err + } + result, err := w.Sandbox.ExecuteCommandSync(ctx, sdk.ExecuteCommandSyncRequest{Name: name, Cmd: "python3", Args: []string{"-c", setupScript, string(request)}}) + if err != nil { + return nil, fmt.Errorf("prepare sandbox SSH configuration: %w", err) + } + if result.Status != sdk.ExecuteCommandStatusExecuteCommandStatusCompleted || result.ExitCode != 0 || result.Truncated { + return nil, fmt.Errorf("sandbox SSH setup failed (%s, exit %d): %s", result.Status, result.ExitCode, result.Stderr) + } + var setup remoteSetup + if err := json.Unmarshal([]byte(result.Stdout), &setup); err != nil { + return nil, fmt.Errorf("read sandbox SSH configuration: %w", err) + } + if setup.User != "sandbox-agent" { + return nil, fmt.Errorf("sandbox SSH requires the sandbox-agent account, got %q", setup.User) + } + if setup.Config == "" || setup.Home == "" || setup.SSHD == "" || setup.User == "" || setup.SessionDir == "" { + return nil, errors.New("incomplete sandbox SSH configuration") + } + if err := identity.PinHostKey([]byte(setup.HostKey)); err != nil { + return nil, err + } + return &setup, nil +} + +// ConnectProxy prepares an invocation's server and carries the SSH stream. It +// never launches a local SSH client or writes diagnostics into the SSH stream. +func ConnectProxy(ctx context.Context, w *databricks.WorkspaceClient, name string, identity *Identity, releasesDir string, input io.ReadCloser, output, diagnostics io.Writer) error { + return connectProxy(ctx, w, name, identity, releasesDir, input, output, diagnostics, 10*time.Minute) +} + +func connectProxy(ctx context.Context, w *databricks.WorkspaceClient, name string, identity *Identity, releasesDir string, input io.ReadCloser, output, diagnostics io.Writer, startupTimeout time.Duration) (err error) { + ctx, cancelStartup := context.WithCancelCause(ctx) + defer cancelStartup(nil) + defer func() { + if err != nil && ctx.Err() != nil { + err = context.Cause(ctx) + } + }() + // Stop this timer after readiness; it must not limit an established session. + startupTimer := time.AfterFunc(startupTimeout, func() { cancelStartup(context.DeadlineExceeded) }) + defer startupTimer.Stop() + if err := WaitForRunning(ctx, w, name); err != nil { + return err + } + setup, err := prepareRemote(ctx, w, name, identity) + if err != nil { + return err + } + defer cleanupRemoteDirectory(ctx, w, name, setup.SessionDir) + binary, err := uploadExecutable(ctx, w, name, setup, releasesDir) + if err != nil { + return err + } + commandClient, err := sandboxexec.New(w, name) + if err != nil { + return err + } + defer commandClient.Close() + commandID := uuid.New().String() + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if err := commandClient.Terminate(cleanupCtx, commandID); err != nil { + log.Warnf(ctx, "Could not stop sandbox SSH server: %v", err) + } + }() + commandCtx, cancel := context.WithCancel(ctx) + defer cancel() + timer := time.AfterFunc(30*time.Second, func() { + cancelStartup(errors.New("timed out waiting for sandbox SSH server readiness")) + }) + defer timer.Stop() + stream, err := commandClient.Execute(commandCtx, commandID, binary, []string{"sandbox", "server", "--sshd-config", setup.Config, "--sshd", setup.SSHD, "--port", "0"}) + if err != nil { + return fmt.Errorf("start sandbox SSH server: %w", err) + } + ready := make(chan Readiness, 1) + stopped := make(chan error, 1) + go readServerOutput(stream, commandID, ready, stopped, diagnostics) + var port int + select { + case result := <-ready: + port = result.Port + case err := <-stopped: + return fmt.Errorf("sandbox SSH server exited before readiness: %w", err) + case <-ctx.Done(): + return ctx.Err() + } + if !timer.Stop() || !startupTimer.Stop() { + return errors.New("sandbox SSH startup timed out") + } + // The output stream stays attached so command failure cancels forwarding. + proxyCtx, cancelProxy := context.WithCancelCause(ctx) + defer cancelProxy(nil) + go func() { + select { + case err := <-stopped: + cancelProxy(fmt.Errorf("sandbox SSH server stopped: %w", err)) + case <-proxyCtx.Done(): + } + }() + proxyErr := proxyStream(proxyCtx, input, output, + func(ctx context.Context, request proxy.DialRequest) (*websocket.Conn, error) { + endpoint, err := forwardingURL(w.Config.Host, name, port, request.ConnID) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + for key, value := range auth.WorkspaceIDHeaders(w.Config) { + req.Header.Set(key, value) + } + if err := w.Config.Authenticate(req); err != nil { + return nil, err + } + dialer := *websocket.DefaultDialer + dialer.HandshakeTimeout = 30 * time.Second + dialer.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: w.Config.InsecureSkipVerify} + conn, response, err := dialer.DialContext(ctx, endpoint, req.Header) + if response != nil { + response.Body.Close() + } + return conn, err + }) + // Seal completion before the server-output watcher can report a later exit. + cancelProxy(nil) + if cause := context.Cause(proxyCtx); cause != nil && !errors.Is(cause, context.Canceled) { + return cause + } + return proxyErr +} + +func forwardingURL(host, name string, port int, id string) (string, error) { + endpoint, err := url.Parse(host) + if err != nil || endpoint.Scheme != "https" { + return "", errors.New("port forwarding requires an HTTPS workspace URL") + } + if !strings.HasPrefix(name, "sandboxes/") || strings.ContainsAny(strings.TrimPrefix(name, "sandboxes/"), "/?#") { + return "", fmt.Errorf("invalid sandbox resource name %q", name) + } + endpoint.Scheme = "wss" + endpoint.Path = fmt.Sprintf("/sandbox-port-forward/%s/ports/%d/ssh", name, port) + endpoint.RawQuery = url.Values{"id": {id}}.Encode() + return endpoint.String(), nil +} + +func readServerOutput(stream interface { + Recv() (*sandboxexec.ExecuteCommandResponse, error) +}, commandID string, ready chan<- Readiness, stopped chan<- error, diagnostics io.Writer, +) { + var stdout strings.Builder + announced := false + for { + response, err := stream.Recv() + if err != nil { + stopped <- err + return + } + event := response.GetProcessEvent() + if start := event.GetStart(); start != nil && start.GetCommandId() != commandID { + stopped <- errors.New("sandbox returned an unexpected command ID") + return + } + if event.GetEnd() != nil { + stopped <- fmt.Errorf("server exited with status %d", event.GetEnd().GetExitCode()) + return + } + if data := event.GetData(); data != nil { + if _, err := diagnostics.Write(data.GetStderr()); err != nil { + stopped <- err + return + } + if !announced { + stdout.Write(data.GetStdout()) + if stdout.Len() > 4096 { + stopped <- errors.New("sandbox server readiness output is too large") + return + } + if strings.Contains(stdout.String(), "\n") { + var result Readiness + if err := json.Unmarshal([]byte(strings.SplitN(stdout.String(), "\n", 2)[0]), &result); err != nil { + stopped <- err + return + } + if result.Port < 1024 || result.Port > 65535 { + stopped <- fmt.Errorf("invalid sandbox relay port %d", result.Port) + return + } + ready <- result + announced = true + } + } + } + } +} + +func cleanupRemoteDirectory(ctx context.Context, w *databricks.WorkspaceClient, name, directory string) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + _, err := w.Sandbox.ExecuteCommandSync(ctx, sdk.ExecuteCommandSyncRequest{Name: name, Cmd: "rm", Args: []string{"-rf", "--", directory}}) + if err != nil { + log.Debugf(ctx, "Could not remove sandbox SSH session files: %v", err) + } +} + +// A read from inherited stdin can remain blocked after Close on Unix. Keep that +// read outside the shared proxy's wait group so a closed WebSocket can end the +// standalone proxy command. Process exit reclaims any remaining stdin read. +func proxyStream(ctx context.Context, input io.ReadCloser, output io.Writer, dial func(context.Context, proxy.DialRequest) (*websocket.Conn, error)) error { + reader, writer := io.Pipe() + defer input.Close() + defer reader.Close() + defer writer.Close() + go func() { + _, err := io.Copy(writer, input) + writer.CloseWithError(err) + }() + return proxy.RunClientProxy(ctx, reader, output, func() <-chan time.Time { return nil }, 30*time.Second, false, dial) +} diff --git a/experimental/ssh/sandbox/remote_integration_test.go b/experimental/ssh/sandbox/remote_integration_test.go new file mode 100644 index 00000000000..6734582bbdf --- /dev/null +++ b/experimental/ssh/sandbox/remote_integration_test.go @@ -0,0 +1,232 @@ +package sandbox_test + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +type execService any + +type remoteFixture struct { + mode string + publicKey string + connected chan struct{} + mu sync.Mutex + terminated, removed, uploaded bool + authenticated bool + commandID string +} + +func (fixture *remoteFixture) execute(_ any, stream grpc.ServerStream) error { + request := &sandboxexec.ExecuteCommandRequest{} + if err := stream.RecvMsg(request); err != nil { + return err + } + fixture.mu.Lock() + fixture.commandID = request.GetCommandId() + fixture.mu.Unlock() + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Start{Start: &sandboxexec.StartEvent{CommandId: request.CommandId}}}}); err != nil { + return err + } + switch fixture.mode { + case "before-ready": + case "cancel", "startup-timeout": + <-stream.Context().Done() + return stream.Context().Err() + case "after-ready", "clean-close", "slow-session": + if err := stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Data{Data: &sandboxexec.DataEvent{Output: &sandboxexec.DataEvent_Stdout{Stdout: []byte("{\"port\":40000}\n")}}}}}); err != nil { + return err + } + select { + case <-fixture.connected: + case <-stream.Context().Done(): + return stream.Context().Err() + } + if fixture.mode == "clean-close" || fixture.mode == "slow-session" { + <-stream.Context().Done() + return stream.Context().Err() + } + } + return stream.SendMsg(&sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_End{End: &sandboxexec.EndEvent{ExitCode: proto.Int32(17)}}}}) +} + +func (fixture *remoteFixture) terminate(_ any, _ context.Context, decode func(any) error, _ grpc.UnaryServerInterceptor) (any, error) { + request := &sandboxexec.TerminateCommandRequest{} + if err := decode(request); err != nil { + return nil, err + } + fixture.mu.Lock() + defer fixture.mu.Unlock() + if request.GetCommandId() != fixture.commandID { + return nil, errors.New("termination used a different command ID") + } + fixture.terminated = true + return &sandboxexec.TerminateCommandResponse{}, nil +} + +func (fixture *remoteFixture) serveHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/.well-known/databricks-config": + _, _ = io.WriteString(w, `{}`) + case r.URL.Path == "/api/2.0/sandboxes/test": + _, _ = io.WriteString(w, `{"status":{"state":"SANDBOX_STATE_RUNNING"}}`) + case strings.HasPrefix(r.URL.Path, "/api/2.0/sandbox-files/"): + fixture.mu.Lock() + fixture.uploaded = true + fixture.mu.Unlock() + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(http.StatusCreated) + case strings.HasSuffix(r.URL.Path, "/exec-sync"): + var request struct { + Cmd string `json:"cmd"` + Args []string `json:"args"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + result := map[string]any{"status": "EXECUTE_COMMAND_STATUS_COMPLETED"} + switch request.Cmd { + case "python3": + setup, _ := json.Marshal(map[string]string{"home": "/home/sandbox-agent", "config": "/home/sandbox-agent/session/sshd_config", "sshd": "/usr/sbin/sshd", "user": "sandbox-agent", "host_key": fixture.publicKey, "session_dir": "/home/sandbox-agent/session", "architecture": "x86_64"}) + result["stdout"] = string(setup) + case "rm": + fixture.mu.Lock() + fixture.removed = true + fixture.mu.Unlock() + } + _ = json.NewEncoder(w).Encode(result) + case strings.HasPrefix(r.URL.Path, "/sandbox-port-forward/"): + if r.Header.Get("Authorization") != "Bearer test-token" || r.Header.Get("X-Databricks-Workspace-Id") != "123" { + http.Error(w, "missing forwarding credentials", http.StatusUnauthorized) + return + } + fixture.mu.Lock() + fixture.authenticated = true + fixture.mu.Unlock() + upgrader := websocket.Upgrader{} + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + _ = conn.WriteMessage(websocket.BinaryMessage, []byte("SSH-2.0-test\r\n")) + close(fixture.connected) + if fixture.mode == "slow-session" { + time.Sleep(1500 * time.Millisecond) + } + if fixture.mode == "clean-close" || fixture.mode == "slow-session" { + _ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + return + } + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } + default: + http.NotFound(w, r) + } +} + +func TestConnectProxyCleansUpAfterCompletionFailureAndCancellation(t *testing.T) { + for _, mode := range []string{"before-ready", "after-ready", "cancel", "clean-close", "startup-timeout", "slow-session"} { + t.Run(mode, func(t *testing.T) { + home := t.TempDir() + ctx := env.WithUserHomeDir(t.Context(), home) + identity, err := sandbox.LocalIdentity(ctx, "https://workspace.test", "caller", "sandboxes/test") + require.NoError(t, err) + fixture := &remoteFixture{mode: mode, publicKey: string(identity.PublicKey), connected: make(chan struct{})} + grpcServer := grpc.NewServer() + grpcServer.RegisterService(&grpc.ServiceDesc{ServiceName: "databricks.sandbox.SandboxService", HandlerType: (*execService)(nil), Streams: []grpc.StreamDesc{{StreamName: "ExecuteCommand", ServerStreams: true, Handler: fixture.execute}}, Methods: []grpc.MethodDesc{{MethodName: "TerminateCommand", Handler: fixture.terminate}}}, fixture) + defer grpcServer.Stop() + endpoint := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") { + grpcServer.ServeHTTP(w, r) + return + } + fixture.serveHTTP(w, r) + })) + endpoint.EnableHTTP2 = true + endpoint.StartTLS() + defer endpoint.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: endpoint.URL, Token: "test-token", WorkspaceID: "123", InsecureSkipVerify: true, HTTPTransport: endpoint.Client().Transport}) + require.NoError(t, err) + filename := "databricks_cli_" + build.GetInfo().Version + "_linux_amd64.zip" + if build.GetInfo().IsDevelopment() { + filename = "databricks_cli_linux_amd64.zip" + } + var buffer bytes.Buffer + archive := zip.NewWriter(&buffer) + file, err := archive.Create("databricks") + require.NoError(t, err) + _, err = file.Write([]byte("executable")) + require.NoError(t, err) + require.NoError(t, archive.Close()) + require.NoError(t, os.WriteFile(filepath.Join(home, filename), buffer.Bytes(), 0o600)) + if mode == "cancel" { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + } + input, writer := io.Pipe() + defer input.Close() + defer writer.Close() + started := time.Now() + startupTimeout := 10 * time.Minute + if mode == "startup-timeout" || mode == "slow-session" { + startupTimeout = time.Second + } + err = sandbox.ConnectProxyForTest(ctx, w, "sandboxes/test", identity, home, input, io.Discard, io.Discard, startupTimeout) + if mode == "clean-close" || mode == "slow-session" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + switch mode { + case "before-ready": + assert.Contains(t, err.Error(), "exited before readiness") + case "after-ready": + assert.Contains(t, err.Error(), "server stopped") + case "cancel": + assert.ErrorIs(t, err, context.DeadlineExceeded) + case "startup-timeout": + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, time.Since(started), 5*time.Second) + case "slow-session": + assert.Greater(t, time.Since(started), startupTimeout) + } + fixture.mu.Lock() + defer fixture.mu.Unlock() + if mode == "after-ready" || mode == "clean-close" { + assert.True(t, fixture.authenticated) + } + assert.True(t, fixture.uploaded) + assert.True(t, fixture.terminated, "server command must be terminated independently of stream cancellation") + assert.True(t, fixture.removed, "session files must be removed on failures") + }) + } +} diff --git a/experimental/ssh/sandbox/remote_test.go b/experimental/ssh/sandbox/remote_test.go new file mode 100644 index 00000000000..26f0ed5250c --- /dev/null +++ b/experimental/ssh/sandbox/remote_test.go @@ -0,0 +1,213 @@ +package sandbox_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/databricks/cli/libs/env" + "github.com/gorilla/websocket" + + "github.com/databricks/cli/experimental/ssh/sandbox/internal/sandboxexec" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWaitForRunningLifecycle(t *testing.T) { + for _, states := range [][]string{ + {"RUNNING"}, {"STOPPED", "RUNNING"}, {"STOPPING", "STOPPED", "PENDING", "RUNNING"}, {"PENDING", "RUNNING"}, {"UNSPECIFIED"}, {"STOPPED", "STOPPED", "RUNNING"}, {"STOPPED", "PENDING", "STOPPED"}, + } { + t.Run(strings.Join(states, "-"), func(t *testing.T) { + index, starts := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + assert.Equal(t, "/api/2.0/sandboxes/test", strings.TrimSuffix(r.URL.Path, "/start")) + assert.Equal(t, "123", r.Header.Get("X-Databricks-Workspace-Id")) + if r.Method == http.MethodPost { + starts++ + _, _ = io.WriteString(w, `{}`) + return + } + state := states[min(index, len(states)-1)] + index++ + _ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]string{"state": "SANDBOX_STATE_" + state}}) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token", WorkspaceID: "123"}) + require.NoError(t, err) + err = sandbox.WaitForRunning(t.Context(), w, "sandboxes/test") + if states[len(states)-1] != "RUNNING" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + if strings.Contains(strings.Join(states, "-"), "STOPPED") { + assert.Equal(t, 1, starts) + } else { + assert.Zero(t, starts) + } + }) + } +} + +type responseStream struct { + events []*sandboxexec.ExecuteCommandResponse +} + +func (stream *responseStream) Recv() (*sandboxexec.ExecuteCommandResponse, error) { + if len(stream.events) == 0 { + return nil, io.EOF + } + event := stream.events[0] + stream.events = stream.events[1:] + return event, nil +} + +func TestReadServerOutput(t *testing.T) { + response := func(data string) *sandboxexec.ExecuteCommandResponse { + return &sandboxexec.ExecuteCommandResponse{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Data{Data: &sandboxexec.DataEvent{Output: &sandboxexec.DataEvent_Stdout{Stdout: []byte(data)}}}}} + } + for _, tc := range []struct { + name string + chunks []string + port int + errorText string + }{ + {"fragmented", []string{"{\"port\":4", "0000}\n"}, 40000, "EOF"}, + {"invalid-json", []string{"hello\n"}, 0, "invalid character"}, + {"invalid-port", []string{"{\"port\":22}\n"}, 0, "invalid sandbox relay port"}, + {"too-large", []string{strings.Repeat("x", 4097)}, 0, "too large"}, + } { + t.Run(tc.name, func(t *testing.T) { + stream := &responseStream{} + for _, chunk := range tc.chunks { + stream.events = append(stream.events, response(chunk)) + } + ready := make(chan sandbox.Readiness, 1) + stopped := make(chan error, 1) + sandbox.ReadServerOutputForTest(stream, "command", ready, stopped, io.Discard) + require.ErrorContains(t, <-stopped, tc.errorText) + if tc.port != 0 { + assert.Equal(t, tc.port, (<-ready).Port) + } else { + assert.Empty(t, ready) + } + }) + } + ready := make(chan sandbox.Readiness, 1) + stopped := make(chan error, 1) + stream := &responseStream{events: []*sandboxexec.ExecuteCommandResponse{{ProcessEvent: &sandboxexec.ProcessEvent{Event: &sandboxexec.ProcessEvent_Start{Start: &sandboxexec.StartEvent{CommandId: new("wrong")}}}}}} + sandbox.ReadServerOutputForTest(stream, "expected", ready, stopped, io.Discard) + require.ErrorContains(t, <-stopped, "unexpected command ID") +} + +func TestForwardingURL(t *testing.T) { + endpoint, err := sandbox.ForwardingURLForTest("https://workspace.test", "sandboxes/test", 40000, "connection") + require.NoError(t, err) + assert.Equal(t, "wss://workspace.test/sandbox-port-forward/sandboxes/test/ports/40000/ssh?id=connection", endpoint) + _, err = sandbox.ForwardingURLForTest("http://workspace.test", "sandboxes/test", 40000, "connection") + require.Error(t, err) + _, err = sandbox.ForwardingURLForTest("https://workspace.test", "sandboxes/../other", 40000, "connection") + require.Error(t, err) +} + +func TestPrepareRemoteRejectsPartialSetup(t *testing.T) { + home := t.TempDir() + ctx := env.WithUserHomeDir(t.Context(), home) + identity, err := sandbox.LocalIdentity(ctx, "https://workspace.test", "caller", "sandboxes/test") + require.NoError(t, err) + valid, _ := json.Marshal(map[string]string{"home": "/home/sandbox-agent", "config": "/home/sandbox-agent/config", "sshd": "/usr/sbin/sshd", "user": "sandbox-agent", "host_key": string(identity.PublicKey), "session_dir": "/home/sandbox-agent/session", "architecture": "x86_64"}) + for _, tc := range []struct { + name, status, stdout string + code int + truncated bool + errorText string + }{ + {name: "valid", status: "COMPLETED", stdout: string(valid)}, + {name: "failed", status: "FAILED", errorText: "setup failed"}, + {name: "exit", status: "COMPLETED", code: 2, errorText: "setup failed"}, + {name: "truncated", status: "COMPLETED", truncated: true, errorText: "setup failed"}, + {name: "invalid-json", status: "COMPLETED", stdout: "hello", errorText: "read sandbox SSH configuration"}, + {name: "unsupported-user", status: "COMPLETED", stdout: `{"user":"root"}`, errorText: "requires the sandbox-agent account"}, + {name: "incomplete", status: "COMPLETED", stdout: `{"user":"sandbox-agent"}`, errorText: "incomplete sandbox SSH configuration"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = io.WriteString(w, `{}`) + return + } + assert.Equal(t, "/api/2.0/sandbox-exec/sandboxes/test/exec-sync", r.URL.Path) + _ = json.NewEncoder(w).Encode(map[string]any{"status": "EXECUTE_COMMAND_STATUS_" + tc.status, "stdout": tc.stdout, "exit_code": tc.code, "truncated": tc.truncated}) + })) + defer server.Close() + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token"}) + require.NoError(t, err) + _, err = sandbox.PrepareRemoteForTest(ctx, w, "sandboxes/test", identity) + if tc.errorText != "" { + require.ErrorContains(t, err, tc.errorText) + } else { + require.NoError(t, err) + contents, err := os.ReadFile(identity.KnownHostsPath) + require.NoError(t, err) + assert.Contains(t, string(contents), identity.HostKeyAlias) + } + }) + } +} + +// Closing an inherited blocking descriptor need not interrupt an in-flight read. +type blockedInput struct{ started, release, finished chan struct{} } + +func (input *blockedInput) Read([]byte) (int, error) { + close(input.started) + <-input.release + close(input.finished) + return 0, io.EOF +} +func (*blockedInput) Close() error { return nil } + +func TestProxyStreamStopsWithoutWaitingForBlockedStdin(t *testing.T) { + input := &blockedInput{started: make(chan struct{}), release: make(chan struct{}), finished: make(chan struct{})} + defer func() { close(input.release); <-input.finished }() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{} + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + _ = conn.WriteMessage(websocket.BinaryMessage, []byte("SSH-2.0-test\r\n")) + <-input.started + _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) + })) + defer server.Close() + finished := make(chan error, 1) + go func() { + finished <- sandbox.ProxyStreamForTest(t.Context(), input, io.Discard, func(ctx context.Context, _ proxy.DialRequest) (*websocket.Conn, error) { + conn, response, err := websocket.DefaultDialer.DialContext(ctx, strings.Replace(server.URL, "http://", "ws://", 1), nil) + if response != nil { + response.Body.Close() + } + return conn, err + }) + }() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("proxy waited for blocked inherited stdin after WebSocket closure") + } +} diff --git a/experimental/ssh/sandbox/server.go b/experimental/ssh/sandbox/server.go index 2c1cb2f0a60..1c95caf38d2 100644 --- a/experimental/ssh/sandbox/server.go +++ b/experimental/ssh/sandbox/server.go @@ -17,9 +17,10 @@ import ( // ServerOptions describes a relay whose SSH configuration was prepared by the caller. type ServerOptions struct { - SSHD string - Config string - IdleTimeout time.Duration + SSHD string + Config string + IdleTimeout time.Duration + OutputHeartbeat time.Duration } // Readiness is written to command output only after the listener has been bound. @@ -29,6 +30,8 @@ type Readiness struct { // Serve runs a sandbox relay without workspace credentials or compute initialization. // Each invocation owns its listener and all the SSH processes it starts. +// With heartbeats enabled, output must support Close interrupting pending writes; +// Serve closes it during shutdown and waits for the output worker to finish. func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, output io.Writer) error { defer listener.Close() if opts.IdleTimeout <= 0 { @@ -37,6 +40,14 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu if opts.Config == "" || opts.SSHD == "" { return errors.New("sshd executable and configuration are required") } + var heartbeatOutput io.WriteCloser + if opts.OutputHeartbeat > 0 { + var ok bool + heartbeatOutput, ok = output.(io.WriteCloser) + if !ok { + return errors.New("command heartbeat output must support closing") + } + } // Validate the configuration before announcing readiness. check := exec.CommandContext(ctx, opts.SSHD, "-t", "-f", opts.Config) if diagnostic, err := check.CombinedOutput(); err != nil { @@ -96,8 +107,36 @@ func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, outpu if err := json.NewEncoder(output).Encode(Readiness{Port: listener.Addr().(*net.TCPAddr).Port}); err != nil { return fmt.Errorf("report relay readiness: %w", err) } + heartbeatErrors := make(chan error, 1) + if opts.OutputHeartbeat > 0 { + heartbeatDone := make(chan struct{}) + defer func() { + cancel() + heartbeatOutput.Close() + <-heartbeatDone + }() + go func() { + defer close(heartbeatDone) + ticker := time.NewTicker(opts.OutputHeartbeat) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + // SSH traffic travels over a separate WebSocket connection. + if _, err := io.WriteString(heartbeatOutput, "\n"); err != nil { + heartbeatErrors <- err + return + } + } + } + }() + } for { select { + case err := <-heartbeatErrors: + return fmt.Errorf("write command heartbeat: %w", err) case <-ctx.Done(): return ctx.Err() case <-connections.TimedOut: diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go index f60683d9cc2..a51bd9af519 100644 --- a/experimental/ssh/sandbox/server_test.go +++ b/experimental/ssh/sandbox/server_test.go @@ -12,6 +12,7 @@ import ( "os" "testing" "time" + "uuid" "github.com/databricks/cli/experimental/ssh/internal/proxy" "github.com/databricks/cli/experimental/ssh/sandbox" @@ -21,6 +22,20 @@ import ( ) func TestMain(m *testing.M) { + if endpoint := os.Getenv("DATABRICKS_TEST_SANDBOX_PROXY_URL"); endpoint != "" { + // TestMain also runs as a standalone SSH transport subprocess. + //nolint:gocritic // TestMain has no testing.T context. + err := sandbox.ProxyStreamForTest(context.Background(), os.Stdin, os.Stdout, func(ctx context.Context, request proxy.DialRequest) (*websocket.Conn, error) { + conn, err := dialWebSocket(ctx, endpoint+"?id="+uuid.New().String()) + return conn, err + }) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + } + if os.Getenv("DATABRICKS_TEST_SANDBOX_SSHD") == "1" { if os.Args[1] == "-t" { os.Exit(0) @@ -95,6 +110,38 @@ func TestServerIdleShutdown(t *testing.T) { } } +func TestCommandHeartbeatDoesNotPreventIdleShutdown(t *testing.T) { + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + reader, writer := io.Pipe() + defer reader.Close() + t.Cleanup(func() { writer.Close() }) + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: executable, Config: "test-config", IdleTimeout: time.Second, OutputHeartbeat: 10 * time.Millisecond}, writer) + }() + var ready sandbox.Readiness + require.NoError(t, json.NewDecoder(reader).Decode(&ready)) + var newline [1]byte + _, err = io.ReadFull(reader, newline[:]) + require.NoError(t, err) + assert.Equal(t, byte('\n'), newline[0]) + // Stop draining output: the next heartbeat blocks, but idle shutdown must not. + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("stalled command output prevented idle shutdown") + } + _, err = writer.Write([]byte("after shutdown")) + require.ErrorIs(t, err, io.ErrClosedPipe) +} + func TestServerRejectsInvalidConfigurationBeforeReadiness(t *testing.T) { listener, err := net.Listen("tcp4", "127.0.0.1:0") require.NoError(t, err) diff --git a/experimental/ssh/sandbox/setup.py b/experimental/ssh/sandbox/setup.py new file mode 100644 index 00000000000..b94336b618d --- /dev/null +++ b/experimental/ssh/sandbox/setup.py @@ -0,0 +1,102 @@ +import fcntl +import hashlib +import json +import os +import pwd +import shutil +import subprocess +import sys +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path + + +def atomic_write(path, content): + fd, temporary = tempfile.mkstemp(dir=path.parent) + try: + with os.fdopen(fd, "w") as output: + output.write(content) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +request = json.loads(sys.argv[1]) +sshd = shutil.which("sshd") or "/usr/sbin/sshd" +if not os.path.isfile(sshd): + raise RuntimeError("Install openssh-server in the sandbox before connecting") +user = pwd.getpwuid(os.getuid()).pw_name +home = Path.home() +if home.stat().st_mode & 0o022: + raise RuntimeError("SSH requires HOME not to be writable by group or other users") +parent = home / ".databricks" +parent.mkdir(mode=0o700, exist_ok=True) +# StrictModes checks every directory above authorized_keys, including existing ones. +parent.chmod(parent.stat().st_mode & ~0o022) +base = home / ".databricks" / "sandbox-ssh" +base.mkdir(mode=0o700, exist_ok=True) +base.chmod(0o700) +key = request["public_key"].strip() +if len(key.split()) != 2 or not key.startswith("ssh-ed25519 "): + raise ValueError("Expected one Ed25519 public key") +key_dir = base / hashlib.sha256(key.encode()).hexdigest() +key_dir.mkdir(mode=0o700, exist_ok=True) +key_dir.chmod(0o700) +# Each client key has its own file, so renewal cannot discard another caller's entry. +authorized = key_dir / "authorized_keys" +# The server runs with TZ=UTC. Older OpenSSH releases reject a trailing Z. +expiry = (datetime.now(timezone.utc) + timedelta(hours=24)).strftime("%Y%m%d%H%M%S") +atomic_write(authorized, 'expiry-time="' + expiry + '" ' + key + "\n") +# Server keys belong to this persistent HOME, and are never returned to the client. +# Serialize generation so simultaneous connections share the same server identity. +with open(base / "host-key.lock", "a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + host_key = base / "host_ed25519" + if not host_key.exists(): + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(host_key)], check=True) + host_public = subprocess.check_output(["ssh-keygen", "-y", "-f", str(host_key)], universal_newlines=True).strip() +session = Path(tempfile.mkdtemp(prefix="session-", dir=base)) +config = session / "sshd_config" + + +def quote(value): + value = str(value) + if any(c in value for c in ('"', "\n", "\r", "\\")): + raise ValueError("Unsupported character in SSH configuration path") + return '"' + value + '"' + + +atomic_write( + config, + "\n".join( + [ + "HostKey " + quote(host_key), + "AuthorizedKeysFile " + quote(authorized), + "PubkeyAuthentication yes", + "PasswordAuthentication no", + "KbdInteractiveAuthentication no", + # OpenSSH requires root for PAM; this server runs as sandbox-agent. + "UsePAM no", + "AllowUsers " + user, + "AllowTcpForwarding yes", + "ClientAliveInterval 30", + "Subsystem sftp internal-sftp", + "PidFile " + quote(session / "sshd.pid"), + ] + ) + + "\n", +) +print( + json.dumps( + { + "home": str(home), + "config": str(config), + "sshd": sshd, + "user": user, + "host_key": host_public, + "architecture": os.uname().machine, + "session_dir": str(session), + } + ) +) diff --git a/go.mod b/go.mod index db7b9e41457..b5ffc2770d7 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,8 @@ require ( golang.org/x/sys v0.49.0 // BSD-3-Clause golang.org/x/term v0.47.0 // BSD-3-Clause golang.org/x/text v0.43.0 // BSD-3-Clause + google.golang.org/grpc v1.83.2 // Apache-2.0 + google.golang.org/protobuf v1.36.11 // BSD-3-Clause gopkg.in/ini.v1 v1.67.3 // Apache-2.0 ) @@ -97,7 +99,5 @@ require ( golang.org/x/time v0.14.0 // indirect google.golang.org/api v0.265.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.83.2 // indirect - google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/build/notice_test.go b/internal/build/notice_test.go index abe1d39287d..bbc9651832c 100644 --- a/internal/build/notice_test.go +++ b/internal/build/notice_test.go @@ -14,9 +14,11 @@ import ( // moduleToGitHub maps non-GitHub go.mod paths to their GitHub org/repo slug. var moduleToGitHub = map[string]string{ - "gopkg.in/ini.v1": "go-ini/ini", - "go.yaml.in/yaml/v3": "yaml/go-yaml", - "dario.cat/mergo": "darccio/mergo", + "google.golang.org/grpc": "grpc/grpc-go", + "google.golang.org/protobuf": "protocolbuffers/protobuf-go", + "gopkg.in/ini.v1": "go-ini/ini", + "go.yaml.in/yaml/v3": "yaml/go-yaml", + "dario.cat/mergo": "darccio/mergo", } // Modules excluded from NOTICE requirements (Databricks-owned).