diff --git a/.golangci.yaml b/.golangci.yaml index f05060c41ae..8c4888b0661 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -57,6 +57,11 @@ linters: - "**" - "!**/cmd/cmd.go" - "!**/cmd/experimental/**" + # Sandbox SSH command files integrate the shared SSH implementation. + - "!**/cmd/sandbox/server.go" + - "!**/cmd/sandbox/ssh.go" + - "!**/cmd/sandbox/ssh_setup.go" + - "!**/cmd/sandbox/ssh_test.go" - "!**/experimental/**" deny: - pkg: "github.com/databricks/cli/experimental" diff --git a/cmd/sandbox/sandbox.go b/cmd/sandbox/sandbox.go index 9b40c16d370..294968b3eac 100644 --- a/cmd/sandbox/sandbox.go +++ b/cmd/sandbox/sandbox.go @@ -32,6 +32,7 @@ Common workflows: `, } + cmd.AddCommand(newServerCommand()) cmd.AddCommand(newRegisterCommand()) cmd.AddCommand(newSSHKeyCommand()) cmd.AddCommand(newSetDefaultCommand()) diff --git a/cmd/sandbox/server.go b/cmd/sandbox/server.go new file mode 100644 index 00000000000..d6e1d1ce602 --- /dev/null +++ b/cmd/sandbox/server.go @@ -0,0 +1,42 @@ +package sandbox + +import ( + "errors" + "fmt" + "net" + "time" + + sshtunnel "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/spf13/cobra" +) + +func newServerCommand() *cobra.Command { + var config, sshd string + var port int + var idleTimeout time.Duration + cmd := &cobra.Command{ + Use: "server", Short: "Run the sandbox WebSocket SSH relay", Hidden: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if port != 0 && (port < 1024 || port > 65535) { + return errors.New("port must be zero or between 1024 and 65535") + } + if idleTimeout <= 0 { + return errors.New("idle timeout must be positive") + } + listener, err := net.Listen("tcp4", fmt.Sprintf("0.0.0.0:%d", port)) + if err != nil { + return fmt.Errorf("listen for SSH WebSockets: %w", err) + } + return sshtunnel.Serve(cmd.Context(), listener, sshtunnel.ServerOptions{ + SSHD: sshd, Config: config, IdleTimeout: idleTimeout, + }, cmd.OutOrStdout()) + }, + } + cmd.Flags().StringVar(&config, "sshd-config", "", "Prepared sshd configuration file") + cmd.MarkFlagRequired("sshd-config") + cmd.Flags().StringVar(&sshd, "sshd", "/usr/sbin/sshd", "OpenSSH server executable") + cmd.Flags().IntVar(&port, "port", 40000, "HTTP/WebSocket listener port (0 selects a free port)") + cmd.Flags().DurationVar(&idleTimeout, "idle-timeout", time.Minute, "Stop after this duration without an SSH connection") + return cmd +} diff --git a/cmd/sandbox/server_test.go b/cmd/sandbox/server_test.go new file mode 100644 index 00000000000..95431b05b1d --- /dev/null +++ b/cmd/sandbox/server_test.go @@ -0,0 +1,56 @@ +package sandbox + +import ( + "bytes" + "encoding/json" + "net" + "os" + "path/filepath" + "runtime" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServerCommandValidatesArgumentsWithoutAuthentication(t *testing.T) { + for _, args := range [][]string{ + {"--sshd-config", "config", "--port", "22"}, + {"--sshd-config", "config", "--idle-timeout", "0s"}, + {"--sshd-config", "config", "unexpected"}, + } { + cmd := newServerCommand() + cmd.SetArgs(args) + err := cmd.ExecuteContext(t.Context()) + require.Error(t, err) + assert.NotContains(t, err.Error(), "credential") + } +} + +func TestServerCommandReportsReadiness(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("sandbox servers run on Linux") + } + binary := filepath.Join(t.TempDir(), "sshd") + require.NoError(t, os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o700)) + cmd := newServerCommand() + var output bytes.Buffer + cmd.SetOut(&output) + cmd.SetArgs([]string{"--sshd", binary, "--sshd-config", "prepared-config", "--port", "0", "--idle-timeout", "20ms"}) + require.NoError(t, cmd.ExecuteContext(t.Context())) + var ready struct { + Port int `json:"port"` + } + require.NoError(t, json.Unmarshal(output.Bytes(), &ready)) + assert.GreaterOrEqual(t, ready.Port, 1024) +} + +func TestServerCommandReportsOccupiedPort(t *testing.T) { + listener, err := net.Listen("tcp4", "0.0.0.0:0") + require.NoError(t, err) + defer listener.Close() + cmd := newServerCommand() + cmd.SetArgs([]string{"--sshd-config", "prepared-config", "--port", strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)}) + require.ErrorContains(t, cmd.ExecuteContext(t.Context()), "listen for SSH WebSockets") +} diff --git a/experimental/ssh/internal/proxy/connections.go b/experimental/ssh/internal/proxy/connections.go index 3e3991571dc..ed9c35d9331 100644 --- a/experimental/ssh/internal/proxy/connections.go +++ b/experimental/ssh/internal/proxy/connections.go @@ -22,7 +22,7 @@ func NewConnectionsManager(maxClients int, shutdownDelay time.Duration) *Connect maxClients: maxClients, shutdownDelay: shutdownDelay, connections: make(map[string]*proxyConnection), - TimedOut: make(chan bool), + TimedOut: make(chan bool, 1), } cm.startShutdownTimer(shutdownDelay) return cm @@ -86,7 +86,10 @@ func (cm *ConnectionsManager) startShutdownTimer(delay time.Duration) { cm.shutdownTimer.Stop() } cm.shutdownTimer = time.AfterFunc(delay, func() { - cm.TimedOut <- true + select { + case cm.TimedOut <- true: + default: + } }) } @@ -97,3 +100,8 @@ func (cm *ConnectionsManager) cancelShutdownTimer() { cm.shutdownTimer.Stop() } } + +// Close releases the idle timer when the relay owning this manager stops. +func (cm *ConnectionsManager) Close() { + cm.cancelShutdownTimer() +} diff --git a/experimental/ssh/internal/proxy/connections_test.go b/experimental/ssh/internal/proxy/connections_test.go index e05c92d1fcb..a55ed3ebe1c 100644 --- a/experimental/ssh/internal/proxy/connections_test.go +++ b/experimental/ssh/internal/proxy/connections_test.go @@ -239,3 +239,16 @@ func TestConnectionsManager_ShutdownTimer_NoRestartWhenConnectionsRemain(t *test assert.False(t, timedOut, "Unexpected timeout signal while connections still exist") }) } + +func TestConnectionsManagerCloseStopsIdleTimer(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cm := NewConnectionsManager(1, time.Second) + cm.Close() + time.Sleep(2 * time.Second) + select { + case <-cm.TimedOut: + t.Fatal("closed manager timed out") + default: + } + }) +} diff --git a/experimental/ssh/sandbox/server.go b/experimental/ssh/sandbox/server.go new file mode 100644 index 00000000000..2c1cb2f0a60 --- /dev/null +++ b/experimental/ssh/sandbox/server.go @@ -0,0 +1,121 @@ +package sandbox + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os/exec" + "sync" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" +) + +// ServerOptions describes a relay whose SSH configuration was prepared by the caller. +type ServerOptions struct { + SSHD string + Config string + IdleTimeout time.Duration +} + +// Readiness is written to command output only after the listener has been bound. +type Readiness struct { + Port int `json:"port"` +} + +// Serve runs a sandbox relay without workspace credentials or compute initialization. +// Each invocation owns its listener and all the SSH processes it starts. +func Serve(ctx context.Context, listener net.Listener, opts ServerOptions, output io.Writer) error { + defer listener.Close() + if opts.IdleTimeout <= 0 { + return errors.New("idle timeout must be positive") + } + if opts.Config == "" || opts.SSHD == "" { + return errors.New("sshd executable and configuration are required") + } + // Validate the configuration before announcing readiness. + check := exec.CommandContext(ctx, opts.SSHD, "-t", "-f", opts.Config) + if diagnostic, err := check.CombinedOutput(); err != nil { + return fmt.Errorf("validate sshd configuration: %w: %s", err, diagnostic) + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + connections := proxy.NewConnectionsManager(1, opts.IdleTimeout) + defer connections.Close() + handler := proxy.NewProxyServer(ctx, connections, func(ctx context.Context) *exec.Cmd { + return exec.CommandContext(ctx, opts.SSHD, "-i", "-f", opts.Config) + }) + + // One relay belongs to one invocation; the sandbox client does not negotiate + // compute's handover or resume protocols. + var requests sync.WaitGroup + var requestMu sync.Mutex + active, closing := false, false + mux := http.NewServeMux() + mux.HandleFunc("GET /ssh", func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + if query.Has("delivered") || query.Has("reattach") || query.Has("resume_version") { + http.Error(w, "Session resume is not supported by this relay", http.StatusBadRequest) + return + } + requestMu.Lock() + if active || closing { + requestMu.Unlock() + http.Error(w, "SSH connection already active or relay is stopping", http.StatusServiceUnavailable) + return + } + active = true + requests.Add(1) + requestMu.Unlock() + defer func() { + requestMu.Lock() + active = false + connections.ExtendIdleTimeout(opts.IdleTimeout) + requestMu.Unlock() + requests.Done() + }() + handler.ServeHTTP(w, r) + }) + server := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second} + stopped := make(chan struct{}) + var serveErr error + go func() { serveErr = server.Serve(listener); close(stopped) }() + defer func() { + requestMu.Lock() + closing = true + requestMu.Unlock() + cancel() + server.Close() + <-stopped + requests.Wait() + }() + if err := json.NewEncoder(output).Encode(Readiness{Port: listener.Addr().(*net.TCPAddr).Port}); err != nil { + return fmt.Errorf("report relay readiness: %w", err) + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-connections.TimedOut: + // A timer may fire just as a connection is admitted. Admission and the idle + // decision share a lock so an already queued timeout cannot end that session. + requestMu.Lock() + if active { + requestMu.Unlock() + continue + } + closing = true + requestMu.Unlock() + return nil + case <-stopped: + if errors.Is(serveErr, http.ErrServerClosed) { + return nil + } + return serveErr + } + } +} diff --git a/experimental/ssh/sandbox/server_test.go b/experimental/ssh/sandbox/server_test.go new file mode 100644 index 00000000000..f60683d9cc2 --- /dev/null +++ b/experimental/ssh/sandbox/server_test.go @@ -0,0 +1,254 @@ +package sandbox_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/databricks/cli/experimental/ssh/internal/proxy" + "github.com/databricks/cli/experimental/ssh/sandbox" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMain(m *testing.M) { + if os.Getenv("DATABRICKS_TEST_SANDBOX_SSHD") == "1" { + if os.Args[1] == "-t" { + os.Exit(0) + } + _, _ = io.Copy(os.Stdout, os.Stdin) + os.Exit(0) + } + os.Exit(m.Run()) +} + +func startServer(t *testing.T, idle time.Duration) (string, context.CancelFunc, <-chan error, net.Listener) { + t.Helper() + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + reader, writer := io.Pipe() + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: executable, Config: "test-config", IdleTimeout: idle}, writer) + writer.Close() + }() + var ready sandbox.Readiness + require.NoError(t, json.NewDecoder(reader).Decode(&ready)) + require.Equal(t, listener.Addr().(*net.TCPAddr).Port, ready.Port) + return "ws://" + listener.Addr().String() + "/ssh", cancel, finished, listener +} + +func TestServerRelaysBinaryDataAndCancels(t *testing.T) { + endpoint, cancel, finished, _ := startServer(t, time.Minute) + conn, err := dialWebSocket(t.Context(), endpoint+"?id=first") + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + payload := bytes.Repeat([]byte{0, 1, 127, 128, 255}, 1000) + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, payload)) + var received []byte + for len(received) < len(payload) { + kind, chunk, err := conn.ReadMessage() + require.NoError(t, err) + require.Equal(t, websocket.BinaryMessage, kind) + received = append(received, chunk...) + } + assert.Equal(t, payload, received) + second, response, err := websocket.DefaultDialer.DialContext(t.Context(), endpoint+"?id=second", nil) + if second != nil { + second.Close() + } + require.Error(t, err) + require.NotNil(t, response) + assert.Equal(t, http.StatusServiceUnavailable, response.StatusCode) + response.Body.Close() + cancel() + select { + case err := <-finished: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("server did not reap its child on cancellation") + } +} + +func TestServerIdleShutdown(t *testing.T) { + _, _, finished, _ := startServer(t, 50*time.Millisecond) + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("idle server did not stop") + } +} + +func TestServerRejectsInvalidConfigurationBeforeReadiness(t *testing.T) { + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + var output bytes.Buffer + err = sandbox.Serve(t.Context(), listener, sandbox.ServerOptions{SSHD: "missing-sshd", Config: "config", IdleTimeout: time.Minute}, &output) + require.ErrorContains(t, err, "validate sshd configuration") + assert.Empty(t, output.String()) + _, err = listener.Accept() + assert.ErrorIs(t, err, net.ErrClosed) +} + +func TestServerReleasesConnectionSlot(t *testing.T) { + endpoint, _, _, _ := startServer(t, time.Minute) + first, err := dialWebSocket(t.Context(), endpoint+"?id=first") + require.NoError(t, err) + first.Close() + var second *websocket.Conn + require.Eventually(t, func() bool { + conn, response, err := websocket.DefaultDialer.DialContext(t.Context(), endpoint+"?id=second", nil) + if response != nil { + response.Body.Close() + } + if err != nil { + return false + } + second = conn + return true + }, 5*time.Second, 10*time.Millisecond) + defer second.Close() + require.NoError(t, second.SetReadDeadline(time.Now().Add(5*time.Second))) + require.NoError(t, second.WriteMessage(websocket.BinaryMessage, []byte("second"))) + _, result, err := second.ReadMessage() + require.NoError(t, err) + assert.Equal(t, "second", string(result)) +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken output") } + +func TestServerReadinessWriteFailure(t *testing.T) { + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + err = sandbox.Serve(t.Context(), listener, sandbox.ServerOptions{SSHD: executable, Config: "test-config", IdleTimeout: time.Minute}, failingWriter{}) + require.ErrorContains(t, err, "report relay readiness") + conn, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second) + if conn != nil { + conn.Close() + } + require.Error(t, err) +} + +func TestServerListenerFailure(t *testing.T) { + _, _, finished, listener := startServer(t, time.Minute) + require.NoError(t, listener.Close()) + select { + case err := <-finished: + require.ErrorIs(t, err, net.ErrClosed) + case <-time.After(5 * time.Second): + t.Fatal("server did not return after listener failure") + } +} + +func TestServerRejectsResumeProtocol(t *testing.T) { + resume := fmt.Sprintf("?id=first&delivered=0&resume_version=%d", proxy.ResumeProtocolVersion) + for _, query := range []string{resume, resume + "&reattach=1"} { + t.Run(query, func(t *testing.T) { + endpoint, _, _, _ := startServer(t, time.Minute) + conn, response, err := websocket.DefaultDialer.DialContext(t.Context(), endpoint+query, nil) + if conn != nil { + conn.Close() + } + require.Error(t, err) + require.NotNil(t, response) + defer response.Body.Close() + assert.Equal(t, http.StatusBadRequest, response.StatusCode) + + conn, err = dialWebSocket(t.Context(), endpoint+"?id=plain") + require.NoError(t, err) + conn.Close() + }) + } +} + +func TestServerKeepsActiveConnectionPastIdleTimeout(t *testing.T) { + endpoint, cancel, finished, _ := startServer(t, 200*time.Millisecond) + conn, err := dialWebSocket(t.Context(), endpoint+"?id=active") + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, []byte("before"))) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + _, _, err = conn.ReadMessage() + require.NoError(t, err) + time.Sleep(300 * time.Millisecond) + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, []byte("after"))) + _, message, err := conn.ReadMessage() + require.NoError(t, err) + assert.Equal(t, "after", string(message)) + cancel() + require.ErrorIs(t, <-finished, context.Canceled) +} + +func dialWebSocket(ctx context.Context, endpoint string) (*websocket.Conn, error) { + conn, response, err := websocket.DefaultDialer.DialContext(ctx, endpoint, nil) + if response != nil { + response.Body.Close() + } + return conn, err +} + +// Blocking readiness lets the idle timer fire before Serve starts receiving it. +type blockedReadiness struct { + entered chan struct{} + release chan struct{} +} + +func (writer blockedReadiness) Write(data []byte) (int, error) { + close(writer.entered) + <-writer.release + return len(data), nil +} + +func TestServerIgnoresQueuedIdleTimeoutAndRestartsItAfterDisconnect(t *testing.T) { + t.Setenv("DATABRICKS_TEST_SANDBOX_SSHD", "1") + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + output := blockedReadiness{entered: make(chan struct{}), release: make(chan struct{})} + finished := make(chan error, 1) + go func() { + finished <- sandbox.Serve(ctx, listener, sandbox.ServerOptions{SSHD: executable, Config: "config", IdleTimeout: 50 * time.Millisecond}, output) + }() + <-output.entered + time.Sleep(100 * time.Millisecond) + conn, err := dialWebSocket(t.Context(), "ws://"+listener.Addr().String()+"/ssh?id=queued") + close(output.release) + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second))) + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, []byte("still active"))) + _, message, err := conn.ReadMessage() + require.NoError(t, err) + assert.Equal(t, "still active", string(message)) + conn.Close() + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("idle timeout was not restarted after disconnect") + } +}