diff --git a/.nextchanges/bundles/7647.md b/.nextchanges/bundles/7647.md new file mode 100644 index 00000000000..d3b25cebd7a --- /dev/null +++ b/.nextchanges/bundles/7647.md @@ -0,0 +1 @@ +* Fix `bundle plan` and `bundle deploy` failing to recreate a cluster deleted outside the bundle when its libraries status lookup responds first ([#7647](https://github.com/databricks/cli/pull/7647)). diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 82167080c4d..2bd1b9f2ef9 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" "github.com/databricks/cli/bundle/config/resources" @@ -17,7 +18,6 @@ import ( "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/compute" - "golang.org/x/sync/errgroup" ) // librariesWaitTimeout bounds how long we poll for libraries to finish installing. @@ -151,21 +151,27 @@ func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState { func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote, error) { var details *compute.ClusterDetails var libraries []compute.Library - - // The cluster GET and the library-status GET are independent, so run them concurrently. - g, ctx := errgroup.WithContext(ctx) - g.Go(func() error { - var err error - details, err = r.client.Clusters.GetByClusterId(ctx, id) - return err + var detailsErr, librariesErr error + + // The two GETs are independent, so they still run together. The result is not + // whichever error finishes first: library-status reports a deleted cluster as + // INVALID_PARAMETER_VALUE, which the planner does not treat as missing, and + // that error aborts the plan whenever it wins the race. A failed cluster read + // decides the outcome. A library error is returned only after the cluster + // itself was read, so a failure on a live cluster still surfaces. + var wg sync.WaitGroup + wg.Go(func() { + details, detailsErr = r.client.Clusters.GetByClusterId(ctx, id) }) - g.Go(func() error { - var err error - libraries, err = r.readLibraries(ctx, id) - return err + wg.Go(func() { + libraries, librariesErr = r.readLibraries(ctx, id) }) - if err := g.Wait(); err != nil { - return nil, err + wg.Wait() + if detailsErr != nil { + return nil, detailsErr + } + if librariesErr != nil { + return nil, librariesErr } remote := &ClusterRemote{ diff --git a/bundle/direct/dresources/cluster_test.go b/bundle/direct/dresources/cluster_test.go index 7d3af4440d4..8157e2aadfc 100644 --- a/bundle/direct/dresources/cluster_test.go +++ b/bundle/direct/dresources/cluster_test.go @@ -1,12 +1,17 @@ package dresources import ( + "context" "testing" + "time" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -66,3 +71,106 @@ func TestClusterOverrideChangeDescLibraries(t *testing.T) { }) } } + +const clusterDoReadID = "0101-000000-deleted00" + +// doReadOrdered runs DoRead with both lookups in flight. The lookup named by +// first returns immediately. The other waits until that call has started and +// then long enough for its result to be the one recorded first. +func doReadOrdered(t *testing.T, id string, details *compute.ClusterDetails, clusterErr error, statuses *compute.ClusterLibraryStatuses, libraryErr error, first string) (*ClusterRemote, error) { + t.Helper() + m := mocks.NewMockWorkspaceClient(t) + r := &ResourceCluster{client: m.WorkspaceClient} + + winnerStarted := make(chan struct{}) + release := func(name string) { + if name == first { + close(winnerStarted) + return + } + select { + case <-winnerStarted: + case <-t.Context().Done(): + return + } + time.Sleep(20 * time.Millisecond) + } + + m.GetMockClustersAPI().EXPECT(). + GetByClusterId(mock.Anything, id). + RunAndReturn(func(context.Context, string) (*compute.ClusterDetails, error) { + release("cluster") + return details, clusterErr + }).Once() + m.GetMockLibrariesAPI().EXPECT(). + ClusterStatusByClusterId(mock.Anything, id). + RunAndReturn(func(context.Context, string) (*compute.ClusterLibraryStatuses, error) { + release("libraries") + return statuses, libraryErr + }).Once() + + return r.DoRead(t.Context(), id) +} + +func TestClusterDoReadDeletedClusterIgnoresLibraryStatusOrder(t *testing.T) { + clusterMissing := &apierr.APIError{ + StatusCode: 400, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + Message: "Cluster " + clusterDoReadID + " does not exist", + } + // library-status uses this code for the same missing cluster. The SDK remaps + // it to "not found" only on GET /clusters/get, so IsMissing is false here. + libraryMissing := &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "Cluster " + clusterDoReadID + " does not exist", + } + + for _, first := range []string{"libraries", "cluster"} { + t.Run(first+" finishes first", func(t *testing.T) { + _, err := doReadOrdered(t, clusterDoReadID, nil, clusterMissing, nil, libraryMissing, first) + require.ErrorIs(t, err, apierr.ErrResourceDoesNotExist) + require.NotErrorIs(t, err, apierr.ErrInvalidParameterValue) + }) + } +} + +func TestClusterDoReadLibraryErrorSurfacesWhenClusterExists(t *testing.T) { + details := &compute.ClusterDetails{ClusterId: clusterDoReadID, State: compute.StateRunning} + libraryErr := &apierr.APIError{ + StatusCode: 500, + ErrorCode: "UNKNOWN", + Message: "library status unavailable", + } + + for _, first := range []string{"libraries", "cluster"} { + t.Run(first+" finishes first", func(t *testing.T) { + _, err := doReadOrdered(t, clusterDoReadID, details, nil, nil, libraryErr, first) + require.ErrorIs(t, err, libraryErr) + require.NotErrorIs(t, err, apierr.ErrNotFound) + }) + } +} + +func TestClusterDoReadReturnsInstalledLibraries(t *testing.T) { + details := &compute.ClusterDetails{ + ClusterId: clusterDoReadID, + State: compute.StateRunning, + Spec: &compute.ClusterSpec{ApplyPolicyDefaultValues: true}, + } + statuses := &compute.ClusterLibraryStatuses{ + LibraryStatuses: []compute.LibraryFullStatus{ + {Library: &compute.Library{Whl: "/Workspace/a.whl"}, Status: compute.LibraryInstallStatusInstalled}, + {Library: &compute.Library{Jar: "/Workspace/all.jar"}, IsLibraryForAllClusters: true, Status: compute.LibraryInstallStatusInstalled}, + }, + } + + remote, err := doReadOrdered(t, clusterDoReadID, details, nil, statuses, nil, "libraries") + require.NoError(t, err) + require.NotNil(t, remote.Lifecycle) + require.NotNil(t, remote.Lifecycle.Started) + assert.True(t, *remote.Lifecycle.Started) + assert.True(t, remote.ApplyPolicyDefaultValues) + require.Len(t, remote.Libraries, 1) + assert.Equal(t, "/Workspace/a.whl", remote.Libraries[0].Whl) +}