From a26508b80201ff602422d066af64b9ec681b3bf5 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sun, 15 Nov 2026 20:41:19 -0400 Subject: [PATCH 1/2] fix(bundle): recreate deleted clusters when library status wins the race DoRead ran the cluster and library-status lookups together and returned the first error. library-status reports a deleted cluster as INVALID_PARAMETER_VALUE, which the planner does not treat as missing, so plan and deploy aborted whenever that lookup finished first. Return the cluster error whenever the cluster read fails, and return a library error only after the cluster itself was read. Fixes #7565 --- .../bundles/deleted-cluster-library-status.md | 1 + bundle/direct/dresources/cluster.go | 34 +++--- bundle/direct/dresources/cluster_test.go | 108 ++++++++++++++++++ 3 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 .nextchanges/bundles/deleted-cluster-library-status.md diff --git a/.nextchanges/bundles/deleted-cluster-library-status.md b/.nextchanges/bundles/deleted-cluster-library-status.md new file mode 100644 index 00000000000..9994761eed5 --- /dev/null +++ b/.nextchanges/bundles/deleted-cluster-library-status.md @@ -0,0 +1 @@ +* Fix `bundle plan` and `bundle deploy` failing to recreate a cluster deleted outside the bundle when its libraries status lookup responds first, as reported in [#7565](https://github.com/databricks/cli/issues/7565). diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 82167080c4d..2bd1b9f2ef9 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" "github.com/databricks/cli/bundle/config/resources" @@ -17,7 +18,6 @@ import ( "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/compute" - "golang.org/x/sync/errgroup" ) // librariesWaitTimeout bounds how long we poll for libraries to finish installing. @@ -151,21 +151,27 @@ func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState { func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote, error) { var details *compute.ClusterDetails var libraries []compute.Library - - // The cluster GET and the library-status GET are independent, so run them concurrently. - g, ctx := errgroup.WithContext(ctx) - g.Go(func() error { - var err error - details, err = r.client.Clusters.GetByClusterId(ctx, id) - return err + var detailsErr, librariesErr error + + // The two GETs are independent, so they still run together. The result is not + // whichever error finishes first: library-status reports a deleted cluster as + // INVALID_PARAMETER_VALUE, which the planner does not treat as missing, and + // that error aborts the plan whenever it wins the race. A failed cluster read + // decides the outcome. A library error is returned only after the cluster + // itself was read, so a failure on a live cluster still surfaces. + var wg sync.WaitGroup + wg.Go(func() { + details, detailsErr = r.client.Clusters.GetByClusterId(ctx, id) }) - g.Go(func() error { - var err error - libraries, err = r.readLibraries(ctx, id) - return err + wg.Go(func() { + libraries, librariesErr = r.readLibraries(ctx, id) }) - if err := g.Wait(); err != nil { - return nil, err + wg.Wait() + if detailsErr != nil { + return nil, detailsErr + } + if librariesErr != nil { + return nil, librariesErr } remote := &ClusterRemote{ diff --git a/bundle/direct/dresources/cluster_test.go b/bundle/direct/dresources/cluster_test.go index 7d3af4440d4..8157e2aadfc 100644 --- a/bundle/direct/dresources/cluster_test.go +++ b/bundle/direct/dresources/cluster_test.go @@ -1,12 +1,17 @@ package dresources import ( + "context" "testing" + "time" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -66,3 +71,106 @@ func TestClusterOverrideChangeDescLibraries(t *testing.T) { }) } } + +const clusterDoReadID = "0101-000000-deleted00" + +// doReadOrdered runs DoRead with both lookups in flight. The lookup named by +// first returns immediately. The other waits until that call has started and +// then long enough for its result to be the one recorded first. +func doReadOrdered(t *testing.T, id string, details *compute.ClusterDetails, clusterErr error, statuses *compute.ClusterLibraryStatuses, libraryErr error, first string) (*ClusterRemote, error) { + t.Helper() + m := mocks.NewMockWorkspaceClient(t) + r := &ResourceCluster{client: m.WorkspaceClient} + + winnerStarted := make(chan struct{}) + release := func(name string) { + if name == first { + close(winnerStarted) + return + } + select { + case <-winnerStarted: + case <-t.Context().Done(): + return + } + time.Sleep(20 * time.Millisecond) + } + + m.GetMockClustersAPI().EXPECT(). + GetByClusterId(mock.Anything, id). + RunAndReturn(func(context.Context, string) (*compute.ClusterDetails, error) { + release("cluster") + return details, clusterErr + }).Once() + m.GetMockLibrariesAPI().EXPECT(). + ClusterStatusByClusterId(mock.Anything, id). + RunAndReturn(func(context.Context, string) (*compute.ClusterLibraryStatuses, error) { + release("libraries") + return statuses, libraryErr + }).Once() + + return r.DoRead(t.Context(), id) +} + +func TestClusterDoReadDeletedClusterIgnoresLibraryStatusOrder(t *testing.T) { + clusterMissing := &apierr.APIError{ + StatusCode: 400, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + Message: "Cluster " + clusterDoReadID + " does not exist", + } + // library-status uses this code for the same missing cluster. The SDK remaps + // it to "not found" only on GET /clusters/get, so IsMissing is false here. + libraryMissing := &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "Cluster " + clusterDoReadID + " does not exist", + } + + for _, first := range []string{"libraries", "cluster"} { + t.Run(first+" finishes first", func(t *testing.T) { + _, err := doReadOrdered(t, clusterDoReadID, nil, clusterMissing, nil, libraryMissing, first) + require.ErrorIs(t, err, apierr.ErrResourceDoesNotExist) + require.NotErrorIs(t, err, apierr.ErrInvalidParameterValue) + }) + } +} + +func TestClusterDoReadLibraryErrorSurfacesWhenClusterExists(t *testing.T) { + details := &compute.ClusterDetails{ClusterId: clusterDoReadID, State: compute.StateRunning} + libraryErr := &apierr.APIError{ + StatusCode: 500, + ErrorCode: "UNKNOWN", + Message: "library status unavailable", + } + + for _, first := range []string{"libraries", "cluster"} { + t.Run(first+" finishes first", func(t *testing.T) { + _, err := doReadOrdered(t, clusterDoReadID, details, nil, nil, libraryErr, first) + require.ErrorIs(t, err, libraryErr) + require.NotErrorIs(t, err, apierr.ErrNotFound) + }) + } +} + +func TestClusterDoReadReturnsInstalledLibraries(t *testing.T) { + details := &compute.ClusterDetails{ + ClusterId: clusterDoReadID, + State: compute.StateRunning, + Spec: &compute.ClusterSpec{ApplyPolicyDefaultValues: true}, + } + statuses := &compute.ClusterLibraryStatuses{ + LibraryStatuses: []compute.LibraryFullStatus{ + {Library: &compute.Library{Whl: "/Workspace/a.whl"}, Status: compute.LibraryInstallStatusInstalled}, + {Library: &compute.Library{Jar: "/Workspace/all.jar"}, IsLibraryForAllClusters: true, Status: compute.LibraryInstallStatusInstalled}, + }, + } + + remote, err := doReadOrdered(t, clusterDoReadID, details, nil, statuses, nil, "libraries") + require.NoError(t, err) + require.NotNil(t, remote.Lifecycle) + require.NotNil(t, remote.Lifecycle.Started) + assert.True(t, *remote.Lifecycle.Started) + assert.True(t, remote.ApplyPolicyDefaultValues) + require.Len(t, remote.Libraries, 1) + assert.Equal(t, "/Workspace/a.whl", remote.Libraries[0].Whl) +} From ad66a91b2c872b0e730d4c593b640e60f806cbfe Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sun, 15 Nov 2026 21:14:27 -0400 Subject: [PATCH 2/2] Name changelog entry after PR #7647 --- .../bundles/{deleted-cluster-library-status.md => 7647.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .nextchanges/bundles/{deleted-cluster-library-status.md => 7647.md} (58%) diff --git a/.nextchanges/bundles/deleted-cluster-library-status.md b/.nextchanges/bundles/7647.md similarity index 58% rename from .nextchanges/bundles/deleted-cluster-library-status.md rename to .nextchanges/bundles/7647.md index 9994761eed5..d3b25cebd7a 100644 --- a/.nextchanges/bundles/deleted-cluster-library-status.md +++ b/.nextchanges/bundles/7647.md @@ -1 +1 @@ -* Fix `bundle plan` and `bundle deploy` failing to recreate a cluster deleted outside the bundle when its libraries status lookup responds first, as reported in [#7565](https://github.com/databricks/cli/issues/7565). +* Fix `bundle plan` and `bundle deploy` failing to recreate a cluster deleted outside the bundle when its libraries status lookup responds first ([#7647](https://github.com/databricks/cli/pull/7647)).