Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7647.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Fix `bundle plan` and `bundle deploy` failing to recreate a cluster deleted outside the bundle when its libraries status lookup responds first ([#7647](https://github.com/databricks/cli/pull/7647)).
34 changes: 20 additions & 14 deletions bundle/direct/dresources/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"strings"
"sync"
"time"

"github.com/databricks/cli/bundle/config/resources"
Expand All @@ -17,7 +18,6 @@ import (
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/retries"
"github.com/databricks/databricks-sdk-go/service/compute"
"golang.org/x/sync/errgroup"
)

// librariesWaitTimeout bounds how long we poll for libraries to finish installing.
Expand Down Expand Up @@ -151,21 +151,27 @@ func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState {
func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote, error) {
var details *compute.ClusterDetails
var libraries []compute.Library

// The cluster GET and the library-status GET are independent, so run them concurrently.
g, ctx := errgroup.WithContext(ctx)
g.Go(func() error {
var err error
details, err = r.client.Clusters.GetByClusterId(ctx, id)
return err
var detailsErr, librariesErr error

// The two GETs are independent, so they still run together. The result is not
// whichever error finishes first: library-status reports a deleted cluster as
// INVALID_PARAMETER_VALUE, which the planner does not treat as missing, and
// that error aborts the plan whenever it wins the race. A failed cluster read
// decides the outcome. A library error is returned only after the cluster
// itself was read, so a failure on a live cluster still surfaces.
var wg sync.WaitGroup
wg.Go(func() {
details, detailsErr = r.client.Clusters.GetByClusterId(ctx, id)
})
g.Go(func() error {
var err error
libraries, err = r.readLibraries(ctx, id)
return err
wg.Go(func() {
libraries, librariesErr = r.readLibraries(ctx, id)
})
if err := g.Wait(); err != nil {
return nil, err
wg.Wait()
if detailsErr != nil {
return nil, detailsErr
}
if librariesErr != nil {
return nil, librariesErr
}

remote := &ClusterRemote{
Expand Down
108 changes: 108 additions & 0 deletions bundle/direct/dresources/cluster_test.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
package dresources

import (
"context"
"testing"
"time"

"github.com/databricks/cli/bundle/deployplan"
"github.com/databricks/cli/libs/structs/structpath"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -66,3 +71,106 @@ func TestClusterOverrideChangeDescLibraries(t *testing.T) {
})
}
}

const clusterDoReadID = "0101-000000-deleted00"

// doReadOrdered runs DoRead with both lookups in flight. The lookup named by
// first returns immediately. The other waits until that call has started and
// then long enough for its result to be the one recorded first.
func doReadOrdered(t *testing.T, id string, details *compute.ClusterDetails, clusterErr error, statuses *compute.ClusterLibraryStatuses, libraryErr error, first string) (*ClusterRemote, error) {
t.Helper()
m := mocks.NewMockWorkspaceClient(t)
r := &ResourceCluster{client: m.WorkspaceClient}

winnerStarted := make(chan struct{})
release := func(name string) {
if name == first {
close(winnerStarted)
return
}
select {
case <-winnerStarted:
case <-t.Context().Done():
return
}
time.Sleep(20 * time.Millisecond)
}

m.GetMockClustersAPI().EXPECT().
GetByClusterId(mock.Anything, id).
RunAndReturn(func(context.Context, string) (*compute.ClusterDetails, error) {
release("cluster")
return details, clusterErr
}).Once()
m.GetMockLibrariesAPI().EXPECT().
ClusterStatusByClusterId(mock.Anything, id).
RunAndReturn(func(context.Context, string) (*compute.ClusterLibraryStatuses, error) {
release("libraries")
return statuses, libraryErr
}).Once()

return r.DoRead(t.Context(), id)
}

func TestClusterDoReadDeletedClusterIgnoresLibraryStatusOrder(t *testing.T) {
clusterMissing := &apierr.APIError{
StatusCode: 400,
ErrorCode: "RESOURCE_DOES_NOT_EXIST",
Message: "Cluster " + clusterDoReadID + " does not exist",
}
// library-status uses this code for the same missing cluster. The SDK remaps
// it to "not found" only on GET /clusters/get, so IsMissing is false here.
libraryMissing := &apierr.APIError{
StatusCode: 400,
ErrorCode: "INVALID_PARAMETER_VALUE",
Message: "Cluster " + clusterDoReadID + " does not exist",
}

for _, first := range []string{"libraries", "cluster"} {
t.Run(first+" finishes first", func(t *testing.T) {
_, err := doReadOrdered(t, clusterDoReadID, nil, clusterMissing, nil, libraryMissing, first)
require.ErrorIs(t, err, apierr.ErrResourceDoesNotExist)
require.NotErrorIs(t, err, apierr.ErrInvalidParameterValue)
})
}
}

func TestClusterDoReadLibraryErrorSurfacesWhenClusterExists(t *testing.T) {
details := &compute.ClusterDetails{ClusterId: clusterDoReadID, State: compute.StateRunning}
libraryErr := &apierr.APIError{
StatusCode: 500,
ErrorCode: "UNKNOWN",
Message: "library status unavailable",
}

for _, first := range []string{"libraries", "cluster"} {
t.Run(first+" finishes first", func(t *testing.T) {
_, err := doReadOrdered(t, clusterDoReadID, details, nil, nil, libraryErr, first)
require.ErrorIs(t, err, libraryErr)
require.NotErrorIs(t, err, apierr.ErrNotFound)
})
}
}

func TestClusterDoReadReturnsInstalledLibraries(t *testing.T) {
details := &compute.ClusterDetails{
ClusterId: clusterDoReadID,
State: compute.StateRunning,
Spec: &compute.ClusterSpec{ApplyPolicyDefaultValues: true},
}
statuses := &compute.ClusterLibraryStatuses{
LibraryStatuses: []compute.LibraryFullStatus{
{Library: &compute.Library{Whl: "/Workspace/a.whl"}, Status: compute.LibraryInstallStatusInstalled},
{Library: &compute.Library{Jar: "/Workspace/all.jar"}, IsLibraryForAllClusters: true, Status: compute.LibraryInstallStatusInstalled},
},
}

remote, err := doReadOrdered(t, clusterDoReadID, details, nil, statuses, nil, "libraries")
require.NoError(t, err)
require.NotNil(t, remote.Lifecycle)
require.NotNil(t, remote.Lifecycle.Started)
assert.True(t, *remote.Lifecycle.Started)
assert.True(t, remote.ApplyPolicyDefaultValues)
require.Len(t, remote.Libraries, 1)
assert.Equal(t, "/Workspace/a.whl", remote.Libraries[0].Whl)
}