Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/fix-app-run-compute-state.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks bundle run` on an app now waits for the app compute to settle before it starts or deploys the app. Previously it could fail with "not in RUNNING state" when the compute was UPDATING or went from STARTING back to STOPPED. ([#7460](https://github.com/databricks/cli/pull/7460))
44 changes: 32 additions & 12 deletions bundle/run/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/run/output"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/retries"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -52,8 +54,9 @@ func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e
logProgress(ctx, "Getting the status of the app "+app.Name)
w := b.WorkspaceClient(ctx)

// Check the status of the app first.
createdApp, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name})
// Check the status of the app once its compute has settled. Acting on a transient state
// (for example UPDATING, or STARTING that goes back to STOPPED) would fail the deployment.
createdApp, err := waitForComputeToSettle(ctx, w, app.Name)
if err != nil {
return nil, err
}
Expand All @@ -77,14 +80,6 @@ func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e
}
}

// If the app is starting, we need to wait for it to be active before we can deploy it.
if isAppComputeStarting(createdApp) {
_, err := w.Apps.WaitGetAppActive(ctx, app.Name, 20*time.Minute, nil)
if err != nil {
return nil, err
}
}

// Deploy the app.
err = a.deploy(ctx)
if err != nil {
Expand All @@ -95,8 +90,33 @@ func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e
return nil, nil
}

func isAppComputeStarting(app *apps.App) bool {
return app.ComputeStatus != nil && app.ComputeStatus.State == apps.ComputeStateStarting
// waitForComputeToSettle polls the app until its compute is ACTIVE, STOPPED or ERROR, or the app has no
// compute status. DELETING fails immediately; every other state is polled through.
func waitForComputeToSettle(ctx context.Context, w *databricks.WorkspaceClient, name string) (*apps.App, error) {
return retries.Poll(ctx, 20*time.Minute, func() (*apps.App, *retries.Err) {
app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: name})
if err != nil {
return nil, retries.Halt(err)
}

// GET always returns compute_status (the SDK waiters dereference it unconditionally, and
// a no_compute app reports STOPPED, not nil). Kept as a guard: isAppComputeStopped treats
// nil as stopped and starts the app, so waiting here would only burn the timeout.
if app.ComputeStatus == nil {
return app, nil

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this supposed to be a terminal condition, or can it happen as an initial transient condition?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Terminal. The API always returns compute_status (the SDK waiters and dresources dereference it unconditionally, a no_compute app reports STOPPED), so nil is just a guard; added a comment saying why.

}

switch app.ComputeStatus.State {
case apps.ComputeStateActive, apps.ComputeStateStopped, apps.ComputeStateError:
return app, nil
case apps.ComputeStateDeleting:
// GET keeps returning the app (not 404) for up to ~20 minutes after DELETE, so polling
// through would only burn the timeout. Same treatment as IsGone in bundle/direct/dresources/app.go.
return nil, retries.Halt(fmt.Errorf("app %s is being deleted, redeploy the bundle to recreate it", name))
default:
Comment thread
andrewnester marked this conversation as resolved.
Resolved
return nil, retries.Continues("app compute is " + string(app.ComputeStatus.State))
}
})
}

func (a *appRunner) start(ctx context.Context) error {
Expand Down
96 changes: 96 additions & 0 deletions bundle/run/app_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,102 @@ func TestAppRunStoppedApp(t *testing.T) {
r.run(t)
}

func TestAppRunWaitsForComputeToSettle(t *testing.T) {
tests := []struct {
name string
states []apps.ComputeState
wantStart bool
wantErr string
}{
{
name: "updating then active",
states: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive},
},
{
name: "stopping then stopped",
states: []apps.ComputeState{apps.ComputeStateStopping, apps.ComputeStateStopped},
wantStart: true,
},
{
name: "starting then active",
states: []apps.ComputeState{apps.ComputeStateStarting, apps.ComputeStateActive},
},
{
name: "starting then stopped",
states: []apps.ComputeState{apps.ComputeStateStarting, apps.ComputeStateStopped},
wantStart: true,
},
{
name: "deleting",
states: []apps.ComputeState{apps.ComputeStateDeleting},
wantErr: "app my_app is being deleted, redeploy the bundle to recreate it",
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ctx, b, mwc := setupBundle(t)
appApi := mwc.GetMockAppsAPI()

for _, state := range tc.states {
appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{
Name: "my_app",
}).Return(&apps.App{
Name: "my_app",
ComputeStatus: &apps.ComputeStatus{
State: state,
},
}, nil).Once()
}

if tc.wantErr != "" {
r := appRunner{
key: "my_app",
bundle: b,
app: b.Config.Resources.Apps["my_app"],
}
_, err := r.Run(ctx, &Options{})
require.EqualError(t, err, tc.wantErr)
return
}

if tc.wantStart {
appApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{
Name: "my_app",
}).Return(&apps.WaitGetAppActive[apps.App]{
Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
return &apps.App{
Name: "my_app",
AppStatus: &apps.ApplicationStatus{
State: apps.ApplicationStateRunning,
},
ComputeStatus: &apps.ComputeStatus{
State: apps.ComputeStateActive,
},
}, nil
},
}, nil).Once()
}

wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{
Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) {
return nil, nil
},
}
appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{
AppName: "my_app",
AppDeployment: apps.AppDeployment{
Mode: apps.AppDeploymentModeSnapshot,
SourceCodePath: "/Workspace/Users/foo@bar.test/files/my_app",
},
}).Return(wait, nil).Once()

r := &testAppRunner{m: mwc, b: b, ctx: ctx}
r.run(t)
})
}
}

func TestAppRunWithAnActiveDeploymentInProgress(t *testing.T) {
r := setupTestApp(t, apps.ApplicationStateCrashed, apps.ComputeStateStopped)

Expand Down
Loading