diff --git a/.nextchanges/bundles/fix-app-run-compute-state.md b/.nextchanges/bundles/fix-app-run-compute-state.md new file mode 100644 index 00000000000..7bdf38fdd9d --- /dev/null +++ b/.nextchanges/bundles/fix-app-run-compute-state.md @@ -0,0 +1 @@ +* `databricks bundle run` on an app now waits for the app compute to settle before it starts or deploys the app. Previously it could fail with "not in RUNNING state" when the compute was UPDATING or went from STARTING back to STOPPED. ([#7460](https://github.com/databricks/cli/pull/7460)) diff --git a/bundle/run/app.go b/bundle/run/app.go index f7b347710de..d10861a14d2 100644 --- a/bundle/run/app.go +++ b/bundle/run/app.go @@ -11,6 +11,8 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/run/output" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" ) @@ -52,8 +54,9 @@ func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e logProgress(ctx, "Getting the status of the app "+app.Name) w := b.WorkspaceClient(ctx) - // Check the status of the app first. - createdApp, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name}) + // Check the status of the app once its compute has settled. Acting on a transient state + // (for example UPDATING, or STARTING that goes back to STOPPED) would fail the deployment. + createdApp, err := waitForComputeToSettle(ctx, w, app.Name) if err != nil { return nil, err } @@ -77,14 +80,6 @@ func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e } } - // If the app is starting, we need to wait for it to be active before we can deploy it. - if isAppComputeStarting(createdApp) { - _, err := w.Apps.WaitGetAppActive(ctx, app.Name, 20*time.Minute, nil) - if err != nil { - return nil, err - } - } - // Deploy the app. err = a.deploy(ctx) if err != nil { @@ -95,8 +90,33 @@ func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e return nil, nil } -func isAppComputeStarting(app *apps.App) bool { - return app.ComputeStatus != nil && app.ComputeStatus.State == apps.ComputeStateStarting +// waitForComputeToSettle polls the app until its compute is ACTIVE, STOPPED or ERROR, or the app has no +// compute status. DELETING fails immediately; every other state is polled through. +func waitForComputeToSettle(ctx context.Context, w *databricks.WorkspaceClient, name string) (*apps.App, error) { + return retries.Poll(ctx, 20*time.Minute, func() (*apps.App, *retries.Err) { + app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: name}) + if err != nil { + return nil, retries.Halt(err) + } + + // GET always returns compute_status (the SDK waiters dereference it unconditionally, and + // a no_compute app reports STOPPED, not nil). Kept as a guard: isAppComputeStopped treats + // nil as stopped and starts the app, so waiting here would only burn the timeout. + if app.ComputeStatus == nil { + return app, nil + } + + switch app.ComputeStatus.State { + case apps.ComputeStateActive, apps.ComputeStateStopped, apps.ComputeStateError: + return app, nil + case apps.ComputeStateDeleting: + // GET keeps returning the app (not 404) for up to ~20 minutes after DELETE, so polling + // through would only burn the timeout. Same treatment as IsGone in bundle/direct/dresources/app.go. + return nil, retries.Halt(fmt.Errorf("app %s is being deleted, redeploy the bundle to recreate it", name)) + default: + return nil, retries.Continues("app compute is " + string(app.ComputeStatus.State)) + } + }) } func (a *appRunner) start(ctx context.Context) error { diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go index 76946938ebb..8b79c56f954 100644 --- a/bundle/run/app_test.go +++ b/bundle/run/app_test.go @@ -146,6 +146,102 @@ func TestAppRunStoppedApp(t *testing.T) { r.run(t) } +func TestAppRunWaitsForComputeToSettle(t *testing.T) { + tests := []struct { + name string + states []apps.ComputeState + wantStart bool + wantErr string + }{ + { + name: "updating then active", + states: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive}, + }, + { + name: "stopping then stopped", + states: []apps.ComputeState{apps.ComputeStateStopping, apps.ComputeStateStopped}, + wantStart: true, + }, + { + name: "starting then active", + states: []apps.ComputeState{apps.ComputeStateStarting, apps.ComputeStateActive}, + }, + { + name: "starting then stopped", + states: []apps.ComputeState{apps.ComputeStateStarting, apps.ComputeStateStopped}, + wantStart: true, + }, + { + name: "deleting", + states: []apps.ComputeState{apps.ComputeStateDeleting}, + wantErr: "app my_app is being deleted, redeploy the bundle to recreate it", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx, b, mwc := setupBundle(t) + appApi := mwc.GetMockAppsAPI() + + for _, state := range tc.states { + appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{ + Name: "my_app", + }).Return(&apps.App{ + Name: "my_app", + ComputeStatus: &apps.ComputeStatus{ + State: state, + }, + }, nil).Once() + } + + if tc.wantErr != "" { + r := appRunner{ + key: "my_app", + bundle: b, + app: b.Config.Resources.Apps["my_app"], + } + _, err := r.Run(ctx, &Options{}) + require.EqualError(t, err, tc.wantErr) + return + } + + if tc.wantStart { + appApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{ + Name: "my_app", + }).Return(&apps.WaitGetAppActive[apps.App]{ + Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) { + return &apps.App{ + Name: "my_app", + AppStatus: &apps.ApplicationStatus{ + State: apps.ApplicationStateRunning, + }, + ComputeStatus: &apps.ComputeStatus{ + State: apps.ComputeStateActive, + }, + }, nil + }, + }, nil).Once() + } + + wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{ + Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) { + return nil, nil + }, + } + appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{ + AppName: "my_app", + AppDeployment: apps.AppDeployment{ + Mode: apps.AppDeploymentModeSnapshot, + SourceCodePath: "/Workspace/Users/foo@bar.test/files/my_app", + }, + }).Return(wait, nil).Once() + + r := &testAppRunner{m: mwc, b: b, ctx: ctx} + r.run(t) + }) + } +} + func TestAppRunWithAnActiveDeploymentInProgress(t *testing.T) { r := setupTestApp(t, apps.ApplicationStateCrashed, apps.ComputeStateStopped)