diff --git a/.nextchanges/bundles/app-compute-settle.md b/.nextchanges/bundles/app-compute-settle.md new file mode 100644 index 00000000000..57322262a6e --- /dev/null +++ b/.nextchanges/bundles/app-compute-settle.md @@ -0,0 +1 @@ +* Retry app updates and deletes in the direct deployment engine while the app compute is `UPDATING`, `STARTING` or `STOPPING`, instead of failing with 400 BAD_REQUEST. ([#7606](https://github.com/databricks/cli/pull/7606)) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index f76681feb98..86964a8ab2a 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "net/http" + "regexp" "slices" "strings" "time" @@ -11,6 +13,7 @@ import ( "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -195,6 +198,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error { + alreadyStarted := remoteIsStarted(entry) // Deploy-only fields (source_code_path, config, git_source, lifecycle) are excluded // from the request body; see appRequestBody. if hasAppChanges(entry) { @@ -204,10 +208,19 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, AppName: id, UpdateMask: appUpdateMask(&app), } - updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) + var updateWaiter *apps.WaitGetUpdateAppSucceeded[apps.AppUpdate] + settled, err := r.retryWhileComputeSettles(ctx, id, func() error { + var err error + updateWaiter, err = r.client.Apps.CreateUpdate(ctx, request) + return err + }) if err != nil { return err } + // The plan-time compute state is stale if the compute moved while we waited. + if settled != nil { + alreadyStarted = !isComputeStopped(settled) + } response, err := updateWaiter.Get() if err != nil { @@ -219,7 +232,7 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, } } - return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) + return r.manageLifecycle(ctx, id, config, alreadyStarted) } // missingServicePrincipalMessage is part of the backend error for an app whose service @@ -343,10 +356,87 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { } func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) error { - _, err := r.client.Apps.DeleteByName(ctx, id) + _, err := r.retryWhileComputeSettles(ctx, id, func() error { + _, err := r.client.Apps.DeleteByName(ctx, id) + return err + }) + if errors.Is(err, errAppGoneWhileSettling) { + return nil + } return err } +// computeTransitionalStateError matches the 400 BAD_REQUEST messages the Apps API returns for +// an update or delete while the compute is in a transitional state: "Cannot update app X as +// its compute is in UPDATING state" and "Cannot delete app X as it is not terminal with state +// UPDATING". DELETING is left out: a delete of a DELETING app is handled by IsGone and +// deleteConfirmedGone, and waiting would not help. +var computeTransitionalStateError = regexp.MustCompile(`(?:is in|with state) (?:UPDATING|STARTING|STOPPING)\b`) + +func isComputeTransitionalStateError(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode == http.StatusBadRequest && computeTransitionalStateError.MatchString(apiErr.Message) +} + +// errAppGoneWhileSettling reports that the app disappeared while waiting for its compute. +var errAppGoneWhileSettling = errors.New("app is gone") + +// computeSettleTimeout bounds one wait for the app compute to leave a transitional state. +const computeSettleTimeout = 20 * time.Minute + +// retryWhileComputeSettles calls fn and, if the Apps API rejected it because the compute is in +// a transitional state, waits for the compute to settle and calls fn again (see retry.With for +// the attempt limit). The compute can enter UPDATING on its own shortly after a create or +// update returned. It returns the app as last read while waiting, or nil if it did not wait, +// so a caller can refresh what it knew about the compute state. +func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) (*apps.App, error) { + var settled *apps.App + var waitErr error + _, err := retry.With(ctx, func(ctx context.Context, err error) bool { + if !isComputeTransitionalStateError(err) { + return false + } + settled, waitErr = r.waitForComputeToSettle(ctx, name) + return waitErr == nil + }, func() (struct{}, error) { + return struct{}{}, fn() + }) + if err != nil && waitErr != nil { + if apierr.IsMissing(waitErr) { + return nil, fmt.Errorf("%w: %w", errAppGoneWhileSettling, waitErr) + } + return nil, errors.Join(err, waitErr) + } + return settled, err +} + +func isComputeTransitional(app *apps.App) bool { + if app.ComputeStatus == nil { + return false + } + switch app.ComputeStatus.State { + case apps.ComputeStateUpdating, apps.ComputeStateStarting, apps.ComputeStateStopping: + return true + default: + return false + } +} + +// waitForComputeToSettle polls the app until its compute leaves a transitional state. +func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (*apps.App, error) { + retrier := retries.New[apps.App](retries.WithTimeout(computeSettleTimeout), retries.WithRetryFunc(shouldRetry)) + return retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { + app, err := r.client.Apps.GetByName(ctx, name) + if err != nil { + return nil, retries.Halt(err) + } + if isComputeTransitional(app) { + return nil, retries.Continues(fmt.Sprintf("app compute is %s", app.ComputeStatus.State)) + } + return app, nil + }) +} + // IsGone treats a DELETING app as already-deleted. The Apps DELETE is // fire-and-forget: it returns success while the app sits in // ComputeState=DELETING for up to ~20 minutes, and a GET during that window diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 01b619bae21..fec5874f471 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -248,3 +250,134 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`) } } + +const appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + + "and was updated less than 20 minutes ago. Please wait before trying again." + +// newAppComputeSettleServer serves GET /api/2.0/apps/{name} with the given compute states in +// order (the last one repeats) and counts the calls. +func newAppComputeSettleServer(t *testing.T, gets *int, states ...apps.ComputeState) *testserver.Server { + t.Setenv(retry.IntervalMsVariable, "1") + server := testserver.New(t) + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + if len(states) == 0 { + return testserver.Response{ + StatusCode: 404, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "App not found."}, + } + } + state := states[min(*gets, len(states)-1)] + *gets++ + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} + }) + return server +} + +func newAppForServer(t *testing.T, server *testserver.Server) *ResourceApp { + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + return (&ResourceApp{}).New(client) +} + +func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { + tests := []struct { + name string + message string + computeStates []apps.ComputeState + wantDeletes int + wantGets int + wantErr string + }{ + { + name: "retries after compute leaves UPDATING", + message: appComputeUpdatingDeleteMessage, + computeStates: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive}, + wantDeletes: 2, + wantGets: 2, + }, + { + name: "retries when compute settled before the first read", + message: appComputeUpdatingDeleteMessage, + computeStates: []apps.ComputeState{apps.ComputeStateActive}, + wantDeletes: 2, + wantGets: 1, + }, + { + name: "treats an app gone while waiting as deleted", + message: appComputeUpdatingDeleteMessage, + wantDeletes: 1, + }, + { + name: "does not retry a delete of a DELETING app", + message: "Cannot delete app test-app as it is not terminal with state DELETING, and was updated less than 20 minutes ago. Please wait before trying again.", + computeStates: []apps.ComputeState{apps.ComputeStateDeleting}, + wantDeletes: 1, + wantErr: "state DELETING", + }, + { + name: "does not retry other bad requests", + message: "Invalid app name.", + computeStates: []apps.ComputeState{apps.ComputeStateUpdating}, + wantDeletes: 1, + wantErr: "Invalid app name.", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gets := 0 + server := newAppComputeSettleServer(t, &gets, tc.computeStates...) + deletes := 0 + server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { + deletes++ + if deletes == 1 { + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "BAD_REQUEST", "message": tc.message}, + } + } + return apps.App{Name: req.Vars["name"]} + }) + r := newAppForServer(t, server) + + err := r.DoDelete(t.Context(), "test-app", nil) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantDeletes, deletes) + assert.Equal(t, tc.wantGets, gets) + }) + } +} + +func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { + gets := 0 + server := newAppComputeSettleServer(t, &gets, apps.ComputeStateUpdating, apps.ComputeStateActive) + updates := 0 + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + updates++ + if updates == 1 { + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "BAD_REQUEST", + "message": "Cannot update app test-app as its compute is in UPDATING state. App compute needs to be ACTIVE or STOPPED to update.", + }, + } + } + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + }) + server.Handle("GET", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + }) + r := newAppForServer(t, server) + + entry := &PlanEntry{Changes: deployplan.Changes{"description": {Action: deployplan.Update}}} + _, err := r.DoUpdate(t.Context(), "test-app", &AppState{Name: "test-app", Description: "new"}, entry) + require.NoError(t, err) + assert.Equal(t, 2, updates) + assert.Equal(t, 2, gets) +}