Repository navigation
direct: retry app update and delete while app compute is UPDATING #7606
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| * Retry app updates and deletes in the direct deployment engine while the app compute is `UPDATING`, `STARTING` or `STOPPING`, instead of failing with 400 BAD_REQUEST. ([#7606](https://github.com/databricks/cli/pull/7606)) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,13 +4,16 @@ import ( | |
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "net/http" | ||
| "regexp" | ||
| "slices" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/databricks/cli/bundle/appdeploy" | ||
| "github.com/databricks/cli/bundle/config/resources" | ||
| "github.com/databricks/cli/bundle/deployplan" | ||
| "github.com/databricks/cli/libs/retry" | ||
| "github.com/databricks/cli/libs/structs/structpath" | ||
| "github.com/databricks/databricks-sdk-go" | ||
| "github.com/databricks/databricks-sdk-go/apierr" | ||
|
|
@@ -195,6 +198,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, | |
| } | ||
|
|
||
| func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error { | ||
| alreadyStarted := remoteIsStarted(entry) | ||
| // Deploy-only fields (source_code_path, config, git_source, lifecycle) are excluded | ||
| // from the request body; see appRequestBody. | ||
| if hasAppChanges(entry) { | ||
|
|
@@ -204,10 +208,19 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, | |
| AppName: id, | ||
| UpdateMask: appUpdateMask(&app), | ||
| } | ||
| updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) | ||
| var updateWaiter *apps.WaitGetUpdateAppSucceeded[apps.AppUpdate] | ||
| settled, err := r.retryWhileComputeSettles(ctx, id, func() error { | ||
| var err error | ||
| updateWaiter, err = r.client.Apps.CreateUpdate(ctx, request) | ||
| return err | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| // The plan-time compute state is stale if the compute moved while we waited. | ||
| if settled != nil { | ||
| alreadyStarted = !isComputeStopped(settled) | ||
| } | ||
|
|
||
| response, err := updateWaiter.Get() | ||
| if err != nil { | ||
|
|
@@ -219,7 +232,7 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, | |
| } | ||
| } | ||
|
|
||
| return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) | ||
| return r.manageLifecycle(ctx, id, config, alreadyStarted) | ||
| } | ||
|
|
||
| // missingServicePrincipalMessage is part of the backend error for an app whose service | ||
|
|
@@ -343,10 +356,87 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { | |
| } | ||
|
|
||
| func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) error { | ||
| _, err := r.client.Apps.DeleteByName(ctx, id) | ||
| _, err := r.retryWhileComputeSettles(ctx, id, func() error { | ||
| _, err := r.client.Apps.DeleteByName(ctx, id) | ||
| return err | ||
| }) | ||
| if errors.Is(err, errAppGoneWhileSettling) { | ||
| return nil | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| // computeTransitionalStateError matches the 400 BAD_REQUEST messages the Apps API returns for | ||
| // an update or delete while the compute is in a transitional state: "Cannot update app X as | ||
| // its compute is in UPDATING state" and "Cannot delete app X as it is not terminal with state | ||
| // UPDATING". DELETING is left out: a delete of a DELETING app is handled by IsGone and | ||
| // deleteConfirmedGone, and waiting would not help. | ||
| var computeTransitionalStateError = regexp.MustCompile(`(?:is in|with state) (?:UPDATING|STARTING|STOPPING)\b`) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: The STARTING/STOPPING message matches have no cited backend message and no test The comment at app.go:369-373 quotes only the two UPDATING messages ("compute is in UPDATING state" and "not terminal with state UPDATING"), which come from the dashboard errors. The regex is Agreed by 3 reviewer model(s). |
||
|
|
||
| func isComputeTransitionalStateError(err error) bool { | ||
| apiErr, ok := errors.AsType[*apierr.APIError](err) | ||
| return ok && apiErr.StatusCode == http.StatusBadRequest && computeTransitionalStateError.MatchString(apiErr.Message) | ||
| } | ||
|
|
||
| // errAppGoneWhileSettling reports that the app disappeared while waiting for its compute. | ||
| var errAppGoneWhileSettling = errors.New("app is gone") | ||
|
|
||
| // computeSettleTimeout bounds one wait for the app compute to leave a transitional state. | ||
| const computeSettleTimeout = 20 * time.Minute | ||
|
|
||
| // retryWhileComputeSettles calls fn and, if the Apps API rejected it because the compute is in | ||
| // a transitional state, waits for the compute to settle and calls fn again (see retry.With for | ||
| // the attempt limit). The compute can enter UPDATING on its own shortly after a create or | ||
| // update returned. It returns the app as last read while waiting, or nil if it did not wait, | ||
| // so a caller can refresh what it knew about the compute state. | ||
| func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) (*apps.App, error) { | ||
| var settled *apps.App | ||
| var waitErr error | ||
| _, err := retry.With(ctx, func(ctx context.Context, err error) bool { | ||
| if !isComputeTransitionalStateError(err) { | ||
| return false | ||
| } | ||
| settled, waitErr = r.waitForComputeToSettle(ctx, name) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. minor: The compute wait runs silently inside retry.With's check, and retry.With then adds another 15s sleep
Suggestions:
Acceptance tests suppress Agreed by 3 reviewer model(s). |
||
| return waitErr == nil | ||
| }, func() (struct{}, error) { | ||
| return struct{}{}, fn() | ||
| }) | ||
| if err != nil && waitErr != nil { | ||
| if apierr.IsMissing(waitErr) { | ||
| return nil, fmt.Errorf("%w: %w", errAppGoneWhileSettling, waitErr) | ||
| } | ||
| return nil, errors.Join(err, waitErr) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. minor: The paths where the wait fails or the retries run out are untested
A unit test for the first two would pin down what the apply layer receives. The first needs a GET handler that returns 500 on the poll. The second needs a DELETE handler that always returns the UPDATING message while GET returns ACTIVE. Agreed by 3 reviewer model(s). |
||
| } | ||
| return settled, err | ||
| } | ||
|
|
||
| func isComputeTransitional(app *apps.App) bool { | ||
| if app.ComputeStatus == nil { | ||
| return false | ||
| } | ||
| switch app.ComputeStatus.State { | ||
| case apps.ComputeStateUpdating, apps.ComputeStateStarting, apps.ComputeStateStopping: | ||
| return true | ||
| default: | ||
| return false | ||
| } | ||
| } | ||
|
|
||
| // waitForComputeToSettle polls the app until its compute leaves a transitional state. | ||
| func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (*apps.App, error) { | ||
| retrier := retries.New[apps.App](retries.WithTimeout(computeSettleTimeout), retries.WithRetryFunc(shouldRetry)) | ||
| return retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { | ||
| app, err := r.client.Apps.GetByName(ctx, name) | ||
| if err != nil { | ||
| return nil, retries.Halt(err) | ||
| } | ||
| if isComputeTransitional(app) { | ||
| return nil, retries.Continues(fmt.Sprintf("app compute is %s", app.ComputeStatus.State)) | ||
| } | ||
| return app, nil | ||
| }) | ||
| } | ||
|
|
||
| // IsGone treats a DELETING app as already-deleted. The Apps DELETE is | ||
| // fire-and-forget: it returns success while the app sits in | ||
| // ComputeState=DELETING for up to ~20 minutes, and a GET during that window | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,8 @@ import ( | |
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/databricks/cli/bundle/deployplan" | ||
| "github.com/databricks/cli/libs/retry" | ||
| "github.com/databricks/cli/libs/testserver" | ||
| "github.com/databricks/databricks-sdk-go" | ||
| "github.com/databricks/databricks-sdk-go/apierr" | ||
|
|
@@ -248,3 +250,134 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { | |
| assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`) | ||
| } | ||
| } | ||
|
|
||
| const appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + | ||
| "and was updated less than 20 minutes ago. Please wait before trying again." | ||
|
|
||
| // newAppComputeSettleServer serves GET /api/2.0/apps/{name} with the given compute states in | ||
| // order (the last one repeats) and counts the calls. | ||
| func newAppComputeSettleServer(t *testing.T, gets *int, states ...apps.ComputeState) *testserver.Server { | ||
| t.Setenv(retry.IntervalMsVariable, "1") | ||
| server := testserver.New(t) | ||
| server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { | ||
| if len(states) == 0 { | ||
| return testserver.Response{ | ||
| StatusCode: 404, | ||
| Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "App not found."}, | ||
| } | ||
| } | ||
| state := states[min(*gets, len(states)-1)] | ||
| *gets++ | ||
| return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} | ||
| }) | ||
| return server | ||
| } | ||
|
|
||
| func newAppForServer(t *testing.T, server *testserver.Server) *ResourceApp { | ||
| testserver.AddDefaultHandlers(server) | ||
| client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) | ||
| require.NoError(t, err) | ||
| return (&ResourceApp{}).New(client) | ||
| } | ||
|
|
||
| func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| message string | ||
| computeStates []apps.ComputeState | ||
| wantDeletes int | ||
| wantGets int | ||
| wantErr string | ||
| }{ | ||
| { | ||
| name: "retries after compute leaves UPDATING", | ||
| message: appComputeUpdatingDeleteMessage, | ||
| computeStates: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive}, | ||
| wantDeletes: 2, | ||
| wantGets: 2, | ||
| }, | ||
| { | ||
| name: "retries when compute settled before the first read", | ||
| message: appComputeUpdatingDeleteMessage, | ||
| computeStates: []apps.ComputeState{apps.ComputeStateActive}, | ||
| wantDeletes: 2, | ||
| wantGets: 1, | ||
| }, | ||
| { | ||
| name: "treats an app gone while waiting as deleted", | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: The "app gone while waiting" case implies zero GETs, but one GET is made In the test helper, Agreed by 3 reviewer model(s). |
||
| message: appComputeUpdatingDeleteMessage, | ||
| wantDeletes: 1, | ||
| }, | ||
| { | ||
| name: "does not retry a delete of a DELETING app", | ||
| message: "Cannot delete app test-app as it is not terminal with state DELETING, and was updated less than 20 minutes ago. Please wait before trying again.", | ||
| computeStates: []apps.ComputeState{apps.ComputeStateDeleting}, | ||
| wantDeletes: 1, | ||
| wantErr: "state DELETING", | ||
| }, | ||
| { | ||
| name: "does not retry other bad requests", | ||
| message: "Invalid app name.", | ||
| computeStates: []apps.ComputeState{apps.ComputeStateUpdating}, | ||
| wantDeletes: 1, | ||
| wantErr: "Invalid app name.", | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range tests { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| gets := 0 | ||
| server := newAppComputeSettleServer(t, &gets, tc.computeStates...) | ||
| deletes := 0 | ||
| server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { | ||
| deletes++ | ||
| if deletes == 1 { | ||
| return testserver.Response{ | ||
| StatusCode: 400, | ||
| Body: map[string]string{"error_code": "BAD_REQUEST", "message": tc.message}, | ||
| } | ||
| } | ||
| return apps.App{Name: req.Vars["name"]} | ||
| }) | ||
| r := newAppForServer(t, server) | ||
|
|
||
| err := r.DoDelete(t.Context(), "test-app", nil) | ||
| if tc.wantErr != "" { | ||
| require.ErrorContains(t, err, tc.wantErr) | ||
| } else { | ||
| require.NoError(t, err) | ||
| } | ||
| assert.Equal(t, tc.wantDeletes, deletes) | ||
| assert.Equal(t, tc.wantGets, gets) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { | ||
| gets := 0 | ||
| server := newAppComputeSettleServer(t, &gets, apps.ComputeStateUpdating, apps.ComputeStateActive) | ||
| updates := 0 | ||
| server.Handle("POST", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { | ||
| updates++ | ||
| if updates == 1 { | ||
| return testserver.Response{ | ||
| StatusCode: 400, | ||
| Body: map[string]string{ | ||
| "error_code": "BAD_REQUEST", | ||
| "message": "Cannot update app test-app as its compute is in UPDATING state. App compute needs to be ACTIVE or STOPPED to update.", | ||
| }, | ||
| } | ||
| } | ||
| return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} | ||
| }) | ||
| server.Handle("GET", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { | ||
| return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} | ||
| }) | ||
| r := newAppForServer(t, server) | ||
|
|
||
| entry := &PlanEntry{Changes: deployplan.Changes{"description": {Action: deployplan.Update}}} | ||
| _, err := r.DoUpdate(t.Context(), "test-app", &AppState{Name: "test-app", Description: "new"}, entry) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, 2, updates) | ||
| assert.Equal(t, 2, gets) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
minor: The started-state refresh after a settle wait has no test
After a settle wait,
doUpdatesetsalreadyStarted = !isComputeStopped(settled)(app.go:220-223) and passes it tomanageLifecycle(app.go:235). That value decides whetherApps.Start,Apps.Stopor a plain deploy happens, so a wrong value issues an unneeded stop or skips a needed start. No test reaches it.TestAppDoUpdate_RetriesWhileComputeSettles(app_test.go:379-380) passes anAppStatewithoutLifecycleand aPlanEntrywith no remote state, somanageLifecyclereturns at app.go:255-257 before it reads the flag. Deleting or inverting the three lines leaves every test green. A search ofapp_test.goforLifecycle,/start,/stop,STARTINGandSTOPPINGfinds nothing.Suggested cases for the update test:
started=true, the first update rejected with a STOPPING message, GET returns STOPPING then STOPPED, andlifecycle.started: true. Assert thatPOST /api/2.0/apps/{name}/startis called. This also exercises a non-UPDATING alternative of the message regex.started=true, the compute settles to STOPPED, and the config hasstarted: false. Assert that no/stopcall is made.started: true, the app settles to ACTIVE. Assert that no/startcall is made.There is also no unit test for an update whose app disappears (404) during the wait. That path returns the wrapped
app is gone: ...error (app.go:406) rather than nil, unlikeDoDelete, which checkserrAppGoneWhileSettling.Agreed by 3 reviewer model(s).