Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/app-compute-settle.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Retry app updates and deletes in the direct deployment engine while the app compute is `UPDATING`, `STARTING` or `STOPPING`, instead of failing with 400 BAD_REQUEST. ([#7606](https://github.com/databricks/cli/pull/7606))
96 changes: 93 additions & 3 deletions bundle/direct/dresources/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@ import (
"context"
"errors"
"fmt"
"net/http"
"regexp"
"slices"
"strings"
"time"

"github.com/databricks/cli/bundle/appdeploy"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/deployplan"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/cli/libs/structs/structpath"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
Expand Down Expand Up @@ -195,6 +198,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState,
}

func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error {
alreadyStarted := remoteIsStarted(entry)
// Deploy-only fields (source_code_path, config, git_source, lifecycle) are excluded
// from the request body; see appRequestBody.
if hasAppChanges(entry) {
Expand All @@ -204,10 +208,19 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState,
AppName: id,
UpdateMask: appUpdateMask(&app),
}
updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request)
var updateWaiter *apps.WaitGetUpdateAppSucceeded[apps.AppUpdate]
settled, err := r.retryWhileComputeSettles(ctx, id, func() error {
var err error
updateWaiter, err = r.client.Apps.CreateUpdate(ctx, request)
return err
})
if err != nil {
return err
}
// The plan-time compute state is stale if the compute moved while we waited.
if settled != nil {
alreadyStarted = !isComputeStopped(settled)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: The started-state refresh after a settle wait has no test

After a settle wait, doUpdate sets alreadyStarted = !isComputeStopped(settled) (app.go:220-223) and passes it to manageLifecycle (app.go:235). That value decides whether Apps.Start, Apps.Stop or a plain deploy happens, so a wrong value issues an unneeded stop or skips a needed start. No test reaches it. TestAppDoUpdate_RetriesWhileComputeSettles (app_test.go:379-380) passes an AppState without Lifecycle and a PlanEntry with no remote state, so manageLifecycle returns at app.go:255-257 before it reads the flag. Deleting or inverting the three lines leaves every test green. A search of app_test.go for Lifecycle, /start, /stop, STARTING and STOPPING finds nothing.

Suggested cases for the update test:

  • Plan-time remote started=true, the first update rejected with a STOPPING message, GET returns STOPPING then STOPPED, and lifecycle.started: true. Assert that POST /api/2.0/apps/{name}/start is called. This also exercises a non-UPDATING alternative of the message regex.
  • Plan-time started=true, the compute settles to STOPPED, and the config has started: false. Assert that no /stop call is made.
  • The mirror case: plan-time STOPPED with started: true, the app settles to ACTIVE. Assert that no /start call is made.

There is also no unit test for an update whose app disappears (404) during the wait. That path returns the wrapped app is gone: ... error (app.go:406) rather than nil, unlike DoDelete, which checks errAppGoneWhileSettling.

Agreed by 3 reviewer model(s).

}

response, err := updateWaiter.Get()
if err != nil {
Expand All @@ -219,7 +232,7 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState,
}
}

return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry))
return r.manageLifecycle(ctx, id, config, alreadyStarted)
}

// missingServicePrincipalMessage is part of the backend error for an app whose service
Expand Down Expand Up @@ -343,10 +356,87 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig {
}

func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) error {
_, err := r.client.Apps.DeleteByName(ctx, id)
_, err := r.retryWhileComputeSettles(ctx, id, func() error {
_, err := r.client.Apps.DeleteByName(ctx, id)
return err
})
if errors.Is(err, errAppGoneWhileSettling) {
return nil
}
return err
}

// computeTransitionalStateError matches the 400 BAD_REQUEST messages the Apps API returns for
// an update or delete while the compute is in a transitional state: "Cannot update app X as
// its compute is in UPDATING state" and "Cannot delete app X as it is not terminal with state
// UPDATING". DELETING is left out: a delete of a DELETING app is handled by IsGone and
// deleteConfirmedGone, and waiting would not help.
var computeTransitionalStateError = regexp.MustCompile(`(?:is in|with state) (?:UPDATING|STARTING|STOPPING)\b`)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: The STARTING/STOPPING message matches have no cited backend message and no test

The comment at app.go:369-373 quotes only the two UPDATING messages ("compute is in UPDATING state" and "not terminal with state UPDATING"), which come from the dashboard errors. The regex is (?:is in|with state) (?:UPDATING|STARTING|STOPPING)\b (app.go:374). All retry cases in app_test.go (lines 254-255 and 283-354) use the UPDATING message, or a DELETING / "Invalid app name." negative case. The STARTING|STOPPING alternatives are therefore unverified against the real backend wording, yet the PR description and the changelog fragment say they are retried. Waiting on those states in isComputeTransitional is reasonable, but matching an error message needs a real message to match. Either quote an observed STARTING/STOPPING 400 in the comment and add a table case for it, or narrow the regex to what has been observed. AGENTS.md asks to avoid speculative flexibility.

Agreed by 3 reviewer model(s).


func isComputeTransitionalStateError(err error) bool {
apiErr, ok := errors.AsType[*apierr.APIError](err)
return ok && apiErr.StatusCode == http.StatusBadRequest && computeTransitionalStateError.MatchString(apiErr.Message)
}

// errAppGoneWhileSettling reports that the app disappeared while waiting for its compute.
var errAppGoneWhileSettling = errors.New("app is gone")

// computeSettleTimeout bounds one wait for the app compute to leave a transitional state.
const computeSettleTimeout = 20 * time.Minute

// retryWhileComputeSettles calls fn and, if the Apps API rejected it because the compute is in
// a transitional state, waits for the compute to settle and calls fn again (see retry.With for
// the attempt limit). The compute can enter UPDATING on its own shortly after a create or
// update returned. It returns the app as last read while waiting, or nil if it did not wait,
// so a caller can refresh what it knew about the compute state.
func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) (*apps.App, error) {
var settled *apps.App
var waitErr error
_, err := retry.With(ctx, func(ctx context.Context, err error) bool {
if !isComputeTransitionalStateError(err) {
return false
}
settled, waitErr = r.waitForComputeToSettle(ctx, name)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: The compute wait runs silently inside retry.With's check, and retry.With then adds another 15s sleep

waitForComputeToSettle runs inside the check callback of retry.With (app.go:395-400), and nothing in app.go:392-438 logs before or during the wait. retry.With evaluates check first, then calls log.Warnf(ctx, "%s", retryMessage(err)), then sleeps interval(ctx) (libs/retry/retry.go:101-114; default 15s at retry.go:26). A bundle deploy or destroy that hits this path can therefore show no output for up to computeSettleTimeout (20 minutes) per wait, with up to two waits per call, which looks like a hang. The warning that follows is retrying after 400 Bad Request from DELETE /api/2.0/apps/<name> (retry.go:128). It names only the status code and endpoint, not the compute state, and appears after the wait has finished. The fixed 15s sleep is then taken even though the compute has already settled.

Suggestions:

  • Log one line before the wait, for example app %s compute is %s; waiting up to 20m before retrying, so users can tell why the command has paused.
  • Optionally, do the wait outside check so the message comes first and the redundant sleep goes away. Keeping retry.With and its pause is also acceptable.

Acceptance tests suppress Warn: retrying after lines (acceptance/test.toml:32), so a separate message should not change golden output.

Agreed by 3 reviewer model(s).

return waitErr == nil
}, func() (struct{}, error) {
return struct{}{}, fn()
})
if err != nil && waitErr != nil {
if apierr.IsMissing(waitErr) {
return nil, fmt.Errorf("%w: %w", errAppGoneWhileSettling, waitErr)
}
return nil, errors.Join(err, waitErr)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: The paths where the wait fails or the retries run out are untested

retryWhileComputeSettles has several outcomes after a wait fails or the API keeps rejecting the call, and only the 404 case is tested (the delete case "treats an app gone while waiting as deleted"). Not covered:

  • The wait fails with something other than 404, such as a GET error or the 20-minute timeout. The function returns errors.Join(err, waitErr) (app.go:404-409). Callers in bundle/direct/apply.go classify the returned error with apierr.IsMissing, errors.Is(err, apierr.ErrPermissionDenied) and retry.IsTransient (apply.go:134-142 Recreate, 174-179 Update, 302-318 Delete). Which of the two joined errors they see matters. retry.IsTransient (libs/retry/retry.go:45-58) checks isInactivityTimeout before the first *apierr.APIError in the chain, so a joined SDK inactivity timeout from the GET can make the outer retry.With in Update/Recreate run the whole DoUpdate/DoDelete again, even though the 400 would not be retried.
  • The API keeps returning the transitional 400 on all attempts. The last 400 is returned as is. The existing delete cases fail the first DELETE and succeed afterwards (app_test.go:334-342), so exhaustion is never reached.
  • An update, as opposed to a delete, whose app disappears during the wait. doUpdate returns the app is gone: ... error, which wraps a 404.

A unit test for the first two would pin down what the apply layer receives. The first needs a GET handler that returns 500 on the poll. The second needs a DELETE handler that always returns the UPDATING message while GET returns ACTIVE.

Agreed by 3 reviewer model(s).

}
return settled, err
}

func isComputeTransitional(app *apps.App) bool {
if app.ComputeStatus == nil {
return false
}
switch app.ComputeStatus.State {
case apps.ComputeStateUpdating, apps.ComputeStateStarting, apps.ComputeStateStopping:
return true
default:
return false
}
}

// waitForComputeToSettle polls the app until its compute leaves a transitional state.
func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (*apps.App, error) {
retrier := retries.New[apps.App](retries.WithTimeout(computeSettleTimeout), retries.WithRetryFunc(shouldRetry))
return retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) {
app, err := r.client.Apps.GetByName(ctx, name)
if err != nil {
return nil, retries.Halt(err)
}
if isComputeTransitional(app) {
return nil, retries.Continues(fmt.Sprintf("app compute is %s", app.ComputeStatus.State))
}
return app, nil
})
}

// IsGone treats a DELETING app as already-deleted. The Apps DELETE is
// fire-and-forget: it returns success while the app sits in
// ComputeState=DELETING for up to ~20 minutes, and a GET during that window
Expand Down
133 changes: 133 additions & 0 deletions bundle/direct/dresources/app_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"strings"
"testing"

"github.com/databricks/cli/bundle/deployplan"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/cli/libs/testserver"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
Expand Down Expand Up @@ -248,3 +250,134 @@ func TestWithMissingServicePrincipalHint(t *testing.T) {
assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`)
}
}

const appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " +
"and was updated less than 20 minutes ago. Please wait before trying again."

// newAppComputeSettleServer serves GET /api/2.0/apps/{name} with the given compute states in
// order (the last one repeats) and counts the calls.
func newAppComputeSettleServer(t *testing.T, gets *int, states ...apps.ComputeState) *testserver.Server {
t.Setenv(retry.IntervalMsVariable, "1")
server := testserver.New(t)
server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any {
if len(states) == 0 {
return testserver.Response{
StatusCode: 404,
Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "App not found."},
}
}
state := states[min(*gets, len(states)-1)]
*gets++
return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}}
})
return server
}

func newAppForServer(t *testing.T, server *testserver.Server) *ResourceApp {
testserver.AddDefaultHandlers(server)
client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"})
require.NoError(t, err)
return (&ResourceApp{}).New(client)
}

func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) {
tests := []struct {
name string
message string
computeStates []apps.ComputeState
wantDeletes int
wantGets int
wantErr string
}{
{
name: "retries after compute leaves UPDATING",
message: appComputeUpdatingDeleteMessage,
computeStates: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive},
wantDeletes: 2,
wantGets: 2,
},
{
name: "retries when compute settled before the first read",
message: appComputeUpdatingDeleteMessage,
computeStates: []apps.ComputeState{apps.ComputeStateActive},
wantDeletes: 2,
wantGets: 1,
},
{
name: "treats an app gone while waiting as deleted",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: The "app gone while waiting" case implies zero GETs, but one GET is made

In the test helper, if len(states) == 0 returns a 404 at app_test.go:263-268 before *gets++ at :270. The "app gone while waiting" case (app_test.go:306-310) sets wantDeletes: 1 and no wantGets, so the expected count defaults to 0 even though waitForComputeToSettle makes one GET. The test still passes for the right reason, because without the GET DoDelete would return the 400. But the count reads as "no GET happened". Please count the 404 response too and set wantGets: 1.

Agreed by 3 reviewer model(s).

message: appComputeUpdatingDeleteMessage,
wantDeletes: 1,
},
{
name: "does not retry a delete of a DELETING app",
message: "Cannot delete app test-app as it is not terminal with state DELETING, and was updated less than 20 minutes ago. Please wait before trying again.",
computeStates: []apps.ComputeState{apps.ComputeStateDeleting},
wantDeletes: 1,
wantErr: "state DELETING",
},
{
name: "does not retry other bad requests",
message: "Invalid app name.",
computeStates: []apps.ComputeState{apps.ComputeStateUpdating},
wantDeletes: 1,
wantErr: "Invalid app name.",
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
gets := 0
server := newAppComputeSettleServer(t, &gets, tc.computeStates...)
deletes := 0
server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any {
deletes++
if deletes == 1 {
return testserver.Response{
StatusCode: 400,
Body: map[string]string{"error_code": "BAD_REQUEST", "message": tc.message},
}
}
return apps.App{Name: req.Vars["name"]}
})
r := newAppForServer(t, server)

err := r.DoDelete(t.Context(), "test-app", nil)
if tc.wantErr != "" {
require.ErrorContains(t, err, tc.wantErr)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.wantDeletes, deletes)
assert.Equal(t, tc.wantGets, gets)
})
}
}

func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) {
gets := 0
server := newAppComputeSettleServer(t, &gets, apps.ComputeStateUpdating, apps.ComputeStateActive)
updates := 0
server.Handle("POST", "/api/2.0/apps/{name}/update", func(req testserver.Request) any {
updates++
if updates == 1 {
return testserver.Response{
StatusCode: 400,
Body: map[string]string{
"error_code": "BAD_REQUEST",
"message": "Cannot update app test-app as its compute is in UPDATING state. App compute needs to be ACTIVE or STOPPED to update.",
},
}
}
return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}}
})
server.Handle("GET", "/api/2.0/apps/{name}/update", func(req testserver.Request) any {
return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}}
})
r := newAppForServer(t, server)

entry := &PlanEntry{Changes: deployplan.Changes{"description": {Action: deployplan.Update}}}
_, err := r.DoUpdate(t.Context(), "test-app", &AppState{Name: "test-app", Description: "new"}, entry)
require.NoError(t, err)
assert.Equal(t, 2, updates)
assert.Equal(t, 2, gets)
}
Loading