From 67a7ab680e110993302b6487c4a5cad813898ff9 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:12:57 +0000 Subject: [PATCH 1/3] direct: retry app update and delete while app compute is UPDATING The Apps API rejects update and delete with 400 BAD_REQUEST while the app compute is in a transitional state, and the compute can enter UPDATING on its own shortly after a create or update returned. On such a 400, wait for the compute to settle and retry. Co-authored-by: Isaac --- bundle/direct/dresources/app.go | 70 ++++++++++++++++++++- bundle/direct/dresources/app_test.go | 91 ++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index f76681feb98..f9cd6634405 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "slices" "strings" "time" @@ -204,7 +205,12 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, AppName: id, UpdateMask: appUpdateMask(&app), } - updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) + var updateWaiter *apps.WaitGetUpdateAppSucceeded[apps.AppUpdate] + err := r.retryWhileComputeSettles(ctx, id, func() error { + var err error + updateWaiter, err = r.client.Apps.CreateUpdate(ctx, request) + return err + }) if err != nil { return err } @@ -343,8 +349,66 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { } func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) error { - _, err := r.client.Apps.DeleteByName(ctx, id) - return err + return r.retryWhileComputeSettles(ctx, id, func() error { + _, err := r.client.Apps.DeleteByName(ctx, id) + return err + }) +} + +// maxComputeSettleRetries bounds how often retryWhileComputeSettles repeats a call, so a +// compute that keeps going back into a transitional state cannot loop forever. +const maxComputeSettleRetries = 3 + +// retryWhileComputeSettles calls fn and, if it fails with 400 BAD_REQUEST while the app +// compute is in a transitional state, waits for the compute to settle and calls fn again. +// The Apps API rejects update and delete while the compute is UPDATING ("App compute needs +// to be ACTIVE or STOPPED to update", "not terminal with state UPDATING"), and the compute +// can enter UPDATING on its own shortly after a create or update returned. Checking the +// state instead of the error text keeps other 400s failing at once. +func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) error { + for attempt := 0; ; attempt++ { + err := fn() + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok || apiErr.StatusCode != http.StatusBadRequest || attempt >= maxComputeSettleRetries { + return err + } + waited, waitErr := r.waitForComputeToSettle(ctx, name) + if waitErr != nil || !waited { + return err + } + } +} + +func isComputeTransitional(app *apps.App) bool { + if app.ComputeStatus == nil { + return false + } + switch app.ComputeStatus.State { + case apps.ComputeStateUpdating, apps.ComputeStateStarting, apps.ComputeStateStopping: + return true + default: + return false + } +} + +// waitForComputeToSettle polls the app until its compute leaves a transitional state. It +// reports whether it saw a transitional state at all, so a caller can tell a failure caused +// by the compute state from any other one. +func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (bool, error) { + waited := false + retrier := retries.New[apps.App](retries.WithTimeout(-1), retries.WithRetryFunc(shouldRetry)) + _, err := retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { + app, err := r.client.Apps.GetByName(ctx, name) + if err != nil { + return nil, retries.Halt(err) + } + if isComputeTransitional(app) { + waited = true + return nil, retries.Continues(string(app.ComputeStatus.State)) + } + return app, nil + }) + return waited, err } // IsGone treats a DELETING app as already-deleted. The Apps DELETE is diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 01b619bae21..4cc291c47f9 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -248,3 +248,94 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`) } } + +func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { + tests := []struct { + name string + // computeStates are the states GET returns in order; the last one repeats. + computeStates []apps.ComputeState + wantDeletes int + wantErr bool + }{ + { + name: "retries after compute leaves UPDATING", + computeStates: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive}, + wantDeletes: 2, + }, + { + name: "fails at once when compute is not transitional", + computeStates: []apps.ComputeState{apps.ComputeStateActive}, + wantDeletes: 1, + wantErr: true, + }, + { + name: "fails at once when app is already deleting", + computeStates: []apps.ComputeState{apps.ComputeStateDeleting}, + wantDeletes: 1, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := testserver.New(t) + deletes := 0 + gets := 0 + + server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { + deletes++ + if deletes == 1 { + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "BAD_REQUEST", + "message": "Cannot delete app test-app as it is not terminal with state UPDATING, and was updated less than 20 minutes ago. Please wait before trying again.", + }, + } + } + return apps.App{Name: req.Vars["name"]} + }) + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + state := tc.computeStates[min(gets, len(tc.computeStates)-1)] + gets++ + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + + r := (&ResourceApp{}).New(client) + err = r.DoDelete(t.Context(), "test-app", nil) + if tc.wantErr { + require.ErrorContains(t, err, "not terminal with state UPDATING") + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantDeletes, deletes) + }) + } +} + +func TestAppRetryWhileComputeSettles_IgnoresNonBadRequest(t *testing.T) { + server := testserver.New(t) + gets := 0 + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + gets++ + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: apps.ComputeStateUpdating}} + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + + r := (&ResourceApp{}).New(client) + calls := 0 + err = r.retryWhileComputeSettles(t.Context(), "test-app", func() error { + calls++ + return &apierr.APIError{StatusCode: 500, ErrorCode: "INTERNAL_ERROR", Message: "boom"} + }) + require.ErrorContains(t, err, "boom") + assert.Equal(t, 1, calls) + assert.Equal(t, 0, gets) +} From f81ba10f2553e51f36c4c6092195631e9fa08709 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:14:15 +0000 Subject: [PATCH 2/3] Add changelog fragment cloud: TestAccept/bundle/apps/job_permissions TestAccept/bundle/invariant/no_drift/DMS=true/INPUT_CONFIG=app.yml.tmpl Co-authored-by: Isaac --- .nextchanges/bundles/app-compute-settle.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/app-compute-settle.md diff --git a/.nextchanges/bundles/app-compute-settle.md b/.nextchanges/bundles/app-compute-settle.md new file mode 100644 index 00000000000..57322262a6e --- /dev/null +++ b/.nextchanges/bundles/app-compute-settle.md @@ -0,0 +1 @@ +* Retry app updates and deletes in the direct deployment engine while the app compute is `UPDATING`, `STARTING` or `STOPPING`, instead of failing with 400 BAD_REQUEST. ([#7606](https://github.com/databricks/cli/pull/7606)) From 489b214dfa3124fc751bd1280b2fa3b246c536bd Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:23:28 +0000 Subject: [PATCH 3/3] Address review: match the compute-state error, bound the wait, refresh started state Retry on the specific 400 messages even if the compute settled before the first read, use retry.With for the attempt limit and pause, cap each wait at 20 minutes, treat an app gone while waiting as deleted, and refresh the started state used by lifecycle after waiting. Add a DoUpdate test. cloud: TestAccept/bundle/apps/job_permissions TestAccept/bundle/invariant/no_drift/DMS=true/INPUT_CONFIG=app.yml.tmpl Co-authored-by: Isaac --- bundle/direct/dresources/app.go | 94 ++++++++++++------- bundle/direct/dresources/app_test.go | 132 ++++++++++++++++++--------- 2 files changed, 147 insertions(+), 79 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index f9cd6634405..86964a8ab2a 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "regexp" "slices" "strings" "time" @@ -12,6 +13,7 @@ import ( "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -196,6 +198,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error { + alreadyStarted := remoteIsStarted(entry) // Deploy-only fields (source_code_path, config, git_source, lifecycle) are excluded // from the request body; see appRequestBody. if hasAppChanges(entry) { @@ -206,7 +209,7 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, UpdateMask: appUpdateMask(&app), } var updateWaiter *apps.WaitGetUpdateAppSucceeded[apps.AppUpdate] - err := r.retryWhileComputeSettles(ctx, id, func() error { + settled, err := r.retryWhileComputeSettles(ctx, id, func() error { var err error updateWaiter, err = r.client.Apps.CreateUpdate(ctx, request) return err @@ -214,6 +217,10 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, if err != nil { return err } + // The plan-time compute state is stale if the compute moved while we waited. + if settled != nil { + alreadyStarted = !isComputeStopped(settled) + } response, err := updateWaiter.Get() if err != nil { @@ -225,7 +232,7 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, } } - return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) + return r.manageLifecycle(ctx, id, config, alreadyStarted) } // missingServicePrincipalMessage is part of the backend error for an app whose service @@ -349,34 +356,58 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { } func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) error { - return r.retryWhileComputeSettles(ctx, id, func() error { + _, err := r.retryWhileComputeSettles(ctx, id, func() error { _, err := r.client.Apps.DeleteByName(ctx, id) return err }) -} - -// maxComputeSettleRetries bounds how often retryWhileComputeSettles repeats a call, so a -// compute that keeps going back into a transitional state cannot loop forever. -const maxComputeSettleRetries = 3 - -// retryWhileComputeSettles calls fn and, if it fails with 400 BAD_REQUEST while the app -// compute is in a transitional state, waits for the compute to settle and calls fn again. -// The Apps API rejects update and delete while the compute is UPDATING ("App compute needs -// to be ACTIVE or STOPPED to update", "not terminal with state UPDATING"), and the compute -// can enter UPDATING on its own shortly after a create or update returned. Checking the -// state instead of the error text keeps other 400s failing at once. -func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) error { - for attempt := 0; ; attempt++ { - err := fn() - apiErr, ok := errors.AsType[*apierr.APIError](err) - if !ok || apiErr.StatusCode != http.StatusBadRequest || attempt >= maxComputeSettleRetries { - return err + if errors.Is(err, errAppGoneWhileSettling) { + return nil + } + return err +} + +// computeTransitionalStateError matches the 400 BAD_REQUEST messages the Apps API returns for +// an update or delete while the compute is in a transitional state: "Cannot update app X as +// its compute is in UPDATING state" and "Cannot delete app X as it is not terminal with state +// UPDATING". DELETING is left out: a delete of a DELETING app is handled by IsGone and +// deleteConfirmedGone, and waiting would not help. +var computeTransitionalStateError = regexp.MustCompile(`(?:is in|with state) (?:UPDATING|STARTING|STOPPING)\b`) + +func isComputeTransitionalStateError(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode == http.StatusBadRequest && computeTransitionalStateError.MatchString(apiErr.Message) +} + +// errAppGoneWhileSettling reports that the app disappeared while waiting for its compute. +var errAppGoneWhileSettling = errors.New("app is gone") + +// computeSettleTimeout bounds one wait for the app compute to leave a transitional state. +const computeSettleTimeout = 20 * time.Minute + +// retryWhileComputeSettles calls fn and, if the Apps API rejected it because the compute is in +// a transitional state, waits for the compute to settle and calls fn again (see retry.With for +// the attempt limit). The compute can enter UPDATING on its own shortly after a create or +// update returned. It returns the app as last read while waiting, or nil if it did not wait, +// so a caller can refresh what it knew about the compute state. +func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) (*apps.App, error) { + var settled *apps.App + var waitErr error + _, err := retry.With(ctx, func(ctx context.Context, err error) bool { + if !isComputeTransitionalStateError(err) { + return false } - waited, waitErr := r.waitForComputeToSettle(ctx, name) - if waitErr != nil || !waited { - return err + settled, waitErr = r.waitForComputeToSettle(ctx, name) + return waitErr == nil + }, func() (struct{}, error) { + return struct{}{}, fn() + }) + if err != nil && waitErr != nil { + if apierr.IsMissing(waitErr) { + return nil, fmt.Errorf("%w: %w", errAppGoneWhileSettling, waitErr) } + return nil, errors.Join(err, waitErr) } + return settled, err } func isComputeTransitional(app *apps.App) bool { @@ -391,24 +422,19 @@ func isComputeTransitional(app *apps.App) bool { } } -// waitForComputeToSettle polls the app until its compute leaves a transitional state. It -// reports whether it saw a transitional state at all, so a caller can tell a failure caused -// by the compute state from any other one. -func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (bool, error) { - waited := false - retrier := retries.New[apps.App](retries.WithTimeout(-1), retries.WithRetryFunc(shouldRetry)) - _, err := retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { +// waitForComputeToSettle polls the app until its compute leaves a transitional state. +func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (*apps.App, error) { + retrier := retries.New[apps.App](retries.WithTimeout(computeSettleTimeout), retries.WithRetryFunc(shouldRetry)) + return retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { app, err := r.client.Apps.GetByName(ctx, name) if err != nil { return nil, retries.Halt(err) } if isComputeTransitional(app) { - waited = true - return nil, retries.Continues(string(app.ComputeStatus.State)) + return nil, retries.Continues(fmt.Sprintf("app compute is %s", app.ComputeStatus.State)) } return app, nil }) - return waited, err } // IsGone treats a DELETING app as already-deleted. The Apps DELETE is diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 4cc291c47f9..fec5874f471 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -249,93 +251,133 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { } } +const appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + + "and was updated less than 20 minutes ago. Please wait before trying again." + +// newAppComputeSettleServer serves GET /api/2.0/apps/{name} with the given compute states in +// order (the last one repeats) and counts the calls. +func newAppComputeSettleServer(t *testing.T, gets *int, states ...apps.ComputeState) *testserver.Server { + t.Setenv(retry.IntervalMsVariable, "1") + server := testserver.New(t) + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + if len(states) == 0 { + return testserver.Response{ + StatusCode: 404, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "App not found."}, + } + } + state := states[min(*gets, len(states)-1)] + *gets++ + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} + }) + return server +} + +func newAppForServer(t *testing.T, server *testserver.Server) *ResourceApp { + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + return (&ResourceApp{}).New(client) +} + func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { tests := []struct { - name string - // computeStates are the states GET returns in order; the last one repeats. + name string + message string computeStates []apps.ComputeState wantDeletes int - wantErr bool + wantGets int + wantErr string }{ { name: "retries after compute leaves UPDATING", + message: appComputeUpdatingDeleteMessage, computeStates: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive}, wantDeletes: 2, + wantGets: 2, }, { - name: "fails at once when compute is not transitional", + name: "retries when compute settled before the first read", + message: appComputeUpdatingDeleteMessage, computeStates: []apps.ComputeState{apps.ComputeStateActive}, - wantDeletes: 1, - wantErr: true, + wantDeletes: 2, + wantGets: 1, + }, + { + name: "treats an app gone while waiting as deleted", + message: appComputeUpdatingDeleteMessage, + wantDeletes: 1, }, { - name: "fails at once when app is already deleting", + name: "does not retry a delete of a DELETING app", + message: "Cannot delete app test-app as it is not terminal with state DELETING, and was updated less than 20 minutes ago. Please wait before trying again.", computeStates: []apps.ComputeState{apps.ComputeStateDeleting}, wantDeletes: 1, - wantErr: true, + wantErr: "state DELETING", + }, + { + name: "does not retry other bad requests", + message: "Invalid app name.", + computeStates: []apps.ComputeState{apps.ComputeStateUpdating}, + wantDeletes: 1, + wantErr: "Invalid app name.", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - server := testserver.New(t) - deletes := 0 gets := 0 - + server := newAppComputeSettleServer(t, &gets, tc.computeStates...) + deletes := 0 server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { deletes++ if deletes == 1 { return testserver.Response{ StatusCode: 400, - Body: map[string]string{ - "error_code": "BAD_REQUEST", - "message": "Cannot delete app test-app as it is not terminal with state UPDATING, and was updated less than 20 minutes ago. Please wait before trying again.", - }, + Body: map[string]string{"error_code": "BAD_REQUEST", "message": tc.message}, } } return apps.App{Name: req.Vars["name"]} }) - server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { - state := tc.computeStates[min(gets, len(tc.computeStates)-1)] - gets++ - return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} - }) - testserver.AddDefaultHandlers(server) + r := newAppForServer(t, server) - client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) - require.NoError(t, err) - - r := (&ResourceApp{}).New(client) - err = r.DoDelete(t.Context(), "test-app", nil) - if tc.wantErr { - require.ErrorContains(t, err, "not terminal with state UPDATING") + err := r.DoDelete(t.Context(), "test-app", nil) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) } else { require.NoError(t, err) } assert.Equal(t, tc.wantDeletes, deletes) + assert.Equal(t, tc.wantGets, gets) }) } } -func TestAppRetryWhileComputeSettles_IgnoresNonBadRequest(t *testing.T) { - server := testserver.New(t) +func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { gets := 0 - server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { - gets++ - return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: apps.ComputeStateUpdating}} + server := newAppComputeSettleServer(t, &gets, apps.ComputeStateUpdating, apps.ComputeStateActive) + updates := 0 + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + updates++ + if updates == 1 { + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "BAD_REQUEST", + "message": "Cannot update app test-app as its compute is in UPDATING state. App compute needs to be ACTIVE or STOPPED to update.", + }, + } + } + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} }) - testserver.AddDefaultHandlers(server) + server.Handle("GET", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + }) + r := newAppForServer(t, server) - client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + entry := &PlanEntry{Changes: deployplan.Changes{"description": {Action: deployplan.Update}}} + _, err := r.DoUpdate(t.Context(), "test-app", &AppState{Name: "test-app", Description: "new"}, entry) require.NoError(t, err) - - r := (&ResourceApp{}).New(client) - calls := 0 - err = r.retryWhileComputeSettles(t.Context(), "test-app", func() error { - calls++ - return &apierr.APIError{StatusCode: 500, ErrorCode: "INTERNAL_ERROR", Message: "boom"} - }) - require.ErrorContains(t, err, "boom") - assert.Equal(t, 1, calls) - assert.Equal(t, 0, gets) + assert.Equal(t, 2, updates) + assert.Equal(t, 2, gets) }