diff --git a/.nextchanges/bundles/7533.md b/.nextchanges/bundles/7533.md new file mode 100644 index 00000000000..59a141f69a9 --- /dev/null +++ b/.nextchanges/bundles/7533.md @@ -0,0 +1 @@ +* Show how to recover when `bundle deploy` fails because an app's service principal does not exist or was deleted. ([#7533](https://github.com/databricks/cli/pull/7533)) diff --git a/acceptance/bundle/resources/apps/missing-service-principal/app/app.py b/acceptance/bundle/resources/apps/missing-service-principal/app/app.py new file mode 100644 index 00000000000..f1a18139c84 --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/missing-service-principal/databricks.yml b/acceptance/bundle/resources/apps/missing-service-principal/databricks.yml new file mode 100644 index 00000000000..daec10a039b --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-bundle + +resources: + apps: + mykey: + name: myappname + description: my_app_description + source_code_path: ./app + lifecycle: + started: false diff --git a/acceptance/bundle/resources/apps/missing-service-principal/out.test.toml b/acceptance/bundle/resources/apps/missing-service-principal/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/apps/missing-service-principal/output.txt b/acceptance/bundle/resources/apps/missing-service-principal/output.txt new file mode 100644 index 00000000000..fa5969f6e1a --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/output.txt @@ -0,0 +1,77 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created apps.mykey +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Update fails: the error explains how to recover + +>>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.apps.mykey: updating id=myappname: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. + +The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again, using the same profile and target as for this deploy. Deleting the app also deletes its deployment history (400 BAD_REQUEST) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/apps/myappname/update +HTTP Status: 400 Bad Request +API error_code: BAD_REQUEST +API message: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py //apps +{ + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + }, + "body": { + "description": "my_app_description", + "name": "myappname" + } +} +{ + "method": "POST", + "path": "/api/2.0/apps/myappname/update", + "body": { + "app": { + "description": "MY_APP_DESCRIPTION", + "name": "myappname" + }, + "update_mask": "description,budget_policy_id,usage_policy_id,resources,user_api_scopes,compute_min_instances,compute_max_instances,git_repository,telemetry_export_destinations" + } +} + +=== Start fails: the error explains how to recover + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION my_app_description + +>>> update_file.py databricks.yml started: false started: true + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.apps.mykey: updating id=myappname: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. + +The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again, using the same profile and target as for this deploy. Deleting the app also deletes its deployment history (400 BAD_REQUEST) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/apps/myappname/start +HTTP Status: 400 Bad Request +API error_code: BAD_REQUEST +API message: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py //apps +{ + "method": "POST", + "path": "/api/2.0/apps/myappname/start", + "body": {} +} diff --git a/acceptance/bundle/resources/apps/missing-service-principal/script b/acceptance/bundle/resources/apps/missing-service-principal/script new file mode 100644 index 00000000000..de1674e4a1b --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/script @@ -0,0 +1,13 @@ +trace $CLI bundle deploy +rm -f out.requests.txt + +title "Update fails: the error explains how to recover" +trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION +trace errcode $CLI bundle deploy +trace print_requests.py //apps + +title "Start fails: the error explains how to recover" +trace update_file.py databricks.yml MY_APP_DESCRIPTION my_app_description +trace update_file.py databricks.yml "started: false" "started: true" +trace errcode $CLI bundle deploy +trace print_requests.py //apps diff --git a/acceptance/bundle/resources/apps/missing-service-principal/test.toml b/acceptance/bundle/resources/apps/missing-service-principal/test.toml new file mode 100644 index 00000000000..03921e4088c --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/test.toml @@ -0,0 +1,15 @@ +Cloud = false +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +# Simulate an app whose service principal was deleted on the backend: updates and starts fail. +[[Server]] +Pattern = "POST /api/2.0/apps/{name}/update" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "BAD_REQUEST", "message": "App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app."}''' + +[[Server]] +Pattern = "POST /api/2.0/apps/{name}/start" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "BAD_REQUEST", "message": "App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app."}''' diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 841371d323b..f76681feb98 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -191,6 +191,10 @@ func appUpdateMask(app *apps.App) string { } func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) (*AppRemote, error) { + return nil, withMissingServicePrincipalHint(r.doUpdate(ctx, id, config, entry), id) +} + +func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error { // Deploy-only fields (source_code_path, config, git_source, lifecycle) are excluded // from the request body; see appRequestBody. if hasAppChanges(entry) { @@ -202,20 +206,36 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) if err != nil { - return nil, err + return err } response, err := updateWaiter.Get() if err != nil { - return nil, err + return err } if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { - return nil, fmt.Errorf("update status: %s: %s", response.Status.State, response.Status.Message) + return fmt.Errorf("update status: %s: %s", response.Status.State, response.Status.Message) } } - return nil, r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) + return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) +} + +// missingServicePrincipalMessage is part of the backend error for an app whose service +// principal is gone. The backend reports it as a generic 400 BAD_REQUEST or as a compute +// ERROR status message, so there is no error code or sentinel to match on. Such an app +// cannot be updated or started again, only recreated. +const missingServicePrincipalMessage = "service principal does not exist or is deleted" + +func withMissingServicePrincipalHint(err error, name string) error { + if err == nil || !strings.Contains(strings.ToLower(err.Error()), missingServicePrincipalMessage) { + return err + } + return fmt.Errorf("%w\n\nThe app cannot be updated or started without its service principal. "+ + "To recreate the app, delete it with \"databricks apps delete %s\", wait for the deletion to finish, "+ + "and run \"databricks bundle deploy\" again, using the same profile and target as for this deploy. "+ + "Deleting the app also deletes its deployment history", err, name) } func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *AppState, alreadyStarted bool) error { @@ -343,6 +363,11 @@ func (*ResourceApp) IsGone(remote *AppRemote) bool { } func (r *ResourceApp) WaitAfterCreate(ctx context.Context, id string, config *AppState) (*AppRemote, error) { + remote, err := r.waitAfterCreate(ctx, config) + return remote, withMissingServicePrincipalHint(err, config.Name) +} + +func (r *ResourceApp) waitAfterCreate(ctx context.Context, config *AppState) (*AppRemote, error) { remote, err := r.waitForApp(ctx, r.client, config.Name) if err != nil { return nil, err diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 1e064a95a20..01b619bae21 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -2,6 +2,8 @@ package dresources import ( "encoding/json" + "errors" + "fmt" "reflect" "slices" "strings" @@ -9,6 +11,7 @@ import ( "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -224,3 +227,24 @@ func TestAppRequestBody_StripsSourceCodePathFromForceSendFields(t *testing.T) { // Other ForceSendFields entries must be preserved. assert.Contains(t, body.ForceSendFields, "Name") } + +func TestWithMissingServicePrincipalHint(t *testing.T) { + assert.NoError(t, withMissingServicePrincipalHint(nil, "myapp")) + + other := errors.New("some other error") + assert.Same(t, other, withMissingServicePrincipalHint(other, "myapp")) + + apiErr := &apierr.APIError{ + StatusCode: 400, + ErrorCode: "BAD_REQUEST", + Message: "App: myapp's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app.", + } + computeErr := errors.New("failed to reach ACTIVE, got ERROR: App service principal does not exist or is deleted, application cannot run without a valid service principal. Please delete the app and re-create it.") + capitalizedErr := errors.New("Service principal does not exist or is deleted") + + for _, err := range []error{apiErr, fmt.Errorf("wrapped: %w", apiErr), computeErr, capitalizedErr} { + got := withMissingServicePrincipalHint(err, "myapp") + assert.ErrorIs(t, got, err) + assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`) + } +}