From 6c5e2ba19b88b593652426112a8d51ff73659746 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 09:44:27 +0000 Subject: [PATCH 1/4] Add a hint when an app's service principal is missing during bundle deploy Co-authored-by: Isaac --- .../apps/missing-service-principal/app/app.py | 1 + .../missing-service-principal/databricks.yml | 11 +++ .../missing-service-principal/out.test.toml | 2 + .../apps/missing-service-principal/output.txt | 77 +++++++++++++++++++ .../apps/missing-service-principal/script | 14 ++++ .../apps/missing-service-principal/test.toml | 15 ++++ bundle/direct/dresources/app.go | 23 ++++-- 7 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 acceptance/bundle/resources/apps/missing-service-principal/app/app.py create mode 100644 acceptance/bundle/resources/apps/missing-service-principal/databricks.yml create mode 100644 acceptance/bundle/resources/apps/missing-service-principal/out.test.toml create mode 100644 acceptance/bundle/resources/apps/missing-service-principal/output.txt create mode 100644 acceptance/bundle/resources/apps/missing-service-principal/script create mode 100644 acceptance/bundle/resources/apps/missing-service-principal/test.toml diff --git a/acceptance/bundle/resources/apps/missing-service-principal/app/app.py b/acceptance/bundle/resources/apps/missing-service-principal/app/app.py new file mode 100644 index 00000000000..f1a18139c84 --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/missing-service-principal/databricks.yml b/acceptance/bundle/resources/apps/missing-service-principal/databricks.yml new file mode 100644 index 00000000000..daec10a039b --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-bundle + +resources: + apps: + mykey: + name: myappname + description: my_app_description + source_code_path: ./app + lifecycle: + started: false diff --git a/acceptance/bundle/resources/apps/missing-service-principal/out.test.toml b/acceptance/bundle/resources/apps/missing-service-principal/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/apps/missing-service-principal/output.txt b/acceptance/bundle/resources/apps/missing-service-principal/output.txt new file mode 100644 index 00000000000..426f22ce20d --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/output.txt @@ -0,0 +1,77 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created apps.mykey +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Update fails: the error explains how to recover + +>>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.apps.mykey: updating id=myappname: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. + +The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again. Deleting the app also deletes its deployment history (400 BAD_REQUEST) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/apps/myappname/update +HTTP Status: 400 Bad Request +API error_code: BAD_REQUEST +API message: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py //apps +{ + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + }, + "body": { + "description": "my_app_description", + "name": "myappname" + } +} +{ + "method": "POST", + "path": "/api/2.0/apps/myappname/update", + "body": { + "app": { + "description": "MY_APP_DESCRIPTION", + "name": "myappname" + }, + "update_mask": "description,budget_policy_id,usage_policy_id,resources,user_api_scopes,compute_min_instances,compute_max_instances,git_repository,telemetry_export_destinations" + } +} + +=== Start fails: the error explains how to recover + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION my_app_description + +>>> update_file.py databricks.yml started: false started: true + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot update resources.apps.mykey: updating id=myappname: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. + +The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again. Deleting the app also deletes its deployment history (400 BAD_REQUEST) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/apps/myappname/start +HTTP Status: 400 Bad Request +API error_code: BAD_REQUEST +API message: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py //apps +{ + "method": "POST", + "path": "/api/2.0/apps/myappname/start", + "body": {} +} diff --git a/acceptance/bundle/resources/apps/missing-service-principal/script b/acceptance/bundle/resources/apps/missing-service-principal/script new file mode 100644 index 00000000000..401f23f4019 --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/script @@ -0,0 +1,14 @@ +trace $CLI bundle deploy +rm -f out.requests.txt + +title "Update fails: the error explains how to recover" +trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION +trace errcode $CLI bundle deploy +trace print_requests.py //apps + +title "Start fails: the error explains how to recover" +trace update_file.py databricks.yml MY_APP_DESCRIPTION my_app_description +trace update_file.py databricks.yml "started: false" "started: true" +trace errcode $CLI bundle deploy +trace print_requests.py //apps + diff --git a/acceptance/bundle/resources/apps/missing-service-principal/test.toml b/acceptance/bundle/resources/apps/missing-service-principal/test.toml new file mode 100644 index 00000000000..03921e4088c --- /dev/null +++ b/acceptance/bundle/resources/apps/missing-service-principal/test.toml @@ -0,0 +1,15 @@ +Cloud = false +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +# Simulate an app whose service principal was deleted on the backend: updates and starts fail. +[[Server]] +Pattern = "POST /api/2.0/apps/{name}/update" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "BAD_REQUEST", "message": "App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app."}''' + +[[Server]] +Pattern = "POST /api/2.0/apps/{name}/start" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "BAD_REQUEST", "message": "App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app."}''' diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 841371d323b..3579f3f3b2f 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -202,12 +202,12 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) if err != nil { - return nil, err + return nil, withMissingServicePrincipalHint(err, id) } response, err := updateWaiter.Get() if err != nil { - return nil, err + return nil, withMissingServicePrincipalHint(err, id) } if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { @@ -215,7 +215,20 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } } - return nil, r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) + return nil, withMissingServicePrincipalHint(r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)), id) +} + +// missingServicePrincipalMessage is part of the backend error for an app whose service +// principal is gone. Such an app cannot be updated or started again, only recreated. +const missingServicePrincipalMessage = "service principal does not exist or is deleted" + +func withMissingServicePrincipalHint(err error, name string) error { + if err == nil || !strings.Contains(err.Error(), missingServicePrincipalMessage) { + return err + } + return fmt.Errorf("%w\n\nThe app cannot be updated or started without its service principal. "+ + "To recreate the app, delete it with \"databricks apps delete %s\", wait for the deletion to finish, "+ + "and run \"databricks bundle deploy\" again. Deleting the app also deletes its deployment history", err, name) } func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *AppState, alreadyStarted bool) error { @@ -345,11 +358,11 @@ func (*ResourceApp) IsGone(remote *AppRemote) bool { func (r *ResourceApp) WaitAfterCreate(ctx context.Context, id string, config *AppState) (*AppRemote, error) { remote, err := r.waitForApp(ctx, r.client, config.Name) if err != nil { - return nil, err + return nil, withMissingServicePrincipalHint(err, config.Name) } alreadyStarted := remote.Lifecycle != nil && remote.Lifecycle.Started != nil && *remote.Lifecycle.Started if err := r.manageLifecycle(ctx, config.Name, config, alreadyStarted); err != nil { - return nil, err + return nil, withMissingServicePrincipalHint(err, config.Name) } return remote, nil } From 935e1a2e16ba9b7ca0341c47e5c512f98af80fed Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 09:45:27 +0000 Subject: [PATCH 2/4] Add changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/7533.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7533.md diff --git a/.nextchanges/bundles/7533.md b/.nextchanges/bundles/7533.md new file mode 100644 index 00000000000..59a141f69a9 --- /dev/null +++ b/.nextchanges/bundles/7533.md @@ -0,0 +1 @@ +* Show how to recover when `bundle deploy` fails because an app's service principal does not exist or was deleted. ([#7533](https://github.com/databricks/cli/pull/7533)) From 40feada1a05118bb044d91efda18ae266dac58e6 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 09:52:01 +0000 Subject: [PATCH 3/4] Wrap app errors once per method; match case-insensitively; add unit test Co-authored-by: Isaac --- .../apps/missing-service-principal/output.txt | 4 +-- bundle/direct/dresources/app.go | 30 +++++++++++++------ bundle/direct/dresources/app_test.go | 24 +++++++++++++++ 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/resources/apps/missing-service-principal/output.txt b/acceptance/bundle/resources/apps/missing-service-principal/output.txt index 426f22ce20d..fa5969f6e1a 100644 --- a/acceptance/bundle/resources/apps/missing-service-principal/output.txt +++ b/acceptance/bundle/resources/apps/missing-service-principal/output.txt @@ -13,7 +13,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: cannot update resources.apps.mykey: updating id=myappname: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. -The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again. Deleting the app also deletes its deployment history (400 BAD_REQUEST) +The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again, using the same profile and target as for this deploy. Deleting the app also deletes its deployment history (400 BAD_REQUEST) Endpoint: POST [DATABRICKS_URL]/api/2.0/apps/myappname/update HTTP Status: 400 Bad Request @@ -58,7 +58,7 @@ Exit code: 1 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: cannot update resources.apps.mykey: updating id=myappname: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app. -The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again. Deleting the app also deletes its deployment history (400 BAD_REQUEST) +The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again, using the same profile and target as for this deploy. Deleting the app also deletes its deployment history (400 BAD_REQUEST) Endpoint: POST [DATABRICKS_URL]/api/2.0/apps/myappname/start HTTP Status: 400 Bad Request diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 3579f3f3b2f..f76681feb98 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -191,6 +191,10 @@ func appUpdateMask(app *apps.App) string { } func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) (*AppRemote, error) { + return nil, withMissingServicePrincipalHint(r.doUpdate(ctx, id, config, entry), id) +} + +func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error { // Deploy-only fields (source_code_path, config, git_source, lifecycle) are excluded // from the request body; see appRequestBody. if hasAppChanges(entry) { @@ -202,33 +206,36 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) if err != nil { - return nil, withMissingServicePrincipalHint(err, id) + return err } response, err := updateWaiter.Get() if err != nil { - return nil, withMissingServicePrincipalHint(err, id) + return err } if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { - return nil, fmt.Errorf("update status: %s: %s", response.Status.State, response.Status.Message) + return fmt.Errorf("update status: %s: %s", response.Status.State, response.Status.Message) } } - return nil, withMissingServicePrincipalHint(r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)), id) + return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) } // missingServicePrincipalMessage is part of the backend error for an app whose service -// principal is gone. Such an app cannot be updated or started again, only recreated. +// principal is gone. The backend reports it as a generic 400 BAD_REQUEST or as a compute +// ERROR status message, so there is no error code or sentinel to match on. Such an app +// cannot be updated or started again, only recreated. const missingServicePrincipalMessage = "service principal does not exist or is deleted" func withMissingServicePrincipalHint(err error, name string) error { - if err == nil || !strings.Contains(err.Error(), missingServicePrincipalMessage) { + if err == nil || !strings.Contains(strings.ToLower(err.Error()), missingServicePrincipalMessage) { return err } return fmt.Errorf("%w\n\nThe app cannot be updated or started without its service principal. "+ "To recreate the app, delete it with \"databricks apps delete %s\", wait for the deletion to finish, "+ - "and run \"databricks bundle deploy\" again. Deleting the app also deletes its deployment history", err, name) + "and run \"databricks bundle deploy\" again, using the same profile and target as for this deploy. "+ + "Deleting the app also deletes its deployment history", err, name) } func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *AppState, alreadyStarted bool) error { @@ -356,13 +363,18 @@ func (*ResourceApp) IsGone(remote *AppRemote) bool { } func (r *ResourceApp) WaitAfterCreate(ctx context.Context, id string, config *AppState) (*AppRemote, error) { + remote, err := r.waitAfterCreate(ctx, config) + return remote, withMissingServicePrincipalHint(err, config.Name) +} + +func (r *ResourceApp) waitAfterCreate(ctx context.Context, config *AppState) (*AppRemote, error) { remote, err := r.waitForApp(ctx, r.client, config.Name) if err != nil { - return nil, withMissingServicePrincipalHint(err, config.Name) + return nil, err } alreadyStarted := remote.Lifecycle != nil && remote.Lifecycle.Started != nil && *remote.Lifecycle.Started if err := r.manageLifecycle(ctx, config.Name, config, alreadyStarted); err != nil { - return nil, withMissingServicePrincipalHint(err, config.Name) + return nil, err } return remote, nil } diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 1e064a95a20..01b619bae21 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -2,6 +2,8 @@ package dresources import ( "encoding/json" + "errors" + "fmt" "reflect" "slices" "strings" @@ -9,6 +11,7 @@ import ( "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -224,3 +227,24 @@ func TestAppRequestBody_StripsSourceCodePathFromForceSendFields(t *testing.T) { // Other ForceSendFields entries must be preserved. assert.Contains(t, body.ForceSendFields, "Name") } + +func TestWithMissingServicePrincipalHint(t *testing.T) { + assert.NoError(t, withMissingServicePrincipalHint(nil, "myapp")) + + other := errors.New("some other error") + assert.Same(t, other, withMissingServicePrincipalHint(other, "myapp")) + + apiErr := &apierr.APIError{ + StatusCode: 400, + ErrorCode: "BAD_REQUEST", + Message: "App: myapp's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app.", + } + computeErr := errors.New("failed to reach ACTIVE, got ERROR: App service principal does not exist or is deleted, application cannot run without a valid service principal. Please delete the app and re-create it.") + capitalizedErr := errors.New("Service principal does not exist or is deleted") + + for _, err := range []error{apiErr, fmt.Errorf("wrapped: %w", apiErr), computeErr, capitalizedErr} { + got := withMissingServicePrincipalHint(err, "myapp") + assert.ErrorIs(t, got, err) + assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`) + } +} From bea51a5a4b65868f22c53e26799e74ab9b8e33ff Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 10:19:05 +0000 Subject: [PATCH 4/4] Remove trailing blank line in the test script aws: TestAccept/bundle/resources/apps/missing-service-principal Co-authored-by: Isaac --- .../bundle/resources/apps/missing-service-principal/script | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/bundle/resources/apps/missing-service-principal/script b/acceptance/bundle/resources/apps/missing-service-principal/script index 401f23f4019..de1674e4a1b 100644 --- a/acceptance/bundle/resources/apps/missing-service-principal/script +++ b/acceptance/bundle/resources/apps/missing-service-principal/script @@ -11,4 +11,3 @@ trace update_file.py databricks.yml MY_APP_DESCRIPTION my_app_description trace update_file.py databricks.yml "started: false" "started: true" trace errcode $CLI bundle deploy trace print_requests.py //apps -