Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7533.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Show how to recover when `bundle deploy` fails because an app's service principal does not exist or was deleted. ([#7533](https://github.com/databricks/cli/pull/7533))
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
print("Hello world!")
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
bundle:
name: test-bundle

resources:
apps:
mykey:
name: myappname
description: my_app_description
source_code_path: ./app
lifecycle:
started: false
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created apps.mykey
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

=== Update fails: the error explains how to recover

>>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION

>>> errcode [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Error: cannot update resources.apps.mykey: updating id=myappname: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app.

The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again, using the same profile and target as for this deploy. Deleting the app also deletes its deployment history (400 BAD_REQUEST)

Endpoint: POST [DATABRICKS_URL]/api/2.0/apps/myappname/update
HTTP Status: 400 Bad Request
API error_code: BAD_REQUEST
API message: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app.

Files: 1 uploaded, 0 deleted

Exit code: 1

>>> print_requests.py //apps
{
"method": "POST",
"path": "/api/2.0/apps",
"q": {
"no_compute": "true"
},
"body": {
"description": "my_app_description",
"name": "myappname"
}
}
{
"method": "POST",
"path": "/api/2.0/apps/myappname/update",
"body": {
"app": {
"description": "MY_APP_DESCRIPTION",
"name": "myappname"
},
"update_mask": "description,budget_policy_id,usage_policy_id,resources,user_api_scopes,compute_min_instances,compute_max_instances,git_repository,telemetry_export_destinations"
}
}

=== Start fails: the error explains how to recover

>>> update_file.py databricks.yml MY_APP_DESCRIPTION my_app_description

>>> update_file.py databricks.yml started: false started: true

>>> errcode [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Error: cannot update resources.apps.mykey: updating id=myappname: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app.

The app cannot be updated or started without its service principal. To recreate the app, delete it with "databricks apps delete myappname", wait for the deletion to finish, and run "databricks bundle deploy" again, using the same profile and target as for this deploy. Deleting the app also deletes its deployment history (400 BAD_REQUEST)

Endpoint: POST [DATABRICKS_URL]/api/2.0/apps/myappname/start
HTTP Status: 400 Bad Request
API error_code: BAD_REQUEST
API message: App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app.

Files: 1 uploaded, 0 deleted

Exit code: 1

>>> print_requests.py //apps
{
"method": "POST",
"path": "/api/2.0/apps/myappname/start",
"body": {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
trace $CLI bundle deploy
rm -f out.requests.txt

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This rm -f is a no-op: the harness records to $OUT_REQUESTS in a sibling dir (acceptance_test.go:913), so the first print_requests also lists the create. Drop the line?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, thanks. It is a no-op; removed in #7545 (the golden output is unchanged, since the first print_requests already listed the create).

AI agent (harness_auto, opus5.5, isaac_claude)


title "Update fails: the error explains how to recover"
trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION
trace errcode $CLI bundle deploy
trace print_requests.py //apps

title "Start fails: the error explains how to recover"
trace update_file.py databricks.yml MY_APP_DESCRIPTION my_app_description
trace update_file.py databricks.yml "started: false" "started: true"
trace errcode $CLI bundle deploy
trace print_requests.py //apps
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
Cloud = false
RecordRequests = true

Ignore = [".databricks", "databricks.yml"]

# Simulate an app whose service principal was deleted on the backend: updates and starts fail.
[[Server]]
Pattern = "POST /api/2.0/apps/{name}/update"
Response.StatusCode = 400
Response.Body = '''{"error_code": "BAD_REQUEST", "message": "App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app."}'''

[[Server]]
Pattern = "POST /api/2.0/apps/{name}/start"
Response.StatusCode = 400
Response.Body = '''{"error_code": "BAD_REQUEST", "message": "App: myappname's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app."}'''
33 changes: 29 additions & 4 deletions bundle/direct/dresources/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,10 @@ func appUpdateMask(app *apps.App) string {
}

func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) (*AppRemote, error) {
return nil, withMissingServicePrincipalHint(r.doUpdate(ctx, id, config, entry), id)
}

func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error {
// Deploy-only fields (source_code_path, config, git_source, lifecycle) are excluded
// from the request body; see appRequestBody.
if hasAppChanges(entry) {
Expand All @@ -202,20 +206,36 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState,
}
updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request)
if err != nil {
return nil, err
return err
}

response, err := updateWaiter.Get()
if err != nil {
return nil, err
return err
}

if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded {
return nil, fmt.Errorf("update status: %s: %s", response.Status.State, response.Status.Message)
return fmt.Errorf("update status: %s: %s", response.Status.State, response.Status.Message)
}
}

return nil, r.manageLifecycle(ctx, id, config, remoteIsStarted(entry))
return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry))
}

// missingServicePrincipalMessage is part of the backend error for an app whose service
// principal is gone. The backend reports it as a generic 400 BAD_REQUEST or as a compute
// ERROR status message, so there is no error code or sentinel to match on. Such an app
// cannot be updated or started again, only recreated.
const missingServicePrincipalMessage = "service principal does not exist or is deleted"

func withMissingServicePrincipalHint(err error, name string) error {
if err == nil || !strings.Contains(strings.ToLower(err.Error()), missingServicePrincipalMessage) {
return err
}
return fmt.Errorf("%w\n\nThe app cannot be updated or started without its service principal. "+
"To recreate the app, delete it with \"databricks apps delete %s\", wait for the deletion to finish, "+

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DoCreate already retries while the app is DELETING (app.go:143) and IsGone plans a create for it, so "wait for the deletion to finish" isn't needed. Drop that clause?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the clause is still needed. IsGone is only checked for resources being deleted (bundle_plan.go, delete branch, and deleteConfirmedGone in apply.go). For an app still in config, the planner reads the app, and while it is DELETING, GET still returns it. A redeploy then plans an update, not a create, and the DoCreate DELETING retry is never reached. I checked on the testserver: bundle deploy, then apps delete myappname, then bundle plan shows update apps.mykey (Plan: 0 to add, 1 to change). A create is planned only after GET returns 404, which means after the deletion finishes. So I left the hint as is in the follow-up #7545. Happy to drop the clause if you would rather handle DELETING in the planner instead.

AI agent (harness_auto, opus5.5, isaac_claude)

"and run \"databricks bundle deploy\" again, using the same profile and target as for this deploy. "+
"Deleting the app also deletes its deployment history", err, name)
}

func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *AppState, alreadyStarted bool) error {
Expand Down Expand Up @@ -343,6 +363,11 @@ func (*ResourceApp) IsGone(remote *AppRemote) bool {
}

func (r *ResourceApp) WaitAfterCreate(ctx context.Context, id string, config *AppState) (*AppRemote, error) {
remote, err := r.waitAfterCreate(ctx, config)
return remote, withMissingServicePrincipalHint(err, config.Name)
}

func (r *ResourceApp) waitAfterCreate(ctx context.Context, config *AppState) (*AppRemote, error) {
remote, err := r.waitForApp(ctx, r.client, config.Name)
if err != nil {
return nil, err
Expand Down
24 changes: 24 additions & 0 deletions bundle/direct/dresources/app_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,16 @@ package dresources

import (
"encoding/json"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"testing"

"github.com/databricks/cli/libs/testserver"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -224,3 +227,24 @@ func TestAppRequestBody_StripsSourceCodePathFromForceSendFields(t *testing.T) {
// Other ForceSendFields entries must be preserved.
assert.Contains(t, body.ForceSendFields, "Name")
}

func TestWithMissingServicePrincipalHint(t *testing.T) {
assert.NoError(t, withMissingServicePrincipalHint(nil, "myapp"))

other := errors.New("some other error")
assert.Same(t, other, withMissingServicePrincipalHint(other, "myapp"))

apiErr := &apierr.APIError{
StatusCode: 400,
ErrorCode: "BAD_REQUEST",
Message: "App: myapp's service principal does not exist or is deleted. App cannot run without a valid service principal. Please delete your app.",
}
computeErr := errors.New("failed to reach ACTIVE, got ERROR: App service principal does not exist or is deleted, application cannot run without a valid service principal. Please delete the app and re-create it.")
capitalizedErr := errors.New("Service principal does not exist or is deleted")

for _, err := range []error{apiErr, fmt.Errorf("wrapped: %w", apiErr), computeErr, capitalizedErr} {
got := withMissingServicePrincipalHint(err, "myapp")
assert.ErrorIs(t, got, err)
assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`)
}
}
Loading