From 0945c6fab168eeeafcc213ce35a34116c79b3cbb Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 09:38:48 +0000 Subject: [PATCH 1/2] Retry the root path read in bundle destroy bundle destroy checks that the bundle root path exists with workspace/get-status before anything else. That read was not retried, so a single cut connection failed the whole destroy. It now retries transient failures like the other bundle get-status reads. azure: TestAccept/bundle/resources/volumes/set-storage-location Co-authored-by: Isaac --- .../destroy/get-status-500/databricks.yml.tmpl | 7 +++++++ .../bundle/destroy/get-status-500/out.test.toml | 2 ++ .../bundle/destroy/get-status-500/output.txt | 17 +++++++++++++++++ acceptance/bundle/destroy/get-status-500/script | 9 +++++++++ .../bundle/destroy/get-status-500/test.toml | 9 +++++++++ bundle/phases/destroy.go | 7 ++++++- 6 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/destroy/get-status-500/databricks.yml.tmpl create mode 100644 acceptance/bundle/destroy/get-status-500/out.test.toml create mode 100644 acceptance/bundle/destroy/get-status-500/output.txt create mode 100644 acceptance/bundle/destroy/get-status-500/script create mode 100644 acceptance/bundle/destroy/get-status-500/test.toml diff --git a/acceptance/bundle/destroy/get-status-500/databricks.yml.tmpl b/acceptance/bundle/destroy/get-status-500/databricks.yml.tmpl new file mode 100644 index 00000000000..f83235c2a9b --- /dev/null +++ b/acceptance/bundle/destroy/get-status-500/databricks.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: destroy-get-status-500-$UNIQUE_NAME + +resources: + jobs: + foo: + name: foo-$UNIQUE_NAME diff --git a/acceptance/bundle/destroy/get-status-500/out.test.toml b/acceptance/bundle/destroy/get-status-500/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/destroy/get-status-500/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/destroy/get-status-500/output.txt b/acceptance/bundle/destroy/get-status-500/output.txt new file mode 100644 index 00000000000..c599ca687e8 --- /dev/null +++ b/acceptance/bundle/destroy/get-status-500/output.txt @@ -0,0 +1,17 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/destroy-get-status-500-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A 500 on the root path get-status is retried + +>>> [CLI] bundle destroy --auto-approve +Warn: retrying after 500 Internal Server Error from GET /api/2.0/workspace/get-status +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/destroy-get-status-500-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/destroy/get-status-500/script b/acceptance/bundle/destroy/get-status-500/script new file mode 100644 index 00000000000..f298c29202c --- /dev/null +++ b/acceptance/bundle/destroy/get-status-500/script @@ -0,0 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +trace $CLI bundle deploy + +# Destroy first checks that the bundle root path exists. A 500 from that read (seen on cloud as +# "unexpected EOF" when the connection is cut) is retried instead of failing the destroy. +title "A 500 on the root path get-status is retried" +fault.py "GET /api/2.0/workspace/get-status" 500 0 1 +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/destroy/get-status-500/test.toml b/acceptance/bundle/destroy/get-status-500/test.toml new file mode 100644 index 00000000000..4a2b07abe0f --- /dev/null +++ b/acceptance/bundle/destroy/get-status-500/test.toml @@ -0,0 +1,9 @@ +Cloud = false + +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" + +Ignore = [ + "databricks.yml", + ".databricks/" +] diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 07fed65ba80..94ee9734912 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -18,12 +18,17 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/workspace" ) func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) { w := b.WorkspaceClient(ctx) - _, err := w.Workspace.GetStatusByPath(ctx, b.Config.Workspace.RootPath) + // The read has no side effects, so a transient failure is safe to replay instead of failing destroy. + _, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) { + return w.Workspace.GetStatusByPath(ctx, b.Config.Workspace.RootPath) + }) if aerr, ok := errors.AsType[*apierr.APIError](err); ok && aerr.StatusCode == http.StatusNotFound { log.Infof(ctx, "Root path does not exist: %s", b.Config.Workspace.RootPath) From 531edd0ffc57c7663e0399794f03275d227a6796 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 09:39:46 +0000 Subject: [PATCH 2/2] Add changelog entry azure: TestAccept/bundle/resources/volumes/set-storage-location Co-authored-by: Isaac --- .nextchanges/bundles/7597.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7597.md diff --git a/.nextchanges/bundles/7597.md b/.nextchanges/bundles/7597.md new file mode 100644 index 00000000000..33fd6e6eb06 --- /dev/null +++ b/.nextchanges/bundles/7597.md @@ -0,0 +1 @@ +* Retry the root path check in `bundle destroy` on transient errors, so a dropped connection no longer fails the destroy. ([#7597](https://github.com/databricks/cli/pull/7597))