From 9935ba05121fbf577aba7a5eef4c3e33f27d5bf7 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 08:55:57 +0000 Subject: [PATCH 1/9] Retry the deploy lock create after transient errors The lock create (import-file with overwrite=false) was the one workspace write left out of the retries added in #7521, so an inactivity timeout or a 5xx on it still failed the deploy. Retrying it is safe: the lock file carries the locker's unique ID, so if an earlier attempt did create the file, the retry gets a conflict and assertLockHeld confirms the lock is ours. Co-authored-by: Isaac --- .nextchanges/bundles/lock-retry.md | 1 + .../bundle/deploy/lock-retry/databricks.yml | 2 ++ .../bundle/deploy/lock-retry/out.test.toml | 2 ++ .../bundle/deploy/lock-retry/output.txt | 34 +++++++++++++++++++ acceptance/bundle/deploy/lock-retry/script | 20 +++++++++++ acceptance/bundle/deploy/lock-retry/test.toml | 3 ++ libs/locker/locker.go | 13 ++++++- 7 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 .nextchanges/bundles/lock-retry.md create mode 100644 acceptance/bundle/deploy/lock-retry/databricks.yml create mode 100644 acceptance/bundle/deploy/lock-retry/out.test.toml create mode 100644 acceptance/bundle/deploy/lock-retry/output.txt create mode 100644 acceptance/bundle/deploy/lock-retry/script create mode 100644 acceptance/bundle/deploy/lock-retry/test.toml diff --git a/.nextchanges/bundles/lock-retry.md b/.nextchanges/bundles/lock-retry.md new file mode 100644 index 00000000000..3dfe66b8fdc --- /dev/null +++ b/.nextchanges/bundles/lock-retry.md @@ -0,0 +1 @@ +* Retry acquiring the deployment lock when the request fails with a transient server error or times out ("request timed out after ... of inactivity"). ([#PR](https://github.com/databricks/cli/pull/PR)) diff --git a/acceptance/bundle/deploy/lock-retry/databricks.yml b/acceptance/bundle/deploy/lock-retry/databricks.yml new file mode 100644 index 00000000000..576d7a9ef25 --- /dev/null +++ b/acceptance/bundle/deploy/lock-retry/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: test-bundle diff --git a/acceptance/bundle/deploy/lock-retry/out.test.toml b/acceptance/bundle/deploy/lock-retry/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/deploy/lock-retry/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/lock-retry/output.txt b/acceptance/bundle/deploy/lock-retry/output.txt new file mode 100644 index 00000000000..598d506d805 --- /dev/null +++ b/acceptance/bundle/deploy/lock-retry/output.txt @@ -0,0 +1,34 @@ + +=== Lock create is applied, but the response is a 504: the retry finds our own lock + +>>> [CLI] bundle deploy +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged +["POST","false"] +["POST","false"] + +=== Lock create is not applied and fails with a 504: the retry creates the lock + +>>> [CLI] bundle deploy +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged +["POST","false"] +["POST","false"] + +=== Lock is held by another deployer: the retry gets a conflict and the error names the holder + +>>> [CLI] bundle deploy +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Error: Failed to acquire deployment lock: deploy lock acquired by someone-else at [TIMESTAMP] +0000 UTC. +Another deployment may be in progress. Use --force-lock to override, but this may +conflict with the other deployment if it is still active +Error: deploy lock acquired by someone-else at [TIMESTAMP] +0000 UTC. +Another deployment may be in progress. Use --force-lock to override, but this may +conflict with the other deployment if it is still active + + +Exit code: 1 diff --git a/acceptance/bundle/deploy/lock-retry/script b/acceptance/bundle/deploy/lock-retry/script new file mode 100644 index 00000000000..60be89b3e08 --- /dev/null +++ b/acceptance/bundle/deploy/lock-retry/script @@ -0,0 +1,20 @@ +# Deploy once so that the state directory exists and the lock create is a single request. +$CLI bundle deploy > /dev/null 2>&1 +print_requests.py > /dev/null + +title "Lock create is applied, but the response is a 504: the retry finds our own lock" +fault.py --after-handler "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 1 +trace $CLI bundle deploy +print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' + +title "Lock create is not applied and fails with a 504: the retry creates the lock" +fault.py "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 1 +trace $CLI bundle deploy +print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' + +title "Lock is held by another deployer: the retry gets a conflict and the error names the holder" +fault.py "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 1 +echo '{"ID":"00000000-0000-0000-0000-000000000000","AcquisitionTime":"2026-01-01T00:00:00Z","IsForced":false,"User":"someone-else"}' > other.lock +$CLI workspace import /Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/deploy.lock --file other.lock --format AUTO > /dev/null +errcode trace $CLI bundle deploy +rm other.lock diff --git a/acceptance/bundle/deploy/lock-retry/test.toml b/acceptance/bundle/deploy/lock-retry/test.toml new file mode 100644 index 00000000000..711ede517b9 --- /dev/null +++ b/acceptance/bundle/deploy/lock-retry/test.toml @@ -0,0 +1,3 @@ +RecordRequests = true +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/libs/locker/locker.go b/libs/locker/locker.go index 857139c79b4..ce5b3c2e007 100644 --- a/libs/locker/locker.go +++ b/libs/locker/locker.go @@ -157,7 +157,18 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { modes = append(modes, filer.OverwriteIfExists) } - err = locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), modes...) + if isForced { + // Overwriting writes are already retried by the retrying filer. + err = locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), modes...) + } else { + // The retrying filer does not repeat writes that do not overwrite. Repeating this + // one is safe because the lock file carries our unique ID: if an attempt that failed + // with a transient error (e.g. an inactivity timeout) did create the file, the retry + // fails with fs.ErrExist and assertLockHeld below confirms that the lock is ours. + _, err = retry.With(ctx, retry.IsTransient, func() (struct{}, error) { + return struct{}{}, locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), modes...) + }) + } if err != nil { // If the write failed because the lock file already exists, don't return // the error and instead fall through to [assertLockHeld] below. From 315bdd07952f0e6055f49afeffa6f22ef575fc1f Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 08:56:35 +0000 Subject: [PATCH 2/9] Link the changelog entry to the PR Co-authored-by: Isaac --- .nextchanges/bundles/lock-retry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/lock-retry.md b/.nextchanges/bundles/lock-retry.md index 3dfe66b8fdc..3ee0c47af7c 100644 --- a/.nextchanges/bundles/lock-retry.md +++ b/.nextchanges/bundles/lock-retry.md @@ -1 +1 @@ -* Retry acquiring the deployment lock when the request fails with a transient server error or times out ("request timed out after ... of inactivity"). ([#PR](https://github.com/databricks/cli/pull/PR)) +* Retry acquiring the deployment lock when the request fails with a transient server error or times out ("request timed out after ... of inactivity"). ([#7594](https://github.com/databricks/cli/pull/7594)) From 2d04b74f22cec7d3ca458aa333e499dbca917158 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 09:04:32 +0000 Subject: [PATCH 3/9] Read the lock back when the last create attempt fails transiently The last attempt may have been applied even though its response was lost; without the read-back, that lock would stay behind with nobody to release it. Also add a lock-timeout test for the inactivity timeout with retries exhausted. Co-authored-by: Isaac --- .../bundle/deploy/lock-retry/output.txt | 12 +++++ acceptance/bundle/deploy/lock-retry/script | 5 ++ .../bundle/deploy/lock-timeout/databricks.yml | 2 + .../bundle/deploy/lock-timeout/out.test.toml | 2 + .../bundle/deploy/lock-timeout/output.txt | 8 ++++ acceptance/bundle/deploy/lock-timeout/script | 4 ++ .../bundle/deploy/lock-timeout/test.toml | 11 +++++ libs/locker/locker.go | 47 ++++++++----------- 8 files changed, 64 insertions(+), 27 deletions(-) create mode 100644 acceptance/bundle/deploy/lock-timeout/databricks.yml create mode 100644 acceptance/bundle/deploy/lock-timeout/out.test.toml create mode 100644 acceptance/bundle/deploy/lock-timeout/output.txt create mode 100644 acceptance/bundle/deploy/lock-timeout/script create mode 100644 acceptance/bundle/deploy/lock-timeout/test.toml diff --git a/acceptance/bundle/deploy/lock-retry/output.txt b/acceptance/bundle/deploy/lock-retry/output.txt index 598d506d805..b2e80bec85e 100644 --- a/acceptance/bundle/deploy/lock-retry/output.txt +++ b/acceptance/bundle/deploy/lock-retry/output.txt @@ -19,6 +19,18 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged ["POST","false"] ["POST","false"] +=== Every attempt returns a 504, but the first one is applied: reading the lock back finds it is ours + +>>> [CLI] bundle deploy +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged +["POST","false"] +["POST","false"] +["POST","false"] + === Lock is held by another deployer: the retry gets a conflict and the error names the holder >>> [CLI] bundle deploy diff --git a/acceptance/bundle/deploy/lock-retry/script b/acceptance/bundle/deploy/lock-retry/script index 60be89b3e08..3ed776d89b3 100644 --- a/acceptance/bundle/deploy/lock-retry/script +++ b/acceptance/bundle/deploy/lock-retry/script @@ -12,6 +12,11 @@ fault.py "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 1 trace $CLI bundle deploy print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' +title "Every attempt returns a 504, but the first one is applied: reading the lock back finds it is ours" +fault.py --after-handler "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 3 +trace $CLI bundle deploy +print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' + title "Lock is held by another deployer: the retry gets a conflict and the error names the holder" fault.py "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 1 echo '{"ID":"00000000-0000-0000-0000-000000000000","AcquisitionTime":"2026-01-01T00:00:00Z","IsForced":false,"User":"someone-else"}' > other.lock diff --git a/acceptance/bundle/deploy/lock-timeout/databricks.yml b/acceptance/bundle/deploy/lock-timeout/databricks.yml new file mode 100644 index 00000000000..576d7a9ef25 --- /dev/null +++ b/acceptance/bundle/deploy/lock-timeout/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: test-bundle diff --git a/acceptance/bundle/deploy/lock-timeout/out.test.toml b/acceptance/bundle/deploy/lock-timeout/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/deploy/lock-timeout/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/lock-timeout/output.txt b/acceptance/bundle/deploy/lock-timeout/output.txt new file mode 100644 index 00000000000..b54178c0ec2 --- /dev/null +++ b/acceptance/bundle/deploy/lock-timeout/output.txt @@ -0,0 +1,8 @@ +Warn: retrying after Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fdeploy.lock?overwrite=false": request timed out after 5s of inactivity +Warn: retrying after Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fdeploy.lock?overwrite=false": request timed out after 5s of inactivity +Error: Failed to acquire deployment lock: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fdeploy.lock?overwrite=false": request timed out after 5s of inactivity +Error: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fdeploy.lock?overwrite=false": request timed out after 5s of inactivity + +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock diff --git a/acceptance/bundle/deploy/lock-timeout/script b/acceptance/bundle/deploy/lock-timeout/script new file mode 100644 index 00000000000..65341c0f0b2 --- /dev/null +++ b/acceptance/bundle/deploy/lock-timeout/script @@ -0,0 +1,4 @@ +musterr $CLI bundle deploy + +# Three attempts: the first one and two retries. +print_requests.py //api/2.0/workspace-files/import-file | jq -r .path | grep 'deploy.lock' diff --git a/acceptance/bundle/deploy/lock-timeout/test.toml b/acceptance/bundle/deploy/lock-timeout/test.toml new file mode 100644 index 00000000000..14ad89aff1f --- /dev/null +++ b/acceptance/bundle/deploy/lock-timeout/test.toml @@ -0,0 +1,11 @@ +Env.DATABRICKS_BUNDLE_HTTP_TIMEOUT_SECONDS = "5" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" +RecordRequests = true + +[[Server]] +# Every lock create attempt outlives the 5s inactivity timeout, so the CLI retries it twice, +# finds no lock when it reads it back, and fails with the timeout. +Delay = "30s" +Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/deploy.lock" +Response.StatusCode = 200 diff --git a/libs/locker/locker.go b/libs/locker/locker.go index ce5b3c2e007..545322ed721 100644 --- a/libs/locker/locker.go +++ b/libs/locker/locker.go @@ -147,40 +147,33 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { return err } - modes := []filer.WriteMode{ - // Always create parent directory if it doesn't yet exist. - filer.CreateParentDirectories, - } - - // Only overwrite lock file if `isForced`. - if isForced { - modes = append(modes, filer.OverwriteIfExists) - } - if isForced { - // Overwriting writes are already retried by the retrying filer. - err = locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), modes...) + // Overwrite writes are already retried by the retrying filer. + err = locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), filer.CreateParentDirectories, filer.OverwriteIfExists) } else { - // The retrying filer does not repeat writes that do not overwrite. Repeating this - // one is safe because the lock file carries our unique ID: if an attempt that failed - // with a transient error (e.g. an inactivity timeout) did create the file, the retry - // fails with fs.ErrExist and assertLockHeld below confirms that the lock is ours. + // The retrying filer does not repeat writes that do not overwrite. Repeating this one is + // safe because the lock file carries our unique ID: if an attempt that failed with a + // transient error did create the file, the retry fails with fs.ErrExist and + // assertLockHeld below tells whether the lock is ours. _, err = retry.With(ctx, retry.IsTransient, func() (struct{}, error) { - return struct{}{}, locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), modes...) + return struct{}{}, locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), filer.CreateParentDirectories) }) } - if err != nil { - // If the write failed because the lock file already exists, don't return - // the error and instead fall through to [assertLockHeld] below. - // This function will return a more descriptive error message that includes - // details about the current holder of the lock. - if !errors.Is(err, fs.ErrExist) { + + switch { + case err == nil || errors.Is(err, fs.ErrExist): + // If the lock file already exists, [assertLockHeld] returns a more descriptive + // error that names the current holder of the lock. + if err := locker.assertLockHeld(ctx); err != nil { return err } - } - - err = locker.assertLockHeld(ctx) - if err != nil { + case retry.IsTransient(ctx, err): + // The last attempt may have been applied even though its response was lost. Read the + // lock back so that such a write does not leave behind a lock nobody releases. + if locker.assertLockHeld(ctx) != nil { + return err + } + default: return err } From e461f6339095ffa03d64bfd6d4ac92dd3dee621e Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 09:34:32 +0000 Subject: [PATCH 4/9] acc: keep the lock-retry workspace path intact on Windows Co-authored-by: Isaac --- acceptance/bundle/deploy/lock-retry/script | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/deploy/lock-retry/script b/acceptance/bundle/deploy/lock-retry/script index 3ed776d89b3..36aac25ef84 100644 --- a/acceptance/bundle/deploy/lock-retry/script +++ b/acceptance/bundle/deploy/lock-retry/script @@ -20,6 +20,7 @@ print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | title "Lock is held by another deployer: the retry gets a conflict and the error names the holder" fault.py "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 1 echo '{"ID":"00000000-0000-0000-0000-000000000000","AcquisitionTime":"2026-01-01T00:00:00Z","IsForced":false,"User":"someone-else"}' > other.lock -$CLI workspace import /Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/deploy.lock --file other.lock --format AUTO > /dev/null -errcode trace $CLI bundle deploy +# MSYS_NO_PATHCONV stops Git Bash on Windows from turning the workspace path into a local one. +MSYS_NO_PATHCONV=1 $CLI workspace import /Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/deploy.lock --file other.lock --format AUTO > /dev/null rm other.lock +errcode trace $CLI bundle deploy From 53eb05665b9a9adfee43339f028de7c8dc8069ba Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Sun, 15 Nov 2026 09:53:17 +0000 Subject: [PATCH 5/9] Also retry the deploy lock create after unexpected EOF A connection cut mid-response leaves the same uncertainty as a timeout, and the lock's unique ID plus the read-back make a retry safe either way. fault.py cannot cut a connection, so this is covered by unit tests in libs/locker. Co-authored-by: Isaac --- .nextchanges/bundles/lock-retry.md | 2 +- libs/locker/locker.go | 7 ++- libs/locker/locker_test.go | 88 ++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 libs/locker/locker_test.go diff --git a/.nextchanges/bundles/lock-retry.md b/.nextchanges/bundles/lock-retry.md index 3ee0c47af7c..6c8c60103d9 100644 --- a/.nextchanges/bundles/lock-retry.md +++ b/.nextchanges/bundles/lock-retry.md @@ -1 +1 @@ -* Retry acquiring the deployment lock when the request fails with a transient server error or times out ("request timed out after ... of inactivity"). ([#7594](https://github.com/databricks/cli/pull/7594)) +* Retry acquiring the deployment lock when the request fails with a transient server error, times out ("request timed out after ... of inactivity") or loses its connection ("unexpected EOF"). ([#7594](https://github.com/databricks/cli/pull/7594)) diff --git a/libs/locker/locker.go b/libs/locker/locker.go index 545322ed721..50c2bd8dd7e 100644 --- a/libs/locker/locker.go +++ b/libs/locker/locker.go @@ -154,8 +154,9 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { // The retrying filer does not repeat writes that do not overwrite. Repeating this one is // safe because the lock file carries our unique ID: if an attempt that failed with a // transient error did create the file, the retry fails with fs.ErrExist and - // assertLockHeld below tells whether the lock is ours. - _, err = retry.With(ctx, retry.IsTransient, func() (struct{}, error) { + // assertLockHeld below tells whether the lock is ours. That holds for a connection cut + // mid-response ("unexpected EOF") too, so this uses the wider check meant for reads. + _, err = retry.With(ctx, retry.IsTransientRead, func() (struct{}, error) { return struct{}{}, locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), filer.CreateParentDirectories) }) } @@ -167,7 +168,7 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { if err := locker.assertLockHeld(ctx); err != nil { return err } - case retry.IsTransient(ctx, err): + case retry.IsTransientRead(ctx, err): // The last attempt may have been applied even though its response was lost. Read the // lock back so that such a write does not leave behind a lock nobody releases. if locker.assertLockHeld(ctx) != nil { diff --git a/libs/locker/locker_test.go b/libs/locker/locker_test.go new file mode 100644 index 00000000000..c5bb8068840 --- /dev/null +++ b/libs/locker/locker_test.go @@ -0,0 +1,88 @@ +package locker + +import ( + "context" + "io" + "net/url" + "testing" + "uuid" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// eofFiler fails the first `failures` writes with the error net/http returns when the connection +// is cut mid-response. The write numbered appliedOn (1-based, 0 for none) still writes the file +// before failing, like a request the server applied before the connection was cut. +type eofFiler struct { + filer.Filer + failures int + appliedOn int + writes int +} + +func (f *eofFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...filer.WriteMode) error { + f.writes++ + if f.writes > f.failures { + return f.Filer.Write(ctx, path, reader, mode...) + } + if f.writes == f.appliedOn { + if err := f.Filer.Write(ctx, path, reader, mode...); err != nil { + return err + } + } + return &url.Error{Op: "Post", URL: "https://example.com/" + path, Err: io.ErrUnexpectedEOF} +} + +func newTestLocker(t *testing.T, f filer.Filer) *Locker { + return &Locker{ + filer: f, + TargetDir: t.TempDir(), + State: &LockState{ID: uuid.New(), User: "tester"}, + } +} + +func TestLockRetriesUnexpectedEOF(t *testing.T) { + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + + tests := []struct { + name string + failures int + appliedOn int + }{ + // The retry creates the lock. + {name: "not applied", failures: 1, appliedOn: 0}, + // The retry fails with "already exists" and the read-back finds our lock. + {name: "applied", failures: 1, appliedOn: 1}, + // Retries run out, but the last attempt was applied: the read-back finds our lock. + {name: "last attempt applied", failures: 3, appliedOn: 3}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + local, err := filer.NewLocalClient(t.TempDir()) + require.NoError(t, err) + f := &eofFiler{Filer: local, failures: tc.failures, appliedOn: tc.appliedOn} + locker := newTestLocker(t, f) + + require.NoError(t, locker.Lock(ctx, false)) + assert.True(t, locker.Active) + assert.Equal(t, min(tc.failures+1, 3), f.writes) + }) + } +} + +func TestLockFailsWhenEveryAttemptHitsUnexpectedEOF(t *testing.T) { + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + local, err := filer.NewLocalClient(t.TempDir()) + require.NoError(t, err) + f := &eofFiler{Filer: local, failures: 3} + locker := newTestLocker(t, f) + + err = locker.Lock(ctx, false) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + assert.False(t, locker.Active) + assert.Equal(t, 3, f.writes) +} From aafff5e2080127ed95e6c241d894b09e2487d652 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Sun, 15 Nov 2026 10:20:45 +0000 Subject: [PATCH 6/9] Name the lock holder after a transient lock create, test forced locks Address review: when the last create attempt fails transiently and the read-back finds another holder's lock, report the holder rather than the transient error. Add lock-retry cases for that and for --force-lock, and shorten the retry pauses in the telemetry test that stubs import-file with a 500. Co-authored-by: Isaac --- .../bundle/deploy/lock-retry/output.txt | 30 +++++++++++++++++++ acceptance/bundle/deploy/lock-retry/script | 11 +++++++ .../deploy-failed-engine-unknown/test.toml | 3 ++ libs/locker/locker.go | 15 ++++++++-- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/deploy/lock-retry/output.txt b/acceptance/bundle/deploy/lock-retry/output.txt index b2e80bec85e..a0893754e71 100644 --- a/acceptance/bundle/deploy/lock-retry/output.txt +++ b/acceptance/bundle/deploy/lock-retry/output.txt @@ -44,3 +44,33 @@ conflict with the other deployment if it is still active Exit code: 1 + +=== Lock is held by another deployer and every attempt returns a 504: reading the lock back names the holder + +>>> [CLI] bundle deploy +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Error: Failed to acquire deployment lock: deploy lock acquired by someone-else at [TIMESTAMP] +0000 UTC. +Another deployment may be in progress. Use --force-lock to override, but this may +conflict with the other deployment if it is still active +Error: deploy lock acquired by someone-else at [TIMESTAMP] +0000 UTC. +Another deployment may be in progress. Use --force-lock to override, but this may +conflict with the other deployment if it is still active + + +Exit code: 1 +["POST","false"] +["POST","false"] +["POST","false"] + +=== Forced lock, every overwrite attempt returns a 504 but is applied: reading the lock back finds it is ours + +>>> [CLI] bundle deploy --force-lock +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged +["POST","true"] +["POST","true"] +["POST","true"] diff --git a/acceptance/bundle/deploy/lock-retry/script b/acceptance/bundle/deploy/lock-retry/script index 36aac25ef84..02fff778afb 100644 --- a/acceptance/bundle/deploy/lock-retry/script +++ b/acceptance/bundle/deploy/lock-retry/script @@ -24,3 +24,14 @@ echo '{"ID":"00000000-0000-0000-0000-000000000000","AcquisitionTime":"2026-01-01 MSYS_NO_PATHCONV=1 $CLI workspace import /Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/deploy.lock --file other.lock --format AUTO > /dev/null rm other.lock errcode trace $CLI bundle deploy +print_requests.py > /dev/null + +title "Lock is held by another deployer and every attempt returns a 504: reading the lock back names the holder" +fault.py --after-handler "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 3 +errcode trace $CLI bundle deploy +print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' + +title "Forced lock, every overwrite attempt returns a 504 but is applied: reading the lock back finds it is ours" +fault.py --after-handler "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 3 +trace $CLI bundle deploy --force-lock +print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml index 86d366ba8c2..1592114326e 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml @@ -11,6 +11,9 @@ # diverges across the engine matrix only in state_engine; it is captured per-engine. Cloud = false +# The lock create fails with a 500, which the CLI retries; keep the pauses between attempts short. +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" + [[Server]] Pattern = "POST /api/2.0/workspace-files/import-file/{path...}" Response.StatusCode = 500 diff --git a/libs/locker/locker.go b/libs/locker/locker.go index 50c2bd8dd7e..e7330b45242 100644 --- a/libs/locker/locker.go +++ b/libs/locker/locker.go @@ -105,6 +105,11 @@ func (locker *Locker) assertLockHeld(ctx context.Context) error { if err != nil { return err } + return locker.checkHolder(activeLockState) +} + +// checkHolder returns a descriptive error naming the holder if activeLockState is not ours. +func (locker *Locker) checkHolder(activeLockState *LockState) error { if activeLockState.ID != locker.State.ID && !activeLockState.IsForced { return fmt.Errorf("deploy lock acquired by %s at %v.\n"+ "Another deployment may be in progress. Use --force-lock to override, but this may\n"+ @@ -170,8 +175,14 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { } case retry.IsTransientRead(ctx, err): // The last attempt may have been applied even though its response was lost. Read the - // lock back so that such a write does not leave behind a lock nobody releases. - if locker.assertLockHeld(ctx) != nil { + // lock back so that such a write does not leave behind a lock nobody releases. This + // covers forced locks too, whose overwrite the retrying filer retried on its own. + activeLockState, readErr := locker.GetActiveLockState(ctx) + if readErr != nil { + // No lock, or the read failed too: the write error says more. + return err + } + if err := locker.checkHolder(activeLockState); err != nil { return err } default: From bd7f041614fd5eddf11c44474e900fb80349d584 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Sun, 15 Nov 2026 11:52:45 +0000 Subject: [PATCH 7/9] Keep the write error for a forced lock that fails transiently Address review: with --force-lock, an overwrite that failed transiently and was not applied returned the holder error, which suggests --force-lock. Return the write error instead. Co-authored-by: Isaac --- .../bundle/deploy/lock-retry/output.txt | 19 +++++++++++++++++++ acceptance/bundle/deploy/lock-retry/script | 9 +++++++++ libs/locker/locker.go | 8 ++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/deploy/lock-retry/output.txt b/acceptance/bundle/deploy/lock-retry/output.txt index a0893754e71..4fb0e7377d7 100644 --- a/acceptance/bundle/deploy/lock-retry/output.txt +++ b/acceptance/bundle/deploy/lock-retry/output.txt @@ -74,3 +74,22 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged ["POST","true"] ["POST","true"] ["POST","true"] + +=== Forced lock over another deployer's lock, every overwrite attempt returns a 504 and is not applied: the error is the 504 + +>>> [CLI] bundle deploy --force-lock +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Error: Failed to acquire deployment lock: Fault injected by test. +Error: Fault injected by test. (504 INJECTED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fdeploy.lock?overwrite=true +HTTP Status: 504 Gateway Timeout +API error_code: INJECTED +API message: Fault injected by test. + + +Exit code: 1 +["POST","true"] +["POST","true"] +["POST","true"] diff --git a/acceptance/bundle/deploy/lock-retry/script b/acceptance/bundle/deploy/lock-retry/script index 02fff778afb..24ccde07227 100644 --- a/acceptance/bundle/deploy/lock-retry/script +++ b/acceptance/bundle/deploy/lock-retry/script @@ -35,3 +35,12 @@ title "Forced lock, every overwrite attempt returns a 504 but is applied: readin fault.py --after-handler "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 3 trace $CLI bundle deploy --force-lock print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' + +title "Forced lock over another deployer's lock, every overwrite attempt returns a 504 and is not applied: the error is the 504" +echo '{"ID":"00000000-0000-0000-0000-000000000000","AcquisitionTime":"2026-01-01T00:00:00Z","IsForced":false,"User":"someone-else"}' > other.lock +MSYS_NO_PATHCONV=1 $CLI workspace import /Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/deploy.lock --file other.lock --format AUTO --overwrite > /dev/null +rm other.lock +print_requests.py > /dev/null +fault.py "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 3 +errcode trace $CLI bundle deploy --force-lock +print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' diff --git a/libs/locker/locker.go b/libs/locker/locker.go index e7330b45242..816aa1daa99 100644 --- a/libs/locker/locker.go +++ b/libs/locker/locker.go @@ -182,8 +182,12 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { // No lock, or the read failed too: the write error says more. return err } - if err := locker.checkHolder(activeLockState); err != nil { - return err + if holderErr := locker.checkHolder(activeLockState); holderErr != nil { + if isForced { + // The holder error suggests --force-lock, which is already in use. + return err + } + return holderErr } default: return err From b51ca4d4b86ac2e35a49ce8803571276aaad79af Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Sun, 15 Nov 2026 12:36:27 +0000 Subject: [PATCH 8/9] Say which errors the forced lock write is retried on Co-authored-by: Isaac --- .nextchanges/bundles/lock-retry.md | 2 +- libs/locker/locker.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.nextchanges/bundles/lock-retry.md b/.nextchanges/bundles/lock-retry.md index 6c8c60103d9..901f64ffb0f 100644 --- a/.nextchanges/bundles/lock-retry.md +++ b/.nextchanges/bundles/lock-retry.md @@ -1 +1 @@ -* Retry acquiring the deployment lock when the request fails with a transient server error, times out ("request timed out after ... of inactivity") or loses its connection ("unexpected EOF"). ([#7594](https://github.com/databricks/cli/pull/7594)) +* Retry creating the deployment lock (without `--force-lock`) when the request fails with a transient server error, times out ("request timed out after ... of inactivity") or loses its connection ("unexpected EOF"). ([#7594](https://github.com/databricks/cli/pull/7594)) diff --git a/libs/locker/locker.go b/libs/locker/locker.go index 816aa1daa99..95b9f86a233 100644 --- a/libs/locker/locker.go +++ b/libs/locker/locker.go @@ -153,7 +153,8 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { } if isForced { - // Overwrite writes are already retried by the retrying filer. + // The retrying filer already retries overwrite writes, on the errors [retry.IsTransient] + // covers; an "unexpected EOF" is not retried there, only read back below. err = locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), filer.CreateParentDirectories, filer.OverwriteIfExists) } else { // The retrying filer does not repeat writes that do not overwrite. Repeating this one is @@ -176,7 +177,7 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { case retry.IsTransientRead(ctx, err): // The last attempt may have been applied even though its response was lost. Read the // lock back so that such a write does not leave behind a lock nobody releases. This - // covers forced locks too, whose overwrite the retrying filer retried on its own. + // covers forced locks too. activeLockState, readErr := locker.GetActiveLockState(ctx) if readErr != nil { // No lock, or the read failed too: the write error says more. From f440de21fa3e70d6f95233cd07ce55c5877a9934 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Sun, 15 Nov 2026 14:09:00 +0000 Subject: [PATCH 9/9] acc: commit the other deployer's lock file in lock-retry Co-authored-by: Isaac --- acceptance/bundle/deploy/lock-retry/other.lock | 1 + acceptance/bundle/deploy/lock-retry/script | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) create mode 100644 acceptance/bundle/deploy/lock-retry/other.lock diff --git a/acceptance/bundle/deploy/lock-retry/other.lock b/acceptance/bundle/deploy/lock-retry/other.lock new file mode 100644 index 00000000000..8a7caa3642c --- /dev/null +++ b/acceptance/bundle/deploy/lock-retry/other.lock @@ -0,0 +1 @@ +{"ID":"00000000-0000-0000-0000-000000000000","AcquisitionTime":"2026-01-01T00:00:00Z","IsForced":false,"User":"someone-else"} diff --git a/acceptance/bundle/deploy/lock-retry/script b/acceptance/bundle/deploy/lock-retry/script index 24ccde07227..eeb95425736 100644 --- a/acceptance/bundle/deploy/lock-retry/script +++ b/acceptance/bundle/deploy/lock-retry/script @@ -19,10 +19,9 @@ print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | title "Lock is held by another deployer: the retry gets a conflict and the error names the holder" fault.py "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 1 -echo '{"ID":"00000000-0000-0000-0000-000000000000","AcquisitionTime":"2026-01-01T00:00:00Z","IsForced":false,"User":"someone-else"}' > other.lock +# other.lock is a lock held by another deployer. # MSYS_NO_PATHCONV stops Git Bash on Windows from turning the workspace path into a local one. MSYS_NO_PATHCONV=1 $CLI workspace import /Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/deploy.lock --file other.lock --format AUTO > /dev/null -rm other.lock errcode trace $CLI bundle deploy print_requests.py > /dev/null @@ -37,9 +36,7 @@ trace $CLI bundle deploy --force-lock print_requests.py //api/2.0/workspace-files/import-file | jq -c 'select(.path | contains("deploy.lock")) | [.method, .q.overwrite]' title "Forced lock over another deployer's lock, every overwrite attempt returns a 504 and is not applied: the error is the 504" -echo '{"ID":"00000000-0000-0000-0000-000000000000","AcquisitionTime":"2026-01-01T00:00:00Z","IsForced":false,"User":"someone-else"}' > other.lock MSYS_NO_PATHCONV=1 $CLI workspace import /Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/deploy.lock --file other.lock --format AUTO --overwrite > /dev/null -rm other.lock print_requests.py > /dev/null fault.py "POST /api/2.0/workspace-files/import-file/**/deploy.lock" 504 0 3 errcode trace $CLI bundle deploy --force-lock