From 33755a977bc15d4adcebdf1fae75afbab35b5968 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 11 Nov 2026 23:03:02 +0000 Subject: [PATCH 1/2] Retry workspace requests after the SDK's inactivity timeout The SDK gives up on a request when the server sends nothing for HTTPTimeout ("request timed out after 1m30s of inactivity") and never retries it. libs/retry already retries 5xx and 408 for overwrite writes, reads and deletes; treat this timeout the same way. Print non-API retry reasons as "retrying after " so the warning says what happened and acceptance tests suppress it like the other retry warnings. Co-authored-by: Isaac --- acceptance/bundle/upload/timeout/output.txt | 5 +++++ acceptance/bundle/upload/timeout/script | 3 +++ acceptance/bundle/upload/timeout/test.toml | 6 +++++- libs/retry/retry.go | 22 +++++++++++++++++++-- libs/retry/retry_test.go | 10 +++++++++- 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/upload/timeout/output.txt b/acceptance/bundle/upload/timeout/output.txt index c314a247707..26b3b8b8bae 100644 --- a/acceptance/bundle/upload/timeout/output.txt +++ b/acceptance/bundle/upload/timeout/output.txt @@ -1,3 +1,8 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: retrying after Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity +Warn: retrying after Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity Error: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt diff --git a/acceptance/bundle/upload/timeout/script b/acceptance/bundle/upload/timeout/script index c1a222958e1..2ae9403fd0a 100644 --- a/acceptance/bundle/upload/timeout/script +++ b/acceptance/bundle/upload/timeout/script @@ -1 +1,4 @@ musterr $CLI bundle deploy + +# Three attempts: the first one and two retries. +print_requests.py //api/2.0/workspace-files/import-file | jq -r .path | grep 'file_to_upload.txt' diff --git a/acceptance/bundle/upload/timeout/test.toml b/acceptance/bundle/upload/timeout/test.toml index d6564ef9659..02682344684 100644 --- a/acceptance/bundle/upload/timeout/test.toml +++ b/acceptance/bundle/upload/timeout/test.toml @@ -1,7 +1,11 @@ Env.DATABRICKS_BUNDLE_HTTP_TIMEOUT_SECONDS = "5" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" +RecordRequests = true [[Server]] -# CLI aborts after a single attempt when the HTTP timeout fires. +# Every upload attempt outlives the 5s inactivity timeout. Uploads overwrite, so the CLI retries +# them twice before giving up. Delay = "30s" Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" Response.StatusCode = 200 diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 86dd9ec232c..5c5e650a5d5 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -38,10 +38,14 @@ func interval(ctx context.Context) time.Duration { return time.Duration(ms) * time.Millisecond } -// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry. +// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry, and for +// the SDK's client-side inactivity timeout. // The SDK retries 504s matching its allTransientErrors patterns (e.g. "deadline exceeded"); // this covers the remaining 504s like TEMPORARILY_UNAVAILABLE. func IsTransient(ctx context.Context, err error) bool { + if isInactivityTimeout(err) { + return true + } apiErr, ok := errors.AsType[*apierr.APIError](err) if !ok { return false @@ -53,6 +57,19 @@ func IsTransient(ctx context.Context, err error) bool { return apiErr.StatusCode >= 500 || apiErr.StatusCode == http.StatusRequestTimeout } +// isInactivityTimeout reports whether the SDK gave up on a request because the server sent nothing +// for HTTPTimeout ("request timed out after 1m30s of inactivity"). The SDK does not retry it. The +// workspace files API stalls like this for a few minutes at a time; the server may or may not have +// applied the request, which is the same uncertainty as a 504. +func isInactivityTimeout(err error) bool { + urlErr, ok := errors.AsType[*url.Error](err) + if !ok { + return false + } + msg := urlErr.Err.Error() + return strings.HasPrefix(msg, "request timed out after ") && strings.HasSuffix(msg, " of inactivity") +} + // transientReadMessages are the transport errors, matched by message, that are worth retrying on a read: // - REFUSED_STREAM, the HTTP/2 refusal in issue #6472, where the server refused the stream before // processing the request. @@ -97,7 +114,8 @@ func With[T any](ctx context.Context, check func(context.Context, error) bool, f // retryMessage describes the error that triggered a retry. func retryMessage(err error) string { - msg := "retrying" + // The "retrying after " prefix is what acceptance tests suppress (SuppressLines.warn_retry). + msg := "retrying after " + err.Error() if apiErr, ok := errors.AsType[*apierr.APIError](err); ok { endpoint := "" if rw := apiErr.ResponseWrapper; rw != nil && rw.Response != nil && rw.Response.Request != nil { diff --git a/libs/retry/retry_test.go b/libs/retry/retry_test.go index 0568fc82b09..396ad9e1803 100644 --- a/libs/retry/retry_test.go +++ b/libs/retry/retry_test.go @@ -26,6 +26,14 @@ func TestIsTransient(t *testing.T) { assert.False(t, IsTransient(ctx, &apierr.APIError{StatusCode: http.StatusGatewayTimeout, Message: "deadline exceeded"})) assert.False(t, IsTransient(ctx, &apierr.APIError{StatusCode: http.StatusForbidden})) assert.False(t, IsTransient(ctx, errors.New("boom"))) + + // The SDK's inactivity timeout, which it does not retry itself. + inactivity := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: errors.New("request timed out after 1m30s of inactivity")} + assert.True(t, IsTransient(ctx, inactivity)) + assert.True(t, IsTransient(ctx, fmt.Errorf("failed to upload: %w", inactivity))) + assert.True(t, IsTransientRead(ctx, inactivity)) + cancelled := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: context.Canceled} + assert.False(t, IsTransient(ctx, cancelled)) } func TestIsTransientRead(t *testing.T) { @@ -115,7 +123,7 @@ func TestWithStopsOnContextCancel(t *testing.T) { func TestRetryMessage(t *testing.T) { apiErr := &apierr.APIError{StatusCode: http.StatusGatewayTimeout, Message: "boom"} - assert.Equal(t, "retrying", retryMessage(errors.New("x"))) + assert.Equal(t, "retrying after x", retryMessage(errors.New("x"))) assert.Equal(t, "retrying after 504 Gateway Timeout", retryMessage(apiErr)) assert.Equal(t, "retrying after 504 Gateway Timeout: failed to upload /a/b: boom", retryMessage(fmt.Errorf("failed to upload /a/b: %w", apiErr))) } From e79c1d9dccce1a707a2311913f08b9d71fced62f Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 11 Nov 2026 23:23:46 +0000 Subject: [PATCH 2/2] Add changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/7521.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7521.md diff --git a/.nextchanges/bundles/7521.md b/.nextchanges/bundles/7521.md new file mode 100644 index 00000000000..0d5a9f56f75 --- /dev/null +++ b/.nextchanges/bundles/7521.md @@ -0,0 +1 @@ +* Retry bundle file uploads, reads and deletes when the server does not respond within the HTTP timeout ("request timed out after ... of inactivity"). ([#7521](https://github.com/databricks/cli/pull/7521))