Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7521.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Retry bundle file uploads, reads and deletes when the server does not respond within the HTTP timeout ("request timed out after ... of inactivity"). ([#7521](https://github.com/databricks/cli/pull/7521))
5 changes: 5 additions & 0 deletions acceptance/bundle/upload/timeout/output.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: retrying after Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity
Warn: retrying after Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity
Error: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity

/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt
/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt
/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt
3 changes: 3 additions & 0 deletions acceptance/bundle/upload/timeout/script
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
musterr $CLI bundle deploy

# Three attempts: the first one and two retries.
print_requests.py //api/2.0/workspace-files/import-file | jq -r .path | grep 'file_to_upload.txt'
6 changes: 5 additions & 1 deletion acceptance/bundle/upload/timeout/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
Env.DATABRICKS_BUNDLE_HTTP_TIMEOUT_SECONDS = "5"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
RecordRequests = true

[[Server]]
# CLI aborts after a single attempt when the HTTP timeout fires.
# Every upload attempt outlives the 5s inactivity timeout. Uploads overwrite, so the CLI retries
# them twice before giving up.
Delay = "30s"
Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt"
Response.StatusCode = 200
22 changes: 20 additions & 2 deletions libs/retry/retry.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,14 @@ func interval(ctx context.Context) time.Duration {
return time.Duration(ms) * time.Millisecond
}

// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry.
// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry, and for
// the SDK's client-side inactivity timeout.
// The SDK retries 504s matching its allTransientErrors patterns (e.g. "deadline exceeded");
// this covers the remaining 504s like TEMPORARILY_UNAVAILABLE.
func IsTransient(ctx context.Context, err error) bool {
if isInactivityTimeout(err) {
return true
}
apiErr, ok := errors.AsType[*apierr.APIError](err)
if !ok {
return false
Expand All @@ -53,6 +57,19 @@ func IsTransient(ctx context.Context, err error) bool {
return apiErr.StatusCode >= 500 || apiErr.StatusCode == http.StatusRequestTimeout
}

// isInactivityTimeout reports whether the SDK gave up on a request because the server sent nothing
// for HTTPTimeout ("request timed out after 1m30s of inactivity"). The SDK does not retry it. The
// workspace files API stalls like this for a few minutes at a time; the server may or may not have
// applied the request, which is the same uncertainty as a 504.
func isInactivityTimeout(err error) bool {
urlErr, ok := errors.AsType[*url.Error](err)
if !ok {
return false
}
msg := urlErr.Err.Error()
return strings.HasPrefix(msg, "request timed out after ") && strings.HasSuffix(msg, " of inactivity")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This matches the SDK's message because httpclient/api_client.go builds it with fmt.Errorf and exports no sentinel; can the comment say so, or should the SDK export one?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed: the comment should say so. The SDK (v0.197.0) builds this error with fmt.Errorf("request timed out after %s of inactivity", ...) in httpclient/api_client.go and exports no sentinel or type, so the CLI has to match the message. #7541 adds that note to the comment. An exported sentinel in the SDK would be the cleaner fix; I've left that for the SDK side.

Written by the cli-tests-fix AI agent (harness_auto).

}

// transientReadMessages are the transport errors, matched by message, that are worth retrying on a read:
// - REFUSED_STREAM, the HTTP/2 refusal in issue #6472, where the server refused the stream before
// processing the request.
Expand Down Expand Up @@ -97,7 +114,8 @@ func With[T any](ctx context.Context, check func(context.Context, error) bool, f

// retryMessage describes the error that triggered a retry.
func retryMessage(err error) string {
msg := "retrying"
// The "retrying after " prefix is what acceptance tests suppress (SuppressLines.warn_retry).
msg := "retrying after " + err.Error()
if apiErr, ok := errors.AsType[*apierr.APIError](err); ok {
endpoint := ""
if rw := apiErr.ResponseWrapper; rw != nil && rw.Response != nil && rw.Response.Request != nil {
Expand Down
10 changes: 9 additions & 1 deletion libs/retry/retry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,14 @@ func TestIsTransient(t *testing.T) {
assert.False(t, IsTransient(ctx, &apierr.APIError{StatusCode: http.StatusGatewayTimeout, Message: "deadline exceeded"}))
assert.False(t, IsTransient(ctx, &apierr.APIError{StatusCode: http.StatusForbidden}))
assert.False(t, IsTransient(ctx, errors.New("boom")))

// The SDK's inactivity timeout, which it does not retry itself.
inactivity := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: errors.New("request timed out after 1m30s of inactivity")}
assert.True(t, IsTransient(ctx, inactivity))
assert.True(t, IsTransient(ctx, fmt.Errorf("failed to upload: %w", inactivity)))
assert.True(t, IsTransientRead(ctx, inactivity))
cancelled := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: context.Canceled}
assert.False(t, IsTransient(ctx, cancelled))
}

func TestIsTransientRead(t *testing.T) {
Expand Down Expand Up @@ -115,7 +123,7 @@ func TestWithStopsOnContextCancel(t *testing.T) {

func TestRetryMessage(t *testing.T) {
apiErr := &apierr.APIError{StatusCode: http.StatusGatewayTimeout, Message: "boom"}
assert.Equal(t, "retrying", retryMessage(errors.New("x")))
assert.Equal(t, "retrying after x", retryMessage(errors.New("x")))
assert.Equal(t, "retrying after 504 Gateway Timeout", retryMessage(apiErr))
assert.Equal(t, "retrying after 504 Gateway Timeout: failed to upload /a/b: boom", retryMessage(fmt.Errorf("failed to upload /a/b: %w", apiErr)))
}
Loading