From 3320221fc6003787f464f48015f702e26090801c Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 13:54:23 +0000 Subject: [PATCH 1/4] filer: retry throttled Files API calls on UC Volumes Since the filer moved to the sdk-go files/v2 client, Files API calls are no longer retried: files/v2 retries nothing unless the caller passes a retrier. The backend reports throttled cloud storage ("AWS S3 is throttling requests; try again later.") as 429 RESOURCE_EXHAUSTED and expects the caller to retry. Retry 429 and 503 on every files/v2 call the filer makes. The single-shot upload cannot take a retrier, so the filer retries it itself, rewinding a seekable reader to its starting offset before each attempt. Co-authored-by: Isaac --- libs/filer/files_client.go | 95 +++++++++++++++++++++---- libs/filer/files_client_test.go | 121 ++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 13 deletions(-) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 1638acb73bd..cfb741974dd 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -16,7 +16,9 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/sdk-go/core/apierr" + "github.com/databricks/sdk-go/core/ops" files "github.com/databricks/sdk-go/files/v2" + "github.com/databricks/sdk-go/options/call" "github.com/databricks/sdk-go/options/client" "golang.org/x/sync/errgroup" ) @@ -37,6 +39,35 @@ func httpStatus(err error) int { return -1 } +// filesRetryStatusCodes are the HTTP statuses on which Files API calls are +// retried. Throttled cloud storage comes back as 429 RESOURCE_EXHAUSTED (e.g. +// "AWS S3 is throttling requests; try again later.") and an unavailable backend +// as 503; both ask the caller to retry. files/v2 retries nothing unless the +// caller passes a retrier, so the filer supplies one to keep the retries the +// databricks-sdk-go client used to do for it. +var filesRetryStatusCodes = []int{http.StatusTooManyRequests, http.StatusServiceUnavailable} + +// filesMaxRetries bounds the retries of one call. Non-streaming files/v2 calls +// also have a 5 minute SDK timeout, but the single-shot upload retried by the +// filer itself does not. +const filesMaxRetries = 10 + +// filesRetryBackoff is the default delay policy between retries. +var filesRetryBackoff = ops.BackoffPolicy{Initial: time.Second, Maximum: 10 * time.Second} + +type filesRetrier struct { + backoff ops.BackoffPolicy + retries int +} + +func (r *filesRetrier) IsRetriable(err error) (time.Duration, bool) { + if r.retries >= filesMaxRetries || !slices.Contains(filesRetryStatusCodes, httpStatus(err)) { + return 0, false + } + r.retries++ + return r.backoff.Delay(), true +} + // As of 19th Feb 2024, the Files API backend has a rate limit of 10 concurrent // requests and 100 QPS. We limit the number of concurrent requests to 5 to // avoid hitting the rate limit. @@ -152,6 +183,18 @@ type FilesClient struct { // (e.g. fs cp -r) draw from one bounded budget and one connection pool. limiter files.Limiter transferClient *http.Client + + // retryBackoff is copied into a fresh retrier for every call. + retryBackoff ops.BackoffPolicy +} + +func (w *FilesClient) newRetrier() ops.Retrier { + return &filesRetrier{backoff: w.retryBackoff} +} + +// retry is the call option that retries a files/v2 call on throttling. +func (w *FilesClient) retry() call.Option { + return call.WithRetrier(w.newRetrier) } func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root string) (Filer, error) { @@ -167,6 +210,7 @@ func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root str limiter: files.NewLimiter(uploadConcurrency), transferClient: newTransferClient(uploadConcurrency), + retryBackoff: filesRetryBackoff, }, nil } @@ -213,7 +257,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // Check that target path exists if CreateParentDirectories mode is not set if !slices.Contains(mode, CreateParentDirectories) { dir := path.Dir(absPath) - _, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}) + _, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}, w.retry()) if err != nil { // This API returns a 404 if the directory doesn't exist. if httpStatus(err) == http.StatusNotFound { @@ -247,11 +291,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, return mapUploadError(uerr, absPath) } - _, err = w.client.UploadFile(ctx, files.UploadFileRequest{ - FilePath: &absPath, - Contents: io.NopCloser(reader), - Overwrite: &overwrite, - }) + err = w.uploadFile(ctx, absPath, reader, overwrite) if err == nil { return nil } @@ -263,6 +303,35 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, return err } +// uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this +// call because it cannot replay a streamed body, so a seekable reader is rewound +// to its starting offset and retried here; other readers get one attempt. +func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error { + upload := func(ctx context.Context) error { + _, err := w.client.UploadFile(ctx, files.UploadFileRequest{ + FilePath: &absPath, + Contents: io.NopCloser(reader), + Overwrite: &overwrite, + }) + return err + } + + seeker, ok := reader.(io.Seeker) + if !ok { + return upload(ctx) + } + start, err := seeker.Seek(0, io.SeekCurrent) + if err != nil { + return upload(ctx) + } + return ops.Execute(ctx, func(ctx context.Context) error { + if _, err := seeker.Seek(start, io.SeekStart); err != nil { + return err + } + return upload(ctx) + }, ops.WithRetrier(w.newRetrier)) +} + // isSeekable reports whether r can be seeked, without moving it: a no-op // Seek(0, io.SeekCurrent) returns the current offset for a working seeker and // errors for a broken one. Probing this way (rather than seeking to the end and @@ -295,7 +364,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err return nil, err } - resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}) + resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}, w.retry()) // Return early on success. if err == nil { @@ -327,7 +396,7 @@ func (w *FilesClient) deleteFile(ctx context.Context, name string) error { return cannotDeleteRootError{} } - _, err = w.client.DeleteFile(ctx, files.DeleteFileRequest{FilePath: &absPath}) + _, err = w.client.DeleteFile(ctx, files.DeleteFileRequest{FilePath: &absPath}, w.retry()) // Return early on success. if err == nil { @@ -353,7 +422,7 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error { return cannotDeleteRootError{} } - _, err = w.client.DeleteDirectory(ctx, files.DeleteDirectoryRequest{DirectoryPath: &absPath}) + _, err = w.client.DeleteDirectory(ctx, files.DeleteDirectoryRequest{DirectoryPath: &absPath}, w.retry()) // Return early on success. if err == nil { @@ -477,7 +546,7 @@ func (w *FilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, var entries []fs.DirEntry for entry, err := range w.client.ListDirectoryContentsIter(ctx, files.ListDirectoryContentsRequest{ DirectoryPath: &absPath, - }) { + }, w.retry()) { if err != nil { // This API returns a 404 if the specified path does not exist. if httpStatus(err) == http.StatusNotFound { @@ -513,7 +582,7 @@ func (w *FilesClient) Mkdir(ctx context.Context, name string) error { return err } - _, err = w.client.CreateDirectory(ctx, files.CreateDirectoryRequest{DirectoryPath: &absPath}) + _, err = w.client.CreateDirectory(ctx, files.CreateDirectoryRequest{DirectoryPath: &absPath}, w.retry()) // This API returns a 409 when a file already exists at the path (the create // is not idempotent over a file). @@ -531,7 +600,7 @@ func (w *FilesClient) statFile(ctx context.Context, name string) (fs.FileInfo, e return nil, err } - resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}) + resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}, w.retry()) // If the HEAD requests succeeds, the file exists. if err == nil { @@ -557,7 +626,7 @@ func (w *FilesClient) statDir(ctx context.Context, name string) (fs.FileInfo, er return nil, err } - _, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}) + _, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}, w.retry()) // If the HEAD requests succeeds, the directory exists. if err == nil { diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 3579c5464c5..0d621d3e542 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -8,13 +8,17 @@ import ( "io/fs" "os" "path/filepath" + "strings" + "sync" "testing" + "time" cliauth "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/sdk-go/core/ops" files "github.com/databricks/sdk-go/files/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -216,3 +220,120 @@ func TestMapUploadError(t *testing.T) { t.Errorf("mapUploadError(other) = %v, want it unchanged", got) } } + +// throttledResponse is how the Files API reports throttled cloud storage. +var throttledResponse = testserver.Response{ + StatusCode: 429, + Body: map[string]any{ + "error_code": "RESOURCE_EXHAUSTED", + "message": "AWS S3 is throttling requests; try again later.", + }, +} + +// throttled wraps a handler so that its first n requests are throttled. +func throttled(n int, handler func(req testserver.Request) any) func(req testserver.Request) any { + var mu sync.Mutex + count := 0 + return func(req testserver.Request) any { + mu.Lock() + count++ + throttle := count <= n + mu.Unlock() + if throttle { + return throttledResponse + } + return handler(req) + } +} + +func newFilesClientWithFastRetries(t *testing.T, server *testserver.Server) *FilesClient { + t.Helper() + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewFilesClient(t.Context(), client, "/Volumes/main/schema/vol") + require.NoError(t, err) + fc := f.(*FilesClient) + fc.retryBackoff = ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond} + return fc +} + +func ok(testserver.Request) any { return testserver.Response{} } + +func TestFilesClientRetriesThrottledCalls(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + var uploads []string + server.Handle("HEAD", "/api/2.0/fs/directories/{path...}", throttled(2, ok)) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + uploads = append(uploads, string(req.Body)) + return testserver.Response{} + }) + server.Handle("GET", "/api/2.0/fs/files/{path...}", throttled(2, func(testserver.Request) any { + return testserver.Response{Body: []byte("abc")} + })) + server.Handle("DELETE", "/api/2.0/fs/files/{path...}", throttled(2, ok)) + f := newFilesClientWithFastRetries(t, server) + + require.NoError(t, f.Write(ctx, "hello", strings.NewReader("abc"))) + + r, err := f.Read(ctx, "hello") + require.NoError(t, err) + data, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, "abc", string(data)) + require.NoError(t, r.Close()) + + require.NoError(t, f.deleteFile(ctx, "hello")) + assert.Equal(t, []string{"abc"}, uploads) +} + +func TestFilesClientRetriesThrottledUploadFromStartOffset(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + var uploads []string + put := throttled(2, ok) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + uploads = append(uploads, string(req.Body)) + return put(req) + }) + f := newFilesClientWithFastRetries(t, server) + + // Each attempt resends the body from where the reader was when Write was + // called, not from where the failed attempt left it. + reader := strings.NewReader("skip:abc") + _, err := reader.Seek(5, io.SeekStart) + require.NoError(t, err) + require.NoError(t, f.Write(ctx, "hello", reader, CreateParentDirectories)) + assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) +} + +func TestFilesClientDoesNotRetryNonSeekableUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", throttled(1, ok)) + f := newFilesClientWithFastRetries(t, server) + + err := f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + assert.ErrorContains(t, err, "AWS S3 is throttling requests") +} + +func TestFilesClientStopsRetryingAfterLimit(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + requests := 0 + del := throttled(filesMaxRetries+1, ok) + server.Handle("DELETE", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + requests++ + return del(req) + }) + f := newFilesClientWithFastRetries(t, server) + + err := f.deleteFile(ctx, "hello") + assert.ErrorContains(t, err, "AWS S3 is throttling requests") + assert.Equal(t, filesMaxRetries+1, requests) +} From ae0f0d8899dd4a6f64fd63f91c57bd97974d867a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 13:55:21 +0000 Subject: [PATCH 2/4] Add changelog fragment Co-authored-by: Isaac --- .nextchanges/cli/7559.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/cli/7559.md diff --git a/.nextchanges/cli/7559.md b/.nextchanges/cli/7559.md new file mode 100644 index 00000000000..4b8b78704f8 --- /dev/null +++ b/.nextchanges/cli/7559.md @@ -0,0 +1 @@ +* Retry UC Volumes file operations (`fs cp`, `fs ls`, `fs rm`, bundle uploads to Volumes) when the Files API returns HTTP 429 or 503, for example "AWS S3 is throttling requests"; this regressed in v1.10.0. ([#7559](https://github.com/databricks/cli/pull/7559)) From 2a27f428f6780edb875c5962e5a441d6c5712986 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 14:00:59 +0000 Subject: [PATCH 3/4] filer: reuse apiretry, retry the SDK's status set, guard upload body reuse Use apiretry.NewRetrier with a retry cap, so Retry-After hints and transient network errors are honored, and retry the same statuses files/v2 retries for its own control-plane calls (408, 429, 502, 503, 504). Wrap each upload attempt's body so a late read by net/http cannot race with rewinding the reader for the next attempt. Co-authored-by: Isaac --- libs/filer/files_client.go | 88 +++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index cfb741974dd..25f8e8de344 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -9,6 +9,7 @@ import ( "net/http" "path" "slices" + "sync" "time" "github.com/databricks/cli/libs/databrickscfg/profile" @@ -16,6 +17,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/sdk-go/core/apierr" + "github.com/databricks/sdk-go/core/apiretry" "github.com/databricks/sdk-go/core/ops" files "github.com/databricks/sdk-go/files/v2" "github.com/databricks/sdk-go/options/call" @@ -40,12 +42,18 @@ func httpStatus(err error) int { } // filesRetryStatusCodes are the HTTP statuses on which Files API calls are -// retried. Throttled cloud storage comes back as 429 RESOURCE_EXHAUSTED (e.g. -// "AWS S3 is throttling requests; try again later.") and an unavailable backend -// as 503; both ask the caller to retry. files/v2 retries nothing unless the -// caller passes a retrier, so the filer supplies one to keep the retries the -// databricks-sdk-go client used to do for it. -var filesRetryStatusCodes = []int{http.StatusTooManyRequests, http.StatusServiceUnavailable} +// retried; the same set files/v2 retries for its own upload control-plane calls. +// Throttled cloud storage comes back as 429 RESOURCE_EXHAUSTED (e.g. "AWS S3 is +// throttling requests; try again later."), which asks the caller to retry. +// files/v2 retries nothing unless the caller passes a retrier, so the filer +// supplies one to keep the retries the databricks-sdk-go client used to do. +var filesRetryStatusCodes = []int{ + http.StatusRequestTimeout, // 408 + http.StatusTooManyRequests, // 429 + http.StatusBadGateway, // 502 + http.StatusServiceUnavailable, // 503 + http.StatusGatewayTimeout, // 504 +} // filesMaxRetries bounds the retries of one call. Non-streaming files/v2 calls // also have a 5 minute SDK timeout, but the single-shot upload retried by the @@ -55,17 +63,18 @@ const filesMaxRetries = 10 // filesRetryBackoff is the default delay policy between retries. var filesRetryBackoff = ops.BackoffPolicy{Initial: time.Second, Maximum: 10 * time.Second} -type filesRetrier struct { - backoff ops.BackoffPolicy +// cappedRetrier stops retrying after filesMaxRetries retries. +type cappedRetrier struct { + ops.Retrier retries int } -func (r *filesRetrier) IsRetriable(err error) (time.Duration, bool) { - if r.retries >= filesMaxRetries || !slices.Contains(filesRetryStatusCodes, httpStatus(err)) { +func (r *cappedRetrier) IsRetriable(err error) (time.Duration, bool) { + if r.retries >= filesMaxRetries { return 0, false } r.retries++ - return r.backoff.Delay(), true + return r.Retrier.IsRetriable(err) } // As of 19th Feb 2024, the Files API backend has a rate limit of 10 concurrent @@ -189,7 +198,7 @@ type FilesClient struct { } func (w *FilesClient) newRetrier() ops.Retrier { - return &filesRetrier{backoff: w.retryBackoff} + return &cappedRetrier{Retrier: apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: filesRetryStatusCodes})} } // retry is the call option that retries a files/v2 call on throttling. @@ -305,33 +314,62 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this // call because it cannot replay a streamed body, so a seekable reader is rewound -// to its starting offset and retried here; other readers get one attempt. +// to its starting offset and retried here. Other readers get one attempt; that +// includes pipes, which are *os.File and so implement io.Seeker but fail Seek. func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error { - upload := func(ctx context.Context) error { - _, err := w.client.UploadFile(ctx, files.UploadFileRequest{ - FilePath: &absPath, - Contents: io.NopCloser(reader), - Overwrite: &overwrite, - }) - return err - } - seeker, ok := reader.(io.Seeker) if !ok { - return upload(ctx) + return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) } start, err := seeker.Seek(0, io.SeekCurrent) if err != nil { - return upload(ctx) + return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) } return ops.Execute(ctx, func(ctx context.Context) error { if _, err := seeker.Seek(start, io.SeekStart); err != nil { return err } - return upload(ctx) + body := &attemptBody{r: reader} + err := w.putFile(ctx, absPath, body, overwrite) + body.Close() + return err }, ops.WithRetrier(w.newRetrier)) } +func (w *FilesClient) putFile(ctx context.Context, absPath string, body io.ReadCloser, overwrite bool) error { + _, err := w.client.UploadFile(ctx, files.UploadFileRequest{ + FilePath: &absPath, + Contents: body, + Overwrite: &overwrite, + }) + return err +} + +// attemptBody is the request body of one upload attempt. net/http may still be +// sending the body after the response has arrived; Close waits for an in-flight +// Read and stops further ones, so the next attempt can safely rewind the reader. +type attemptBody struct { + mu sync.Mutex + r io.Reader + closed bool +} + +func (b *attemptBody) Read(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return 0, io.ErrClosedPipe + } + return b.r.Read(p) +} + +func (b *attemptBody) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + b.closed = true + return nil +} + // isSeekable reports whether r can be seeked, without moving it: a no-op // Seek(0, io.SeekCurrent) returns the current offset for a working seeker and // errors for a broken one. Probing this way (rather than seeking to the end and From 59079aa68644ed77142d1d592f4b4b7e9a7427c0 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 17:05:14 +0000 Subject: [PATCH 4/4] Fix changelog entry: list every retried status and network errors Co-authored-by: Isaac --- .nextchanges/cli/7559.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/7559.md b/.nextchanges/cli/7559.md index 4b8b78704f8..9ba846aea9d 100644 --- a/.nextchanges/cli/7559.md +++ b/.nextchanges/cli/7559.md @@ -1 +1 @@ -* Retry UC Volumes file operations (`fs cp`, `fs ls`, `fs rm`, bundle uploads to Volumes) when the Files API returns HTTP 429 or 503, for example "AWS S3 is throttling requests"; this regressed in v1.10.0. ([#7559](https://github.com/databricks/cli/pull/7559)) +* Retry UC Volumes file operations (`fs cp`, `fs ls`, `fs rm`, bundle uploads to Volumes) when the Files API returns HTTP 408, 429, 502, 503 or 504 (for example "AWS S3 is throttling requests"), or on a transient network error; this regressed in v1.10.0. ([#7559](https://github.com/databricks/cli/pull/7559))