diff --git a/.nextchanges/cli/7559.md b/.nextchanges/cli/7559.md new file mode 100644 index 00000000000..9ba846aea9d --- /dev/null +++ b/.nextchanges/cli/7559.md @@ -0,0 +1 @@ +* Retry UC Volumes file operations (`fs cp`, `fs ls`, `fs rm`, bundle uploads to Volumes) when the Files API returns HTTP 408, 429, 502, 503 or 504 (for example "AWS S3 is throttling requests"), or on a transient network error; this regressed in v1.10.0. ([#7559](https://github.com/databricks/cli/pull/7559)) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 1638acb73bd..25f8e8de344 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -9,6 +9,7 @@ import ( "net/http" "path" "slices" + "sync" "time" "github.com/databricks/cli/libs/databrickscfg/profile" @@ -16,7 +17,10 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/sdk-go/core/apierr" + "github.com/databricks/sdk-go/core/apiretry" + "github.com/databricks/sdk-go/core/ops" files "github.com/databricks/sdk-go/files/v2" + "github.com/databricks/sdk-go/options/call" "github.com/databricks/sdk-go/options/client" "golang.org/x/sync/errgroup" ) @@ -37,6 +41,42 @@ func httpStatus(err error) int { return -1 } +// filesRetryStatusCodes are the HTTP statuses on which Files API calls are +// retried; the same set files/v2 retries for its own upload control-plane calls. +// Throttled cloud storage comes back as 429 RESOURCE_EXHAUSTED (e.g. "AWS S3 is +// throttling requests; try again later."), which asks the caller to retry. +// files/v2 retries nothing unless the caller passes a retrier, so the filer +// supplies one to keep the retries the databricks-sdk-go client used to do. +var filesRetryStatusCodes = []int{ + http.StatusRequestTimeout, // 408 + http.StatusTooManyRequests, // 429 + http.StatusBadGateway, // 502 + http.StatusServiceUnavailable, // 503 + http.StatusGatewayTimeout, // 504 +} + +// filesMaxRetries bounds the retries of one call. Non-streaming files/v2 calls +// also have a 5 minute SDK timeout, but the single-shot upload retried by the +// filer itself does not. +const filesMaxRetries = 10 + +// filesRetryBackoff is the default delay policy between retries. +var filesRetryBackoff = ops.BackoffPolicy{Initial: time.Second, Maximum: 10 * time.Second} + +// cappedRetrier stops retrying after filesMaxRetries retries. +type cappedRetrier struct { + ops.Retrier + retries int +} + +func (r *cappedRetrier) IsRetriable(err error) (time.Duration, bool) { + if r.retries >= filesMaxRetries { + return 0, false + } + r.retries++ + return r.Retrier.IsRetriable(err) +} + // As of 19th Feb 2024, the Files API backend has a rate limit of 10 concurrent // requests and 100 QPS. We limit the number of concurrent requests to 5 to // avoid hitting the rate limit. @@ -152,6 +192,18 @@ type FilesClient struct { // (e.g. fs cp -r) draw from one bounded budget and one connection pool. limiter files.Limiter transferClient *http.Client + + // retryBackoff is copied into a fresh retrier for every call. + retryBackoff ops.BackoffPolicy +} + +func (w *FilesClient) newRetrier() ops.Retrier { + return &cappedRetrier{Retrier: apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: filesRetryStatusCodes})} +} + +// retry is the call option that retries a files/v2 call on throttling. +func (w *FilesClient) retry() call.Option { + return call.WithRetrier(w.newRetrier) } func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root string) (Filer, error) { @@ -167,6 +219,7 @@ func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root str limiter: files.NewLimiter(uploadConcurrency), transferClient: newTransferClient(uploadConcurrency), + retryBackoff: filesRetryBackoff, }, nil } @@ -213,7 +266,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // Check that target path exists if CreateParentDirectories mode is not set if !slices.Contains(mode, CreateParentDirectories) { dir := path.Dir(absPath) - _, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}) + _, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}, w.retry()) if err != nil { // This API returns a 404 if the directory doesn't exist. if httpStatus(err) == http.StatusNotFound { @@ -247,11 +300,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, return mapUploadError(uerr, absPath) } - _, err = w.client.UploadFile(ctx, files.UploadFileRequest{ - FilePath: &absPath, - Contents: io.NopCloser(reader), - Overwrite: &overwrite, - }) + err = w.uploadFile(ctx, absPath, reader, overwrite) if err == nil { return nil } @@ -263,6 +312,64 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, return err } +// uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this +// call because it cannot replay a streamed body, so a seekable reader is rewound +// to its starting offset and retried here. Other readers get one attempt; that +// includes pipes, which are *os.File and so implement io.Seeker but fail Seek. +func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error { + seeker, ok := reader.(io.Seeker) + if !ok { + return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) + } + start, err := seeker.Seek(0, io.SeekCurrent) + if err != nil { + return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) + } + return ops.Execute(ctx, func(ctx context.Context) error { + if _, err := seeker.Seek(start, io.SeekStart); err != nil { + return err + } + body := &attemptBody{r: reader} + err := w.putFile(ctx, absPath, body, overwrite) + body.Close() + return err + }, ops.WithRetrier(w.newRetrier)) +} + +func (w *FilesClient) putFile(ctx context.Context, absPath string, body io.ReadCloser, overwrite bool) error { + _, err := w.client.UploadFile(ctx, files.UploadFileRequest{ + FilePath: &absPath, + Contents: body, + Overwrite: &overwrite, + }) + return err +} + +// attemptBody is the request body of one upload attempt. net/http may still be +// sending the body after the response has arrived; Close waits for an in-flight +// Read and stops further ones, so the next attempt can safely rewind the reader. +type attemptBody struct { + mu sync.Mutex + r io.Reader + closed bool +} + +func (b *attemptBody) Read(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return 0, io.ErrClosedPipe + } + return b.r.Read(p) +} + +func (b *attemptBody) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + b.closed = true + return nil +} + // isSeekable reports whether r can be seeked, without moving it: a no-op // Seek(0, io.SeekCurrent) returns the current offset for a working seeker and // errors for a broken one. Probing this way (rather than seeking to the end and @@ -295,7 +402,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err return nil, err } - resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}) + resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}, w.retry()) // Return early on success. if err == nil { @@ -327,7 +434,7 @@ func (w *FilesClient) deleteFile(ctx context.Context, name string) error { return cannotDeleteRootError{} } - _, err = w.client.DeleteFile(ctx, files.DeleteFileRequest{FilePath: &absPath}) + _, err = w.client.DeleteFile(ctx, files.DeleteFileRequest{FilePath: &absPath}, w.retry()) // Return early on success. if err == nil { @@ -353,7 +460,7 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error { return cannotDeleteRootError{} } - _, err = w.client.DeleteDirectory(ctx, files.DeleteDirectoryRequest{DirectoryPath: &absPath}) + _, err = w.client.DeleteDirectory(ctx, files.DeleteDirectoryRequest{DirectoryPath: &absPath}, w.retry()) // Return early on success. if err == nil { @@ -477,7 +584,7 @@ func (w *FilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, var entries []fs.DirEntry for entry, err := range w.client.ListDirectoryContentsIter(ctx, files.ListDirectoryContentsRequest{ DirectoryPath: &absPath, - }) { + }, w.retry()) { if err != nil { // This API returns a 404 if the specified path does not exist. if httpStatus(err) == http.StatusNotFound { @@ -513,7 +620,7 @@ func (w *FilesClient) Mkdir(ctx context.Context, name string) error { return err } - _, err = w.client.CreateDirectory(ctx, files.CreateDirectoryRequest{DirectoryPath: &absPath}) + _, err = w.client.CreateDirectory(ctx, files.CreateDirectoryRequest{DirectoryPath: &absPath}, w.retry()) // This API returns a 409 when a file already exists at the path (the create // is not idempotent over a file). @@ -531,7 +638,7 @@ func (w *FilesClient) statFile(ctx context.Context, name string) (fs.FileInfo, e return nil, err } - resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}) + resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}, w.retry()) // If the HEAD requests succeeds, the file exists. if err == nil { @@ -557,7 +664,7 @@ func (w *FilesClient) statDir(ctx context.Context, name string) (fs.FileInfo, er return nil, err } - _, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}) + _, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}, w.retry()) // If the HEAD requests succeeds, the directory exists. if err == nil { diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 3579c5464c5..0d621d3e542 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -8,13 +8,17 @@ import ( "io/fs" "os" "path/filepath" + "strings" + "sync" "testing" + "time" cliauth "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/sdk-go/core/ops" files "github.com/databricks/sdk-go/files/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -216,3 +220,120 @@ func TestMapUploadError(t *testing.T) { t.Errorf("mapUploadError(other) = %v, want it unchanged", got) } } + +// throttledResponse is how the Files API reports throttled cloud storage. +var throttledResponse = testserver.Response{ + StatusCode: 429, + Body: map[string]any{ + "error_code": "RESOURCE_EXHAUSTED", + "message": "AWS S3 is throttling requests; try again later.", + }, +} + +// throttled wraps a handler so that its first n requests are throttled. +func throttled(n int, handler func(req testserver.Request) any) func(req testserver.Request) any { + var mu sync.Mutex + count := 0 + return func(req testserver.Request) any { + mu.Lock() + count++ + throttle := count <= n + mu.Unlock() + if throttle { + return throttledResponse + } + return handler(req) + } +} + +func newFilesClientWithFastRetries(t *testing.T, server *testserver.Server) *FilesClient { + t.Helper() + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewFilesClient(t.Context(), client, "/Volumes/main/schema/vol") + require.NoError(t, err) + fc := f.(*FilesClient) + fc.retryBackoff = ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond} + return fc +} + +func ok(testserver.Request) any { return testserver.Response{} } + +func TestFilesClientRetriesThrottledCalls(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + var uploads []string + server.Handle("HEAD", "/api/2.0/fs/directories/{path...}", throttled(2, ok)) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + uploads = append(uploads, string(req.Body)) + return testserver.Response{} + }) + server.Handle("GET", "/api/2.0/fs/files/{path...}", throttled(2, func(testserver.Request) any { + return testserver.Response{Body: []byte("abc")} + })) + server.Handle("DELETE", "/api/2.0/fs/files/{path...}", throttled(2, ok)) + f := newFilesClientWithFastRetries(t, server) + + require.NoError(t, f.Write(ctx, "hello", strings.NewReader("abc"))) + + r, err := f.Read(ctx, "hello") + require.NoError(t, err) + data, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, "abc", string(data)) + require.NoError(t, r.Close()) + + require.NoError(t, f.deleteFile(ctx, "hello")) + assert.Equal(t, []string{"abc"}, uploads) +} + +func TestFilesClientRetriesThrottledUploadFromStartOffset(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + var uploads []string + put := throttled(2, ok) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + uploads = append(uploads, string(req.Body)) + return put(req) + }) + f := newFilesClientWithFastRetries(t, server) + + // Each attempt resends the body from where the reader was when Write was + // called, not from where the failed attempt left it. + reader := strings.NewReader("skip:abc") + _, err := reader.Seek(5, io.SeekStart) + require.NoError(t, err) + require.NoError(t, f.Write(ctx, "hello", reader, CreateParentDirectories)) + assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) +} + +func TestFilesClientDoesNotRetryNonSeekableUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", throttled(1, ok)) + f := newFilesClientWithFastRetries(t, server) + + err := f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + assert.ErrorContains(t, err, "AWS S3 is throttling requests") +} + +func TestFilesClientStopsRetryingAfterLimit(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + requests := 0 + del := throttled(filesMaxRetries+1, ok) + server.Handle("DELETE", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + requests++ + return del(req) + }) + f := newFilesClientWithFastRetries(t, server) + + err := f.deleteFile(ctx, "hello") + assert.ErrorContains(t, err, "AWS S3 is throttling requests") + assert.Equal(t, filesMaxRetries+1, requests) +}