Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/7559.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Retry UC Volumes file operations (`fs cp`, `fs ls`, `fs rm`, bundle uploads to Volumes) when the Files API returns HTTP 408, 429, 502, 503 or 504 (for example "AWS S3 is throttling requests"), or on a transient network error; this regressed in v1.10.0. ([#7559](https://github.com/databricks/cli/pull/7559))
133 changes: 120 additions & 13 deletions libs/filer/files_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,18 @@ import (
"net/http"
"path"
"slices"
"sync"
"time"

"github.com/databricks/cli/libs/databrickscfg/profile"
"github.com/databricks/cli/libs/env"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/config"
"github.com/databricks/sdk-go/core/apierr"
"github.com/databricks/sdk-go/core/apiretry"
"github.com/databricks/sdk-go/core/ops"
files "github.com/databricks/sdk-go/files/v2"
"github.com/databricks/sdk-go/options/call"
"github.com/databricks/sdk-go/options/client"
"golang.org/x/sync/errgroup"
)
Expand All @@ -37,6 +41,42 @@ func httpStatus(err error) int {
return -1
}

// filesRetryStatusCodes are the HTTP statuses on which Files API calls are
// retried; the same set files/v2 retries for its own upload control-plane calls.
// Throttled cloud storage comes back as 429 RESOURCE_EXHAUSTED (e.g. "AWS S3 is
// throttling requests; try again later."), which asks the caller to retry.
// files/v2 retries nothing unless the caller passes a retrier, so the filer
// supplies one to keep the retries the databricks-sdk-go client used to do.
var filesRetryStatusCodes = []int{
http.StatusRequestTimeout, // 408
http.StatusTooManyRequests, // 429
http.StatusBadGateway, // 502
http.StatusServiceUnavailable, // 503
http.StatusGatewayTimeout, // 504
}

// filesMaxRetries bounds the retries of one call. Non-streaming files/v2 calls
// also have a 5 minute SDK timeout, but the single-shot upload retried by the
// filer itself does not.
const filesMaxRetries = 10

// filesRetryBackoff is the default delay policy between retries.
var filesRetryBackoff = ops.BackoffPolicy{Initial: time.Second, Maximum: 10 * time.Second}

// cappedRetrier stops retrying after filesMaxRetries retries.
type cappedRetrier struct {
ops.Retrier
retries int
}

func (r *cappedRetrier) IsRetriable(err error) (time.Duration, bool) {
if r.retries >= filesMaxRetries {
return 0, false
}
r.retries++
return r.Retrier.IsRetriable(err)
}

// As of 19th Feb 2024, the Files API backend has a rate limit of 10 concurrent
// requests and 100 QPS. We limit the number of concurrent requests to 5 to
// avoid hitting the rate limit.
Expand Down Expand Up @@ -152,6 +192,18 @@ type FilesClient struct {
// (e.g. fs cp -r) draw from one bounded budget and one connection pool.
limiter files.Limiter
transferClient *http.Client

// retryBackoff is copied into a fresh retrier for every call.
retryBackoff ops.BackoffPolicy
}

func (w *FilesClient) newRetrier() ops.Retrier {
return &cappedRetrier{Retrier: apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: filesRetryStatusCodes})}
}

// retry is the call option that retries a files/v2 call on throttling.
func (w *FilesClient) retry() call.Option {
return call.WithRetrier(w.newRetrier)
}

func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root string) (Filer, error) {
Expand All @@ -167,6 +219,7 @@ func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root str

limiter: files.NewLimiter(uploadConcurrency),
transferClient: newTransferClient(uploadConcurrency),
retryBackoff: filesRetryBackoff,
}, nil
}

Expand Down Expand Up @@ -213,7 +266,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
// Check that target path exists if CreateParentDirectories mode is not set
if !slices.Contains(mode, CreateParentDirectories) {
dir := path.Dir(absPath)
_, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir})
_, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}, w.retry())
if err != nil {
// This API returns a 404 if the directory doesn't exist.
if httpStatus(err) == http.StatusNotFound {
Expand Down Expand Up @@ -247,11 +300,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
return mapUploadError(uerr, absPath)
}

_, err = w.client.UploadFile(ctx, files.UploadFileRequest{
FilePath: &absPath,
Contents: io.NopCloser(reader),
Overwrite: &overwrite,
})
err = w.uploadFile(ctx, absPath, reader, overwrite)
if err == nil {
return nil
}
Expand All @@ -263,6 +312,64 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
return err
}

// uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this
// call because it cannot replay a streamed body, so a seekable reader is rewound
// to its starting offset and retried here. Other readers get one attempt; that
// includes pipes, which are *os.File and so implement io.Seeker but fail Seek.
func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error {
seeker, ok := reader.(io.Seeker)
if !ok {
return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite)
}
start, err := seeker.Seek(0, io.SeekCurrent)
if err != nil {
return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite)
}
return ops.Execute(ctx, func(ctx context.Context) error {
if _, err := seeker.Seek(start, io.SeekStart); err != nil {
return err
}
body := &attemptBody{r: reader}
err := w.putFile(ctx, absPath, body, overwrite)
body.Close()
return err
}, ops.WithRetrier(w.newRetrier))
}

func (w *FilesClient) putFile(ctx context.Context, absPath string, body io.ReadCloser, overwrite bool) error {
_, err := w.client.UploadFile(ctx, files.UploadFileRequest{
FilePath: &absPath,
Contents: body,
Overwrite: &overwrite,
})
return err
}

// attemptBody is the request body of one upload attempt. net/http may still be
// sending the body after the response has arrived; Close waits for an in-flight
// Read and stops further ones, so the next attempt can safely rewind the reader.
type attemptBody struct {
mu sync.Mutex
r io.Reader
closed bool
}

func (b *attemptBody) Read(p []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.closed {
return 0, io.ErrClosedPipe
}
return b.r.Read(p)
}

func (b *attemptBody) Close() error {
b.mu.Lock()
defer b.mu.Unlock()
b.closed = true
return nil
}

// isSeekable reports whether r can be seeked, without moving it: a no-op
// Seek(0, io.SeekCurrent) returns the current offset for a working seeker and
// errors for a broken one. Probing this way (rather than seeking to the end and
Expand Down Expand Up @@ -295,7 +402,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err
return nil, err
}

resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath})
resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}, w.retry())

// Return early on success.
if err == nil {
Expand Down Expand Up @@ -327,7 +434,7 @@ func (w *FilesClient) deleteFile(ctx context.Context, name string) error {
return cannotDeleteRootError{}
}

_, err = w.client.DeleteFile(ctx, files.DeleteFileRequest{FilePath: &absPath})
_, err = w.client.DeleteFile(ctx, files.DeleteFileRequest{FilePath: &absPath}, w.retry())

// Return early on success.
if err == nil {
Expand All @@ -353,7 +460,7 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error {
return cannotDeleteRootError{}
}

_, err = w.client.DeleteDirectory(ctx, files.DeleteDirectoryRequest{DirectoryPath: &absPath})
_, err = w.client.DeleteDirectory(ctx, files.DeleteDirectoryRequest{DirectoryPath: &absPath}, w.retry())

// Return early on success.
if err == nil {
Expand Down Expand Up @@ -477,7 +584,7 @@ func (w *FilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry,
var entries []fs.DirEntry
for entry, err := range w.client.ListDirectoryContentsIter(ctx, files.ListDirectoryContentsRequest{
DirectoryPath: &absPath,
}) {
}, w.retry()) {
if err != nil {
// This API returns a 404 if the specified path does not exist.
if httpStatus(err) == http.StatusNotFound {
Expand Down Expand Up @@ -513,7 +620,7 @@ func (w *FilesClient) Mkdir(ctx context.Context, name string) error {
return err
}

_, err = w.client.CreateDirectory(ctx, files.CreateDirectoryRequest{DirectoryPath: &absPath})
_, err = w.client.CreateDirectory(ctx, files.CreateDirectoryRequest{DirectoryPath: &absPath}, w.retry())

// This API returns a 409 when a file already exists at the path (the create
// is not idempotent over a file).
Expand All @@ -531,7 +638,7 @@ func (w *FilesClient) statFile(ctx context.Context, name string) (fs.FileInfo, e
return nil, err
}

resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath})
resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}, w.retry())

// If the HEAD requests succeeds, the file exists.
if err == nil {
Expand All @@ -557,7 +664,7 @@ func (w *FilesClient) statDir(ctx context.Context, name string) (fs.FileInfo, er
return nil, err
}

_, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath})
_, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}, w.retry())

// If the HEAD requests succeeds, the directory exists.
if err == nil {
Expand Down
121 changes: 121 additions & 0 deletions libs/filer/files_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,17 @@ import (
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"

cliauth "github.com/databricks/cli/libs/auth"
"github.com/databricks/cli/libs/databrickscfg/profile"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/testserver"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/sdk-go/core/ops"
files "github.com/databricks/sdk-go/files/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -216,3 +220,120 @@ func TestMapUploadError(t *testing.T) {
t.Errorf("mapUploadError(other) = %v, want it unchanged", got)
}
}

// throttledResponse is how the Files API reports throttled cloud storage.
var throttledResponse = testserver.Response{
StatusCode: 429,
Body: map[string]any{
"error_code": "RESOURCE_EXHAUSTED",
"message": "AWS S3 is throttling requests; try again later.",
},
}

// throttled wraps a handler so that its first n requests are throttled.
func throttled(n int, handler func(req testserver.Request) any) func(req testserver.Request) any {
var mu sync.Mutex
count := 0
return func(req testserver.Request) any {
mu.Lock()
count++
throttle := count <= n
mu.Unlock()
if throttle {
return throttledResponse
}
return handler(req)
}
}

func newFilesClientWithFastRetries(t *testing.T, server *testserver.Server) *FilesClient {
t.Helper()

client, err := databricks.NewWorkspaceClient(&databricks.Config{
Host: server.URL,
Token: "testtoken",
})
require.NoError(t, err)

f, err := NewFilesClient(t.Context(), client, "/Volumes/main/schema/vol")
require.NoError(t, err)
fc := f.(*FilesClient)
fc.retryBackoff = ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond}
return fc
}

func ok(testserver.Request) any { return testserver.Response{} }

func TestFilesClientRetriesThrottledCalls(t *testing.T) {
ctx := t.Context()
server := testserver.New(t)
var uploads []string
server.Handle("HEAD", "/api/2.0/fs/directories/{path...}", throttled(2, ok))
server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any {
uploads = append(uploads, string(req.Body))
return testserver.Response{}
})
server.Handle("GET", "/api/2.0/fs/files/{path...}", throttled(2, func(testserver.Request) any {
return testserver.Response{Body: []byte("abc")}
}))
server.Handle("DELETE", "/api/2.0/fs/files/{path...}", throttled(2, ok))
f := newFilesClientWithFastRetries(t, server)

require.NoError(t, f.Write(ctx, "hello", strings.NewReader("abc")))

r, err := f.Read(ctx, "hello")
require.NoError(t, err)
data, err := io.ReadAll(r)
require.NoError(t, err)
assert.Equal(t, "abc", string(data))
require.NoError(t, r.Close())

require.NoError(t, f.deleteFile(ctx, "hello"))
assert.Equal(t, []string{"abc"}, uploads)
}

func TestFilesClientRetriesThrottledUploadFromStartOffset(t *testing.T) {
ctx := t.Context()
server := testserver.New(t)
var uploads []string
put := throttled(2, ok)
server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any {
uploads = append(uploads, string(req.Body))
return put(req)
})
f := newFilesClientWithFastRetries(t, server)

// Each attempt resends the body from where the reader was when Write was
// called, not from where the failed attempt left it.
reader := strings.NewReader("skip:abc")
_, err := reader.Seek(5, io.SeekStart)
require.NoError(t, err)
require.NoError(t, f.Write(ctx, "hello", reader, CreateParentDirectories))
assert.Equal(t, []string{"abc", "abc", "abc"}, uploads)
}

func TestFilesClientDoesNotRetryNonSeekableUpload(t *testing.T) {
ctx := t.Context()
server := testserver.New(t)
server.Handle("PUT", "/api/2.0/fs/files/{path...}", throttled(1, ok))
f := newFilesClientWithFastRetries(t, server)

err := f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)
assert.ErrorContains(t, err, "AWS S3 is throttling requests")
}

func TestFilesClientStopsRetryingAfterLimit(t *testing.T) {
ctx := t.Context()
server := testserver.New(t)
requests := 0
del := throttled(filesMaxRetries+1, ok)
server.Handle("DELETE", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any {
requests++
return del(req)
})
f := newFilesClientWithFastRetries(t, server)

err := f.deleteFile(ctx, "hello")
assert.ErrorContains(t, err, "AWS S3 is throttling requests")
assert.Equal(t, filesMaxRetries+1, requests)
}
Loading