From 3b8984e4be7a6ea2427eb815131f1f876bb4e68c Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:13:53 +0000 Subject: [PATCH 1/3] filer: retry small non-seekable uploads to UC Volumes fs cp from a Volume or DBFS to a Volume streams the source download into the Files API upload. #7559 retries uploads only for seekable readers, so a single 503 (e.g. "Azure is temporarily unavailable; try again later.") on that upload still failed the copy. Buffer non-seekable bodies of up to 8 MiB so they can be replayed; larger streams keep one attempt. Co-authored-by: Isaac --- libs/filer/files_client.go | 32 +++++++++++++++++++++++++------- libs/filer/files_client_test.go | 28 +++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 25f8e8de344..fa4646095f8 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -1,6 +1,7 @@ package filer import ( + "bytes" "cmp" "context" "errors" @@ -312,18 +313,35 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, return err } +// filesReplayBufferSize is the largest non-seekable upload that is buffered in +// memory so it can be retried. fs cp between remote paths (Volumes or DBFS to +// Volumes) streams the source download into the upload, so without the buffer +// a single 503 fails the copy; larger streams are not buffered to keep memory +// bounded and get one attempt. +const filesReplayBufferSize = 8 << 20 + // uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this // call because it cannot replay a streamed body, so a seekable reader is rewound -// to its starting offset and retried here. Other readers get one attempt; that -// includes pipes, which are *os.File and so implement io.Seeker but fail Seek. +// to its starting offset and retried here. A non-seekable reader (including a +// pipe, which is an *os.File whose Seek fails) is buffered if it is at most +// filesReplayBufferSize bytes; a longer one gets one attempt. func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error { seeker, ok := reader.(io.Seeker) - if !ok { - return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) + var start int64 + var err error + if ok { + start, err = seeker.Seek(0, io.SeekCurrent) } - start, err := seeker.Seek(0, io.SeekCurrent) - if err != nil { - return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) + if !ok || err != nil { + head, err := io.ReadAll(io.LimitReader(reader, filesReplayBufferSize+1)) + if err != nil { + return err + } + if len(head) > filesReplayBufferSize { + return w.putFile(ctx, absPath, io.NopCloser(io.MultiReader(bytes.NewReader(head), reader)), overwrite) + } + buffered := bytes.NewReader(head) + reader, seeker, start = buffered, buffered, 0 } return ops.Execute(ctx, func(ctx context.Context) error { if _, err := seeker.Seek(start, io.SeekStart); err != nil { diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 0d621d3e542..96d67dd04b9 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -312,14 +312,36 @@ func TestFilesClientRetriesThrottledUploadFromStartOffset(t *testing.T) { assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) } -func TestFilesClientDoesNotRetryNonSeekableUpload(t *testing.T) { +func TestFilesClientRetriesSmallNonSeekableUpload(t *testing.T) { ctx := t.Context() server := testserver.New(t) - server.Handle("PUT", "/api/2.0/fs/files/{path...}", throttled(1, ok)) + var uploads []string + put := throttled(2, ok) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + uploads = append(uploads, string(req.Body)) + return put(req) + }) + f := newFilesClientWithFastRetries(t, server) + + require.NoError(t, f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)) + assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) +} + +func TestFilesClientDoesNotRetryLargeNonSeekableUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + var sizes []int + put := throttled(1, ok) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + sizes = append(sizes, len(req.Body)) + return put(req) + }) f := newFilesClientWithFastRetries(t, server) - err := f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + content := strings.Repeat("x", filesReplayBufferSize+1) + err := f.Write(ctx, "hello", onlyReader{strings.NewReader(content)}, CreateParentDirectories) assert.ErrorContains(t, err, "AWS S3 is throttling requests") + assert.Equal(t, []int{len(content)}, sizes) } func TestFilesClientStopsRetryingAfterLimit(t *testing.T) { From a6fbc0ecb9ebd2200987d2aae8fccd4852cdcd7b Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:14:48 +0000 Subject: [PATCH 2/3] Add changelog entry Co-authored-by: Isaac --- .nextchanges/cli/7607.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/cli/7607.md diff --git a/.nextchanges/cli/7607.md b/.nextchanges/cli/7607.md new file mode 100644 index 00000000000..268ce6ee6b9 --- /dev/null +++ b/.nextchanges/cli/7607.md @@ -0,0 +1 @@ +* Retry `fs cp` uploads to UC Volumes from another Volume or from DBFS (files up to 8 MiB) when the Files API returns a transient error such as HTTP 503. ([#7607](https://github.com/databricks/cli/pull/7607)) From 4a696ee9074811b521c87c5cc204d871b3232b1a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:20:39 +0000 Subject: [PATCH 3/3] filer: split replay buffering into a helper, test boundary and pipe Co-authored-by: Isaac --- libs/filer/files_client.go | 61 +++++++++++++++++++-------------- libs/filer/files_client_test.go | 55 +++++++++++++++++++++-------- 2 files changed, 77 insertions(+), 39 deletions(-) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index fa4646095f8..05957c34b10 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -5,6 +5,7 @@ import ( "cmp" "context" "errors" + "fmt" "io" "io/fs" "net/http" @@ -283,7 +284,8 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // upload engine, which sends small files in a single PUT and splits large ones // into parts. Non-seekable streams and the flag-off case use the single-shot // UploadFile endpoint below: the former to avoid buffering the whole stream in - // memory, the latter to keep the default behavior a plain PUT. The engine + // memory (uploadFile buffers only small ones, to retry them), the latter to + // keep the default behavior a plain PUT. The engine // recovers an io.ReaderAt for concurrent positioned reads when the source // provides one (a local file). if MultipartUploadEnabled(ctx) && isSeekable(reader) { @@ -316,38 +318,47 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // filesReplayBufferSize is the largest non-seekable upload that is buffered in // memory so it can be retried. fs cp between remote paths (Volumes or DBFS to // Volumes) streams the source download into the upload, so without the buffer -// a single 503 fails the copy; larger streams are not buffered to keep memory -// bounded and get one attempt. -const filesReplayBufferSize = 8 << 20 +// a single 503 fails the copy. Larger streams get one attempt, so memory per +// upload stays bounded. A variable so tests can lower it. +var filesReplayBufferSize = 8 << 20 + +// replayable returns a reader that can be rewound to start before each upload +// attempt. A non-seekable reader (including a pipe, which is an *os.File whose +// Seek fails) is read into memory if it is at most filesReplayBufferSize bytes; +// otherwise stream holds the bytes read so far followed by the +// rest of reader, to be sent once. +func replayable(reader io.Reader) (rs io.ReadSeeker, start int64, stream io.Reader, err error) { + if s, ok := reader.(io.ReadSeeker); ok { + if start, err := s.Seek(0, io.SeekCurrent); err == nil { + return s, start, nil, nil + } + } + head, err := io.ReadAll(io.LimitReader(reader, int64(filesReplayBufferSize)+1)) + if err != nil { + return nil, 0, nil, fmt.Errorf("reading upload body: %w", err) + } + if len(head) > filesReplayBufferSize { + return nil, 0, io.MultiReader(bytes.NewReader(head), reader), nil + } + return bytes.NewReader(head), 0, nil, nil +} // uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this -// call because it cannot replay a streamed body, so a seekable reader is rewound -// to its starting offset and retried here. A non-seekable reader (including a -// pipe, which is an *os.File whose Seek fails) is buffered if it is at most -// filesReplayBufferSize bytes; a longer one gets one attempt. +// call because it cannot replay a streamed body, so the filer retries it here +// with a reader from replayable, rewound to its starting offset each attempt. func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error { - seeker, ok := reader.(io.Seeker) - var start int64 - var err error - if ok { - start, err = seeker.Seek(0, io.SeekCurrent) - } - if !ok || err != nil { - head, err := io.ReadAll(io.LimitReader(reader, filesReplayBufferSize+1)) - if err != nil { - return err - } - if len(head) > filesReplayBufferSize { - return w.putFile(ctx, absPath, io.NopCloser(io.MultiReader(bytes.NewReader(head), reader)), overwrite) - } - buffered := bytes.NewReader(head) - reader, seeker, start = buffered, buffered, 0 + seeker, start, stream, err := replayable(reader) + if err != nil { + return err + } + if stream != nil { + return w.putFile(ctx, absPath, io.NopCloser(stream), overwrite) } return ops.Execute(ctx, func(ctx context.Context) error { if _, err := seeker.Seek(start, io.SeekStart); err != nil { return err } - body := &attemptBody{r: reader} + body := &attemptBody{r: seeker} err := w.putFile(ctx, absPath, body, overwrite) body.Close() return err diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 96d67dd04b9..270394175e5 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -312,36 +312,63 @@ func TestFilesClientRetriesThrottledUploadFromStartOffset(t *testing.T) { assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) } -func TestFilesClientRetriesSmallNonSeekableUpload(t *testing.T) { - ctx := t.Context() - server := testserver.New(t) +// uploadsWithOneThrottle records each PUT body; the first PUT is throttled. +func uploadsWithOneThrottle(server *testserver.Server) *[]string { var uploads []string - put := throttled(2, ok) + put := throttled(1, ok) server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { uploads = append(uploads, string(req.Body)) return put(req) }) + return &uploads +} + +func TestFilesClientRetriesSmallNonSeekableUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + uploads := uploadsWithOneThrottle(server) f := newFilesClientWithFastRetries(t, server) + setReplayBufferSize(t, 3) + // Exactly filesReplayBufferSize bytes still fit in the buffer. require.NoError(t, f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)) - assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) + assert.Equal(t, []string{"abc", "abc"}, *uploads) +} + +func TestFilesClientRetriesSmallPipeUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + uploads := uploadsWithOneThrottle(server) + f := newFilesClientWithFastRetries(t, server) + + // A pipe is an *os.File whose Seek fails. + pr, pw, err := os.Pipe() + require.NoError(t, err) + _, err = pw.WriteString("abc") + require.NoError(t, err) + require.NoError(t, pw.Close()) + defer pr.Close() + + require.NoError(t, f.Write(ctx, "hello", pr, CreateParentDirectories)) + assert.Equal(t, []string{"abc", "abc"}, *uploads) } func TestFilesClientDoesNotRetryLargeNonSeekableUpload(t *testing.T) { ctx := t.Context() server := testserver.New(t) - var sizes []int - put := throttled(1, ok) - server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { - sizes = append(sizes, len(req.Body)) - return put(req) - }) + uploads := uploadsWithOneThrottle(server) f := newFilesClientWithFastRetries(t, server) + setReplayBufferSize(t, 2) - content := strings.Repeat("x", filesReplayBufferSize+1) - err := f.Write(ctx, "hello", onlyReader{strings.NewReader(content)}, CreateParentDirectories) + err := f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) assert.ErrorContains(t, err, "AWS S3 is throttling requests") - assert.Equal(t, []int{len(content)}, sizes) + assert.Equal(t, []string{"abc"}, *uploads) +} + +func setReplayBufferSize(t *testing.T, n int) { + old := filesReplayBufferSize + filesReplayBufferSize = n + t.Cleanup(func() { filesReplayBufferSize = old }) } func TestFilesClientStopsRetryingAfterLimit(t *testing.T) {