From 60e843cd13603852c1198dba9a1ee413ba81710a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 13:48:10 +0000 Subject: [PATCH 1/3] Retry fs command reads on transient errors databricks fs cp/ls/cat/rm now retry Stat, Read and ReadDir against DBFS and the Files API on transient errors, including the SDK's inactivity timeout, using the same retry policy as bundle file operations (libs/retry). Writes and deletes are not retried. Co-authored-by: Isaac --- cmd/fs/helpers.go | 18 +++++++++++++----- cmd/fs/helpers_test.go | 4 +++- libs/retry/filer.go | 20 +++++++++++++++++++- libs/retry/filer_test.go | 21 +++++++++++++++++++++ 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index d2e98563314..af7d37166d5 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/filer/completer" + "github.com/databricks/cli/libs/retry" "github.com/spf13/cobra" ) @@ -39,14 +40,21 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er w := cmdctx.WorkspaceClient(ctx) // If the specified path has the "Volumes" or "Skills" prefix, use the Files API. + // Otherwise the file is a dbfs file, and uses the DBFS APIs. + var f filer.Filer + var err error if strings.HasPrefix(path, "/Volumes/") || strings.HasPrefix(path, "/Skills/") { - f, err := filer.NewFilesClient(ctx, w, "/") - return f, path, err + f, err = filer.NewFilesClient(ctx, w, "/") + } else { + f, err = filer.NewDbfsClient(w, "/") + } + if err != nil { + return nil, "", err } - // The file is a dbfs file, and uses the DBFS APIs - f, err := filer.NewDbfsClient(w, "/") - return f, path, err + // Writes are not retried: a write without overwrite may fail on the file its own earlier + // attempt created, and an overwrite would need to buffer the streamed source to replay it. + return retry.ReadFiler(f), path, nil } const dbfsPrefix string = "dbfs:" diff --git a/cmd/fs/helpers_test.go b/cmd/fs/helpers_test.go index ee1a88e23fb..dc54914ee0c 100644 --- a/cmd/fs/helpers_test.go +++ b/cmd/fs/helpers_test.go @@ -62,7 +62,9 @@ func TestFilerForPathForDbfsScheme(t *testing.T) { f, path, err := filerForPath(ctx, tc.fullPath) require.NoError(t, err) assert.Equal(t, tc.path, path) - assert.IsType(t, tc.filer, f) + unwrapper, ok := f.(interface{ Unwrap() filer.Filer }) + require.True(t, ok, "remote filers retry reads") + assert.IsType(t, tc.filer, unwrapper.Unwrap()) }) } } diff --git a/libs/retry/filer.go b/libs/retry/filer.go index 11b991b18fc..6ebd529ddee 100644 --- a/libs/retry/filer.go +++ b/libs/retry/filer.go @@ -14,6 +14,9 @@ import ( // retryingFiler retries calls that fail with a transient server error and are safe to repeat. type retryingFiler struct { filer.Filer + + // readsOnly passes Write and Delete through unchanged. + readsOnly bool } // Filer wraps f so that Write, Read, ReadDir, Stat and Delete retry on transient errors. @@ -25,8 +28,20 @@ func Filer(f filer.Filer) filer.Filer { return &retryingFiler{Filer: f} } +// ReadFiler wraps f so that only Read, ReadDir and Stat retry on transient errors. +// It suits callers that stream arbitrarily large bodies to Write: retrying those would mean +// buffering each non-seekable body in memory to replay it. +func ReadFiler(f filer.Filer) filer.Filer { + return &retryingFiler{Filer: f, readsOnly: true} +} + +// Unwrap returns the wrapped filer. +func (r *retryingFiler) Unwrap() filer.Filer { + return r.Filer +} + func (r *retryingFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...filer.WriteMode) error { - if !slices.Contains(mode, filer.OverwriteIfExists) { + if r.readsOnly || !slices.Contains(mode, filer.OverwriteIfExists) { return r.Filer.Write(ctx, path, reader, mode...) } @@ -78,6 +93,9 @@ func (r *retryingFiler) Stat(ctx context.Context, path string) (fs.FileInfo, err } func (r *retryingFiler) Delete(ctx context.Context, path string, mode ...filer.DeleteMode) error { + if r.readsOnly { + return r.Filer.Delete(ctx, path, mode...) + } retried := false _, err := With(ctx, IsTransient, func() (struct{}, error) { err := r.Filer.Delete(ctx, path, mode...) diff --git a/libs/retry/filer_test.go b/libs/retry/filer_test.go index 90e57faedb9..4faf4e0e9da 100644 --- a/libs/retry/filer_test.go +++ b/libs/retry/filer_test.go @@ -166,3 +166,24 @@ func (f *sequenceFiler) Delete(ctx context.Context, path string, mode ...filer.D f.errs = f.errs[1:] return err } + +func TestReadFilerRetriesReads(t *testing.T) { + ctx := testContext(t) + f := &readFiler{failures: 1, err: errTransient} + _, err := ReadFiler(f).Stat(ctx, "a") + require.NoError(t, err) + assert.Equal(t, 2, f.calls) +} + +func TestReadFilerDoesNotRetryWrites(t *testing.T) { + f := &flakyFiler{failures: 100, err: errTransient} + err := ReadFiler(f).Write(testContext(t), "a", strings.NewReader("x"), filer.OverwriteIfExists) + require.Error(t, err) + assert.Len(t, f.bodies, 1) +} + +func TestReadFilerDoesNotRetryDelete(t *testing.T) { + f := &readFiler{failures: 1, err: errTransient} + require.Error(t, ReadFiler(f).Delete(testContext(t), "a")) + assert.Equal(t, 1, f.calls) +} From 67a71b0a7058ef338f63ac2a7b768777b4f60c2f Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 13:49:20 +0000 Subject: [PATCH 2/3] Add changelog entry Co-authored-by: Isaac --- .nextchanges/cli/7556.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/cli/7556.md diff --git a/.nextchanges/cli/7556.md b/.nextchanges/cli/7556.md new file mode 100644 index 00000000000..13133a4b19d --- /dev/null +++ b/.nextchanges/cli/7556.md @@ -0,0 +1 @@ +* `databricks fs` commands retry reads (stat, list, open) on DBFS and Volumes when the server returns a transient error or does not respond within the HTTP timeout. ([#7556](https://github.com/databricks/cli/pull/7556)) From f153cb29c9b6807888cf5b63f4055eac6e350127 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 15:00:27 +0000 Subject: [PATCH 3/3] Retry fs deletes too; add acceptance test for fs retries Rename retry.ReadFiler to retry.FilerExceptWrites: it keeps Filer's retry-safe Delete and skips only Write. Add acceptance/cmd/fs/retry-504, which injects a 504 on dbfs list, get-status and delete. Co-authored-by: Isaac --- .nextchanges/cli/7556.md | 2 +- acceptance/cmd/fs/retry-504/out.test.toml | 1 + acceptance/cmd/fs/retry-504/output.txt | 19 +++++++++++++++++ acceptance/cmd/fs/retry-504/script | 25 +++++++++++++++++++++++ acceptance/cmd/fs/retry-504/test.toml | 2 ++ cmd/fs/helpers.go | 2 +- libs/retry/filer.go | 15 ++++++-------- libs/retry/filer_test.go | 15 +++++++------- 8 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 acceptance/cmd/fs/retry-504/out.test.toml create mode 100644 acceptance/cmd/fs/retry-504/output.txt create mode 100644 acceptance/cmd/fs/retry-504/script create mode 100644 acceptance/cmd/fs/retry-504/test.toml diff --git a/.nextchanges/cli/7556.md b/.nextchanges/cli/7556.md index 13133a4b19d..f95e30adf3c 100644 --- a/.nextchanges/cli/7556.md +++ b/.nextchanges/cli/7556.md @@ -1 +1 @@ -* `databricks fs` commands retry reads (stat, list, open) on DBFS and Volumes when the server returns a transient error or does not respond within the HTTP timeout. ([#7556](https://github.com/databricks/cli/pull/7556)) +* `databricks fs` commands retry reads (stat, list, open) and deletes on DBFS and Volumes when the server returns a transient error or does not respond within the HTTP timeout. ([#7556](https://github.com/databricks/cli/pull/7556)) diff --git a/acceptance/cmd/fs/retry-504/out.test.toml b/acceptance/cmd/fs/retry-504/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/retry-504/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/retry-504/output.txt b/acceptance/cmd/fs/retry-504/output.txt new file mode 100644 index 00000000000..9133ded87a7 --- /dev/null +++ b/acceptance/cmd/fs/retry-504/output.txt @@ -0,0 +1,19 @@ + +=== A 504 on list is retried + +>>> [CLI] fs ls dbfs:/tmp/fs-retry-504-[UNIQUE_NAME] +Warn: retrying after 504 Gateway Timeout from GET /api/2.0/dbfs/list +hello.txt + +=== A 504 on get-status is retried + +>>> [CLI] fs cat dbfs:/tmp/fs-retry-504-[UNIQUE_NAME]/hello.txt +Warn: retrying after 504 Gateway Timeout from GET /api/2.0/dbfs/get-status: dbfs open: Fault injected by test. +hello + +=== A 504 on delete is retried + +>>> [CLI] fs rm dbfs:/tmp/fs-retry-504-[UNIQUE_NAME]/hello.txt +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/dbfs/delete + +>>> [CLI] fs ls dbfs:/tmp/fs-retry-504-[UNIQUE_NAME] diff --git a/acceptance/cmd/fs/retry-504/script b/acceptance/cmd/fs/retry-504/script new file mode 100644 index 00000000000..8ed1816830f --- /dev/null +++ b/acceptance/cmd/fs/retry-504/script @@ -0,0 +1,25 @@ +# A 504 the SDK does not retry itself is retried by the fs commands for reads and deletes. +DBFS="dbfs:/tmp/fs-retry-504-${UNIQUE_NAME}" + +cleanup() { + $CLI fs rm -r ${DBFS} 2>/dev/null || true + rm -f hello.txt +} +trap cleanup EXIT + +echo hello > hello.txt +$CLI fs mkdir ${DBFS} +$CLI fs cp hello.txt ${DBFS}/hello.txt > /dev/null + +title "A 504 on list is retried" +fault.py "GET /api/2.0/dbfs/list" 504 0 1 +trace $CLI fs ls ${DBFS} + +title "A 504 on get-status is retried" +fault.py "GET /api/2.0/dbfs/get-status" 504 0 1 +trace $CLI fs cat ${DBFS}/hello.txt + +title "A 504 on delete is retried" +fault.py "POST /api/2.0/dbfs/delete" 504 0 1 +trace $CLI fs rm ${DBFS}/hello.txt +trace $CLI fs ls ${DBFS} diff --git a/acceptance/cmd/fs/retry-504/test.toml b/acceptance/cmd/fs/retry-504/test.toml new file mode 100644 index 00000000000..89b52a06240 --- /dev/null +++ b/acceptance/cmd/fs/retry-504/test.toml @@ -0,0 +1,2 @@ +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index af7d37166d5..5a42f41b903 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -54,7 +54,7 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er // Writes are not retried: a write without overwrite may fail on the file its own earlier // attempt created, and an overwrite would need to buffer the streamed source to replay it. - return retry.ReadFiler(f), path, nil + return retry.FilerExceptWrites(f), path, nil } const dbfsPrefix string = "dbfs:" diff --git a/libs/retry/filer.go b/libs/retry/filer.go index 6ebd529ddee..7dc3b946f29 100644 --- a/libs/retry/filer.go +++ b/libs/retry/filer.go @@ -15,8 +15,8 @@ import ( type retryingFiler struct { filer.Filer - // readsOnly passes Write and Delete through unchanged. - readsOnly bool + // skipWrites passes Write through unchanged. + skipWrites bool } // Filer wraps f so that Write, Read, ReadDir, Stat and Delete retry on transient errors. @@ -28,11 +28,11 @@ func Filer(f filer.Filer) filer.Filer { return &retryingFiler{Filer: f} } -// ReadFiler wraps f so that only Read, ReadDir and Stat retry on transient errors. +// FilerExceptWrites is like [Filer] but passes Write through without retrying. // It suits callers that stream arbitrarily large bodies to Write: retrying those would mean // buffering each non-seekable body in memory to replay it. -func ReadFiler(f filer.Filer) filer.Filer { - return &retryingFiler{Filer: f, readsOnly: true} +func FilerExceptWrites(f filer.Filer) filer.Filer { + return &retryingFiler{Filer: f, skipWrites: true} } // Unwrap returns the wrapped filer. @@ -41,7 +41,7 @@ func (r *retryingFiler) Unwrap() filer.Filer { } func (r *retryingFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...filer.WriteMode) error { - if r.readsOnly || !slices.Contains(mode, filer.OverwriteIfExists) { + if r.skipWrites || !slices.Contains(mode, filer.OverwriteIfExists) { return r.Filer.Write(ctx, path, reader, mode...) } @@ -93,9 +93,6 @@ func (r *retryingFiler) Stat(ctx context.Context, path string) (fs.FileInfo, err } func (r *retryingFiler) Delete(ctx context.Context, path string, mode ...filer.DeleteMode) error { - if r.readsOnly { - return r.Filer.Delete(ctx, path, mode...) - } retried := false _, err := With(ctx, IsTransient, func() (struct{}, error) { err := r.Filer.Delete(ctx, path, mode...) diff --git a/libs/retry/filer_test.go b/libs/retry/filer_test.go index 4faf4e0e9da..61550ec8204 100644 --- a/libs/retry/filer_test.go +++ b/libs/retry/filer_test.go @@ -167,23 +167,22 @@ func (f *sequenceFiler) Delete(ctx context.Context, path string, mode ...filer.D return err } -func TestReadFilerRetriesReads(t *testing.T) { - ctx := testContext(t) +func TestFilerExceptWritesRetriesReads(t *testing.T) { f := &readFiler{failures: 1, err: errTransient} - _, err := ReadFiler(f).Stat(ctx, "a") + _, err := FilerExceptWrites(f).Stat(testContext(t), "a") require.NoError(t, err) assert.Equal(t, 2, f.calls) } -func TestReadFilerDoesNotRetryWrites(t *testing.T) { +func TestFilerExceptWritesDoesNotRetryWrites(t *testing.T) { f := &flakyFiler{failures: 100, err: errTransient} - err := ReadFiler(f).Write(testContext(t), "a", strings.NewReader("x"), filer.OverwriteIfExists) + err := FilerExceptWrites(f).Write(testContext(t), "a", strings.NewReader("x"), filer.OverwriteIfExists) require.Error(t, err) assert.Len(t, f.bodies, 1) } -func TestReadFilerDoesNotRetryDelete(t *testing.T) { +func TestFilerExceptWritesRetriesDelete(t *testing.T) { f := &readFiler{failures: 1, err: errTransient} - require.Error(t, ReadFiler(f).Delete(testContext(t), "a")) - assert.Equal(t, 1, f.calls) + require.NoError(t, FilerExceptWrites(f).Delete(testContext(t), "a")) + assert.Equal(t, 2, f.calls) }