diff --git a/.nextchanges/cli/7556.md b/.nextchanges/cli/7556.md new file mode 100644 index 00000000000..f95e30adf3c --- /dev/null +++ b/.nextchanges/cli/7556.md @@ -0,0 +1 @@ +* `databricks fs` commands retry reads (stat, list, open) and deletes on DBFS and Volumes when the server returns a transient error or does not respond within the HTTP timeout. ([#7556](https://github.com/databricks/cli/pull/7556)) diff --git a/acceptance/cmd/fs/retry-504/out.test.toml b/acceptance/cmd/fs/retry-504/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/retry-504/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/retry-504/output.txt b/acceptance/cmd/fs/retry-504/output.txt new file mode 100644 index 00000000000..9133ded87a7 --- /dev/null +++ b/acceptance/cmd/fs/retry-504/output.txt @@ -0,0 +1,19 @@ + +=== A 504 on list is retried + +>>> [CLI] fs ls dbfs:/tmp/fs-retry-504-[UNIQUE_NAME] +Warn: retrying after 504 Gateway Timeout from GET /api/2.0/dbfs/list +hello.txt + +=== A 504 on get-status is retried + +>>> [CLI] fs cat dbfs:/tmp/fs-retry-504-[UNIQUE_NAME]/hello.txt +Warn: retrying after 504 Gateway Timeout from GET /api/2.0/dbfs/get-status: dbfs open: Fault injected by test. +hello + +=== A 504 on delete is retried + +>>> [CLI] fs rm dbfs:/tmp/fs-retry-504-[UNIQUE_NAME]/hello.txt +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/dbfs/delete + +>>> [CLI] fs ls dbfs:/tmp/fs-retry-504-[UNIQUE_NAME] diff --git a/acceptance/cmd/fs/retry-504/script b/acceptance/cmd/fs/retry-504/script new file mode 100644 index 00000000000..8ed1816830f --- /dev/null +++ b/acceptance/cmd/fs/retry-504/script @@ -0,0 +1,25 @@ +# A 504 the SDK does not retry itself is retried by the fs commands for reads and deletes. +DBFS="dbfs:/tmp/fs-retry-504-${UNIQUE_NAME}" + +cleanup() { + $CLI fs rm -r ${DBFS} 2>/dev/null || true + rm -f hello.txt +} +trap cleanup EXIT + +echo hello > hello.txt +$CLI fs mkdir ${DBFS} +$CLI fs cp hello.txt ${DBFS}/hello.txt > /dev/null + +title "A 504 on list is retried" +fault.py "GET /api/2.0/dbfs/list" 504 0 1 +trace $CLI fs ls ${DBFS} + +title "A 504 on get-status is retried" +fault.py "GET /api/2.0/dbfs/get-status" 504 0 1 +trace $CLI fs cat ${DBFS}/hello.txt + +title "A 504 on delete is retried" +fault.py "POST /api/2.0/dbfs/delete" 504 0 1 +trace $CLI fs rm ${DBFS}/hello.txt +trace $CLI fs ls ${DBFS} diff --git a/acceptance/cmd/fs/retry-504/test.toml b/acceptance/cmd/fs/retry-504/test.toml new file mode 100644 index 00000000000..89b52a06240 --- /dev/null +++ b/acceptance/cmd/fs/retry-504/test.toml @@ -0,0 +1,2 @@ +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index d2e98563314..5a42f41b903 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/filer/completer" + "github.com/databricks/cli/libs/retry" "github.com/spf13/cobra" ) @@ -39,14 +40,21 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er w := cmdctx.WorkspaceClient(ctx) // If the specified path has the "Volumes" or "Skills" prefix, use the Files API. + // Otherwise the file is a dbfs file, and uses the DBFS APIs. + var f filer.Filer + var err error if strings.HasPrefix(path, "/Volumes/") || strings.HasPrefix(path, "/Skills/") { - f, err := filer.NewFilesClient(ctx, w, "/") - return f, path, err + f, err = filer.NewFilesClient(ctx, w, "/") + } else { + f, err = filer.NewDbfsClient(w, "/") + } + if err != nil { + return nil, "", err } - // The file is a dbfs file, and uses the DBFS APIs - f, err := filer.NewDbfsClient(w, "/") - return f, path, err + // Writes are not retried: a write without overwrite may fail on the file its own earlier + // attempt created, and an overwrite would need to buffer the streamed source to replay it. + return retry.FilerExceptWrites(f), path, nil } const dbfsPrefix string = "dbfs:" diff --git a/cmd/fs/helpers_test.go b/cmd/fs/helpers_test.go index ee1a88e23fb..dc54914ee0c 100644 --- a/cmd/fs/helpers_test.go +++ b/cmd/fs/helpers_test.go @@ -62,7 +62,9 @@ func TestFilerForPathForDbfsScheme(t *testing.T) { f, path, err := filerForPath(ctx, tc.fullPath) require.NoError(t, err) assert.Equal(t, tc.path, path) - assert.IsType(t, tc.filer, f) + unwrapper, ok := f.(interface{ Unwrap() filer.Filer }) + require.True(t, ok, "remote filers retry reads") + assert.IsType(t, tc.filer, unwrapper.Unwrap()) }) } } diff --git a/libs/retry/filer.go b/libs/retry/filer.go index 11b991b18fc..7dc3b946f29 100644 --- a/libs/retry/filer.go +++ b/libs/retry/filer.go @@ -14,6 +14,9 @@ import ( // retryingFiler retries calls that fail with a transient server error and are safe to repeat. type retryingFiler struct { filer.Filer + + // skipWrites passes Write through unchanged. + skipWrites bool } // Filer wraps f so that Write, Read, ReadDir, Stat and Delete retry on transient errors. @@ -25,8 +28,20 @@ func Filer(f filer.Filer) filer.Filer { return &retryingFiler{Filer: f} } +// FilerExceptWrites is like [Filer] but passes Write through without retrying. +// It suits callers that stream arbitrarily large bodies to Write: retrying those would mean +// buffering each non-seekable body in memory to replay it. +func FilerExceptWrites(f filer.Filer) filer.Filer { + return &retryingFiler{Filer: f, skipWrites: true} +} + +// Unwrap returns the wrapped filer. +func (r *retryingFiler) Unwrap() filer.Filer { + return r.Filer +} + func (r *retryingFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...filer.WriteMode) error { - if !slices.Contains(mode, filer.OverwriteIfExists) { + if r.skipWrites || !slices.Contains(mode, filer.OverwriteIfExists) { return r.Filer.Write(ctx, path, reader, mode...) } diff --git a/libs/retry/filer_test.go b/libs/retry/filer_test.go index 90e57faedb9..61550ec8204 100644 --- a/libs/retry/filer_test.go +++ b/libs/retry/filer_test.go @@ -166,3 +166,23 @@ func (f *sequenceFiler) Delete(ctx context.Context, path string, mode ...filer.D f.errs = f.errs[1:] return err } + +func TestFilerExceptWritesRetriesReads(t *testing.T) { + f := &readFiler{failures: 1, err: errTransient} + _, err := FilerExceptWrites(f).Stat(testContext(t), "a") + require.NoError(t, err) + assert.Equal(t, 2, f.calls) +} + +func TestFilerExceptWritesDoesNotRetryWrites(t *testing.T) { + f := &flakyFiler{failures: 100, err: errTransient} + err := FilerExceptWrites(f).Write(testContext(t), "a", strings.NewReader("x"), filer.OverwriteIfExists) + require.Error(t, err) + assert.Len(t, f.bodies, 1) +} + +func TestFilerExceptWritesRetriesDelete(t *testing.T) { + f := &readFiler{failures: 1, err: errTransient} + require.NoError(t, FilerExceptWrites(f).Delete(testContext(t), "a")) + assert.Equal(t, 2, f.calls) +}