Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/7556.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks fs` commands retry reads (stat, list, open) and deletes on DBFS and Volumes when the server returns a transient error or does not respond within the HTTP timeout. ([#7556](https://github.com/databricks/cli/pull/7556))
1 change: 1 addition & 0 deletions acceptance/cmd/fs/retry-504/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
19 changes: 19 additions & 0 deletions acceptance/cmd/fs/retry-504/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

=== A 504 on list is retried

>>> [CLI] fs ls dbfs:/tmp/fs-retry-504-[UNIQUE_NAME]
Warn: retrying after 504 Gateway Timeout from GET /api/2.0/dbfs/list
hello.txt

=== A 504 on get-status is retried

>>> [CLI] fs cat dbfs:/tmp/fs-retry-504-[UNIQUE_NAME]/hello.txt
Warn: retrying after 504 Gateway Timeout from GET /api/2.0/dbfs/get-status: dbfs open: Fault injected by test.
hello

=== A 504 on delete is retried

>>> [CLI] fs rm dbfs:/tmp/fs-retry-504-[UNIQUE_NAME]/hello.txt
Warn: retrying after 504 Gateway Timeout from POST /api/2.0/dbfs/delete

>>> [CLI] fs ls dbfs:/tmp/fs-retry-504-[UNIQUE_NAME]
25 changes: 25 additions & 0 deletions acceptance/cmd/fs/retry-504/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# A 504 the SDK does not retry itself is retried by the fs commands for reads and deletes.
DBFS="dbfs:/tmp/fs-retry-504-${UNIQUE_NAME}"

cleanup() {
$CLI fs rm -r ${DBFS} 2>/dev/null || true
rm -f hello.txt
}
trap cleanup EXIT

echo hello > hello.txt
$CLI fs mkdir ${DBFS}
$CLI fs cp hello.txt ${DBFS}/hello.txt > /dev/null

title "A 504 on list is retried"
fault.py "GET /api/2.0/dbfs/list" 504 0 1
trace $CLI fs ls ${DBFS}

title "A 504 on get-status is retried"
fault.py "GET /api/2.0/dbfs/get-status" 504 0 1
trace $CLI fs cat ${DBFS}/hello.txt

title "A 504 on delete is retried"
fault.py "POST /api/2.0/dbfs/delete" 504 0 1
trace $CLI fs rm ${DBFS}/hello.txt
trace $CLI fs ls ${DBFS}
2 changes: 2 additions & 0 deletions acceptance/cmd/fs/retry-504/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
18 changes: 13 additions & 5 deletions cmd/fs/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/filer/completer"
"github.com/databricks/cli/libs/retry"
"github.com/spf13/cobra"
)

Expand Down Expand Up @@ -39,14 +40,21 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er
w := cmdctx.WorkspaceClient(ctx)

// If the specified path has the "Volumes" or "Skills" prefix, use the Files API.
// Otherwise the file is a dbfs file, and uses the DBFS APIs.
var f filer.Filer
var err error
if strings.HasPrefix(path, "/Volumes/") || strings.HasPrefix(path, "/Skills/") {
f, err := filer.NewFilesClient(ctx, w, "/")
return f, path, err
f, err = filer.NewFilesClient(ctx, w, "/")
} else {
f, err = filer.NewDbfsClient(w, "/")
}
if err != nil {
return nil, "", err
}

// The file is a dbfs file, and uses the DBFS APIs
f, err := filer.NewDbfsClient(w, "/")
return f, path, err
// Writes are not retried: a write without overwrite may fail on the file its own earlier
// attempt created, and an overwrite would need to buffer the streamed source to replay it.
return retry.FilerExceptWrites(f), path, nil
}

const dbfsPrefix string = "dbfs:"
Expand Down
4 changes: 3 additions & 1 deletion cmd/fs/helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,9 @@ func TestFilerForPathForDbfsScheme(t *testing.T) {
f, path, err := filerForPath(ctx, tc.fullPath)
require.NoError(t, err)
assert.Equal(t, tc.path, path)
assert.IsType(t, tc.filer, f)
unwrapper, ok := f.(interface{ Unwrap() filer.Filer })
require.True(t, ok, "remote filers retry reads")
assert.IsType(t, tc.filer, unwrapper.Unwrap())
})
}
}
Expand Down
17 changes: 16 additions & 1 deletion libs/retry/filer.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ import (
// retryingFiler retries calls that fail with a transient server error and are safe to repeat.
type retryingFiler struct {
filer.Filer

// skipWrites passes Write through unchanged.
skipWrites bool
}

// Filer wraps f so that Write, Read, ReadDir, Stat and Delete retry on transient errors.
Expand All @@ -25,8 +28,20 @@ func Filer(f filer.Filer) filer.Filer {
return &retryingFiler{Filer: f}
}

// FilerExceptWrites is like [Filer] but passes Write through without retrying.
// It suits callers that stream arbitrarily large bodies to Write: retrying those would mean
// buffering each non-seekable body in memory to replay it.
func FilerExceptWrites(f filer.Filer) filer.Filer {
return &retryingFiler{Filer: f, skipWrites: true}
}

// Unwrap returns the wrapped filer.
func (r *retryingFiler) Unwrap() filer.Filer {
return r.Filer
}

func (r *retryingFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...filer.WriteMode) error {
if !slices.Contains(mode, filer.OverwriteIfExists) {
if r.skipWrites || !slices.Contains(mode, filer.OverwriteIfExists) {
return r.Filer.Write(ctx, path, reader, mode...)
}

Expand Down
20 changes: 20 additions & 0 deletions libs/retry/filer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,3 +166,23 @@ func (f *sequenceFiler) Delete(ctx context.Context, path string, mode ...filer.D
f.errs = f.errs[1:]
return err
}

func TestFilerExceptWritesRetriesReads(t *testing.T) {
f := &readFiler{failures: 1, err: errTransient}
_, err := FilerExceptWrites(f).Stat(testContext(t), "a")
require.NoError(t, err)
assert.Equal(t, 2, f.calls)
}

func TestFilerExceptWritesDoesNotRetryWrites(t *testing.T) {
f := &flakyFiler{failures: 100, err: errTransient}
err := FilerExceptWrites(f).Write(testContext(t), "a", strings.NewReader("x"), filer.OverwriteIfExists)
require.Error(t, err)
assert.Len(t, f.bodies, 1)
}

func TestFilerExceptWritesRetriesDelete(t *testing.T) {
f := &readFiler{failures: 1, err: errTransient}
require.NoError(t, FilerExceptWrites(f).Delete(testContext(t), "a"))
assert.Equal(t, 2, f.calls)
}
Loading