Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion libs/retry/filer.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ func (r *retryingFiler) Write(ctx context.Context, path string, reader io.Reader
rewind = func() (io.Reader, error) { return bytes.NewReader(body), nil }
}

_, err := With(ctx, IsTransient, func() (struct{}, error) {
// An overwrite is idempotent, so it can also retry the transport errors that IsTransientRead covers.
_, err := With(ctx, IsTransientRead, func() (struct{}, error) {
attempt, err := rewind()
if err != nil {
return struct{}{}, err
Expand Down
9 changes: 9 additions & 0 deletions libs/retry/filer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"io"
"io/fs"
"net/http"
"net/url"
"strings"
"testing"

Expand Down Expand Up @@ -62,6 +63,14 @@ func TestFilerRewindsToInitialReaderPosition(t *testing.T) {
assert.Equal(t, []string{"hello", "hello"}, f.bodies)
}

func TestFilerRetriesWriteOnUnexpectedEOF(t *testing.T) {
eof := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: io.ErrUnexpectedEOF}
f := &flakyFiler{failures: 1, err: eof}
err := Filer(f).Write(testContext(t), "a", strings.NewReader("hello"), filer.OverwriteIfExists)
require.NoError(t, err)
assert.Equal(t, []string{"hello", "hello"}, f.bodies)
}

func TestFilerGivesUp(t *testing.T) {
f := &flakyFiler{failures: 100, err: errTransient}
err := Filer(f).Write(testContext(t), "a", strings.NewReader("x"), filer.OverwriteIfExists)
Expand Down
19 changes: 11 additions & 8 deletions libs/retry/retry.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,18 +73,21 @@ func isInactivityTimeout(err error) bool {
return strings.HasPrefix(msg, "request timed out after ") && strings.HasSuffix(msg, " of inactivity")
}

// transientReadMessages are the transport errors, matched by message, that are worth retrying on a read:
// transientReadMessages are the transport errors, matched by message, that are worth retrying on
// an idempotent call:
// - REFUSED_STREAM, the HTTP/2 refusal in issue #6472, where the server refused the stream before
// processing the request.
// - unexpected EOF, where the connection was cut mid-response. Observed on GET /clusters/get.
// - unexpected EOF, where the connection was cut mid-response. Observed on GET /clusters/get and
// POST /workspace-files/import-file.
var transientReadMessages = []string{"REFUSED_STREAM", "unexpected EOF"}

// IsTransientRead returns true for errors worth retrying on an idempotent read: the transient
// server errors IsTransient covers, plus the transport errors in transientReadMessages. The SDK
// already retries the transport errors it recognizes (connection reset/refused, i/o timeout,
// TLS handshake) for its full RetryTimeout, so retrying every *url.Error would restart that
// multi-minute loop; these are the gaps it leaves. net/http bundles unexported http2
// types, so we match the message, as the SDK does for its own transient url errors.
// IsTransientRead returns true for errors worth retrying on an idempotent call (a read or an
// overwriting write): the transient server errors IsTransient covers, plus the transport errors
// in transientReadMessages. The SDK already retries the transport errors it recognizes
// (connection reset/refused, i/o timeout, TLS handshake) for its full RetryTimeout, so retrying
// every *url.Error would restart that multi-minute loop; these are the gaps it leaves. net/http
// bundles unexported http2 types, so we match the message, as the SDK does for its own transient
// url errors.
func IsTransientRead(ctx context.Context, err error) bool {
if IsTransient(ctx, err) {
return true
Expand Down
Loading