diff --git a/libs/retry/filer.go b/libs/retry/filer.go index 7dc3b946f29..f6365cf47c9 100644 --- a/libs/retry/filer.go +++ b/libs/retry/filer.go @@ -64,7 +64,8 @@ func (r *retryingFiler) Write(ctx context.Context, path string, reader io.Reader rewind = func() (io.Reader, error) { return bytes.NewReader(body), nil } } - _, err := With(ctx, IsTransient, func() (struct{}, error) { + // An overwrite is idempotent, so it can also retry the transport errors that IsTransientRead covers. + _, err := With(ctx, IsTransientRead, func() (struct{}, error) { attempt, err := rewind() if err != nil { return struct{}{}, err diff --git a/libs/retry/filer_test.go b/libs/retry/filer_test.go index 61550ec8204..d8fac4c2a8c 100644 --- a/libs/retry/filer_test.go +++ b/libs/retry/filer_test.go @@ -6,6 +6,7 @@ import ( "io" "io/fs" "net/http" + "net/url" "strings" "testing" @@ -62,6 +63,14 @@ func TestFilerRewindsToInitialReaderPosition(t *testing.T) { assert.Equal(t, []string{"hello", "hello"}, f.bodies) } +func TestFilerRetriesWriteOnUnexpectedEOF(t *testing.T) { + eof := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: io.ErrUnexpectedEOF} + f := &flakyFiler{failures: 1, err: eof} + err := Filer(f).Write(testContext(t), "a", strings.NewReader("hello"), filer.OverwriteIfExists) + require.NoError(t, err) + assert.Equal(t, []string{"hello", "hello"}, f.bodies) +} + func TestFilerGivesUp(t *testing.T) { f := &flakyFiler{failures: 100, err: errTransient} err := Filer(f).Write(testContext(t), "a", strings.NewReader("x"), filer.OverwriteIfExists) diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 454a309016e..95bf5a7250e 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -73,18 +73,21 @@ func isInactivityTimeout(err error) bool { return strings.HasPrefix(msg, "request timed out after ") && strings.HasSuffix(msg, " of inactivity") } -// transientReadMessages are the transport errors, matched by message, that are worth retrying on a read: +// transientReadMessages are the transport errors, matched by message, that are worth retrying on +// an idempotent call: // - REFUSED_STREAM, the HTTP/2 refusal in issue #6472, where the server refused the stream before // processing the request. -// - unexpected EOF, where the connection was cut mid-response. Observed on GET /clusters/get. +// - unexpected EOF, where the connection was cut mid-response. Observed on GET /clusters/get and +// POST /workspace-files/import-file. var transientReadMessages = []string{"REFUSED_STREAM", "unexpected EOF"} -// IsTransientRead returns true for errors worth retrying on an idempotent read: the transient -// server errors IsTransient covers, plus the transport errors in transientReadMessages. The SDK -// already retries the transport errors it recognizes (connection reset/refused, i/o timeout, -// TLS handshake) for its full RetryTimeout, so retrying every *url.Error would restart that -// multi-minute loop; these are the gaps it leaves. net/http bundles unexported http2 -// types, so we match the message, as the SDK does for its own transient url errors. +// IsTransientRead returns true for errors worth retrying on an idempotent call (a read or an +// overwriting write): the transient server errors IsTransient covers, plus the transport errors +// in transientReadMessages. The SDK already retries the transport errors it recognizes +// (connection reset/refused, i/o timeout, TLS handshake) for its full RetryTimeout, so retrying +// every *url.Error would restart that multi-minute loop; these are the gaps it leaves. net/http +// bundles unexported http2 types, so we match the message, as the SDK does for its own transient +// url errors. func IsTransientRead(ctx context.Context, err error) bool { if IsTransient(ctx, err) { return true