From 607ee142efe92c42c922e914c9292ee9d783ba14 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:17:17 +0000 Subject: [PATCH] Retry overwriting file writes on unexpected EOF Writes with OverwriteIfExists are idempotent and already retry 5xx and the SDK inactivity timeout. Also retry the transport errors reads retry (unexpected EOF, REFUSED_STREAM): import-file uploads fail with "unexpected EOF" when the connection is cut mid-request. Co-authored-by: Isaac --- libs/retry/filer.go | 3 ++- libs/retry/filer_test.go | 9 +++++++++ libs/retry/retry.go | 19 +++++++++++-------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/libs/retry/filer.go b/libs/retry/filer.go index 7dc3b946f29..f6365cf47c9 100644 --- a/libs/retry/filer.go +++ b/libs/retry/filer.go @@ -64,7 +64,8 @@ func (r *retryingFiler) Write(ctx context.Context, path string, reader io.Reader rewind = func() (io.Reader, error) { return bytes.NewReader(body), nil } } - _, err := With(ctx, IsTransient, func() (struct{}, error) { + // An overwrite is idempotent, so it can also retry the transport errors that IsTransientRead covers. + _, err := With(ctx, IsTransientRead, func() (struct{}, error) { attempt, err := rewind() if err != nil { return struct{}{}, err diff --git a/libs/retry/filer_test.go b/libs/retry/filer_test.go index 61550ec8204..d8fac4c2a8c 100644 --- a/libs/retry/filer_test.go +++ b/libs/retry/filer_test.go @@ -6,6 +6,7 @@ import ( "io" "io/fs" "net/http" + "net/url" "strings" "testing" @@ -62,6 +63,14 @@ func TestFilerRewindsToInitialReaderPosition(t *testing.T) { assert.Equal(t, []string{"hello", "hello"}, f.bodies) } +func TestFilerRetriesWriteOnUnexpectedEOF(t *testing.T) { + eof := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: io.ErrUnexpectedEOF} + f := &flakyFiler{failures: 1, err: eof} + err := Filer(f).Write(testContext(t), "a", strings.NewReader("hello"), filer.OverwriteIfExists) + require.NoError(t, err) + assert.Equal(t, []string{"hello", "hello"}, f.bodies) +} + func TestFilerGivesUp(t *testing.T) { f := &flakyFiler{failures: 100, err: errTransient} err := Filer(f).Write(testContext(t), "a", strings.NewReader("x"), filer.OverwriteIfExists) diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 454a309016e..95bf5a7250e 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -73,18 +73,21 @@ func isInactivityTimeout(err error) bool { return strings.HasPrefix(msg, "request timed out after ") && strings.HasSuffix(msg, " of inactivity") } -// transientReadMessages are the transport errors, matched by message, that are worth retrying on a read: +// transientReadMessages are the transport errors, matched by message, that are worth retrying on +// an idempotent call: // - REFUSED_STREAM, the HTTP/2 refusal in issue #6472, where the server refused the stream before // processing the request. -// - unexpected EOF, where the connection was cut mid-response. Observed on GET /clusters/get. +// - unexpected EOF, where the connection was cut mid-response. Observed on GET /clusters/get and +// POST /workspace-files/import-file. var transientReadMessages = []string{"REFUSED_STREAM", "unexpected EOF"} -// IsTransientRead returns true for errors worth retrying on an idempotent read: the transient -// server errors IsTransient covers, plus the transport errors in transientReadMessages. The SDK -// already retries the transport errors it recognizes (connection reset/refused, i/o timeout, -// TLS handshake) for its full RetryTimeout, so retrying every *url.Error would restart that -// multi-minute loop; these are the gaps it leaves. net/http bundles unexported http2 -// types, so we match the message, as the SDK does for its own transient url errors. +// IsTransientRead returns true for errors worth retrying on an idempotent call (a read or an +// overwriting write): the transient server errors IsTransient covers, plus the transport errors +// in transientReadMessages. The SDK already retries the transport errors it recognizes +// (connection reset/refused, i/o timeout, TLS handshake) for its full RetryTimeout, so retrying +// every *url.Error would restart that multi-minute loop; these are the gaps it leaves. net/http +// bundles unexported http2 types, so we match the message, as the SDK does for its own transient +// url errors. func IsTransientRead(ctx context.Context, err error) bool { if IsTransient(ctx, err) { return true