From e2609a569f7607fa29dd6c8434d7f7611f600703 Mon Sep 17 00:00:00 2001 From: Anton Timofieiev Date: Thu, 12 Nov 2026 22:31:55 +0100 Subject: [PATCH 1/8] fix(carbonapi): recover store errors within answered replica groups When a replica group (stores sharing dc and cluster) answered the request, an error from another member of that group no longer fails a required fetch: the group's answer stands for the whole group. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/app/carbonapi/partial_render_test.go | 11 ++- .../carbonapi/replica_group_render_test.go | 94 +++++++++++++++++++ pkg/app/carbonapi/upstream.go | 4 +- pkg/backend/render_fanin.go | 71 ++++++++++++++ pkg/backend/render_fanin_test.go | 91 ++++++++++++++++++ pkg/backend/rpc.go | 20 +--- pkg/backend/stable_render.go | 23 ++--- 7 files changed, 279 insertions(+), 35 deletions(-) create mode 100644 pkg/app/carbonapi/replica_group_render_test.go create mode 100644 pkg/backend/render_fanin.go create mode 100644 pkg/backend/render_fanin_test.go diff --git a/pkg/app/carbonapi/partial_render_test.go b/pkg/app/carbonapi/partial_render_test.go index 35b882a4..109f8343 100644 --- a/pkg/app/carbonapi/partial_render_test.go +++ b/pkg/app/carbonapi/partial_render_test.go @@ -29,6 +29,9 @@ type partialRenderBackend struct { path string err error empty bool + // Leave cluster empty for the strict, ungrouped-backend regression tests. + cluster string + dc string } func (b *partialRenderBackend) Contains(paths []string) bool { @@ -39,9 +42,11 @@ func (b *partialRenderBackend) Contains(paths []string) bool { } return false } -func (b *partialRenderBackend) Logger() *zap.Logger { return zap.NewNop() } -func (b *partialRenderBackend) GetServerAddress() string { return b.path } -func (b *partialRenderBackend) BackendInfo() (string, string, string) { return b.path, "test", "test" } +func (b *partialRenderBackend) Logger() *zap.Logger { return zap.NewNop() } +func (b *partialRenderBackend) GetServerAddress() string { return b.path } +func (b *partialRenderBackend) BackendInfo() (string, string, string) { + return b.path, b.cluster, b.dc +} func (b *partialRenderBackend) Render(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { if b.err != nil || b.empty { return nil, b.err diff --git a/pkg/app/carbonapi/replica_group_render_test.go b/pkg/app/carbonapi/replica_group_render_test.go new file mode 100644 index 00000000..8392593a --- /dev/null +++ b/pkg/app/carbonapi/replica_group_render_test.go @@ -0,0 +1,94 @@ +package carbonapi + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "syscall" + "testing" + + "github.com/bookingcom/carbonapi/pkg/carbonapipb" + "github.com/bookingcom/carbonapi/pkg/types" + "go.uber.org/zap" +) + +func TestRenderReplicaGroupFailure(t *testing.T) { + refused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED} + dialTimeout := &net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded} + ioTimeout := &net.OpError{Op: "read", Net: "tcp", Err: syscall.ETIMEDOUT} + for _, tc := range []struct { + name string + failure error + cluster string + peerCluster string + peerDC string + peerError error + peerEmpty bool + wantError bool + wantData bool + }{ + {name: "Vanguard_one_refused_93_not_found", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: types.ErrMetricsNotFound}, + {name: "dial_timeout_and_not_found", failure: dialTimeout, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: types.ErrMetricsNotFound}, + {name: "io_timeout_and_not_found", failure: ioTimeout, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: types.ErrMetricsNotFound}, + {name: "wrapped_not_found", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: fmt.Errorf("store: %w", types.ErrMetricsNotFound)}, + {name: "empty_success", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerEmpty: true}, + {name: "all_group_members_refused", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: refused, wantError: true}, + {name: "data_replica_wins", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", wantData: true}, + {name: "other_group_cannot_recover", failure: refused, cluster: "cy", peerCluster: "infra", peerDC: "dc1", peerError: types.ErrMetricsNotFound, wantError: true}, + {name: "other_dc_cannot_recover", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc4", peerError: types.ErrMetricsNotFound, wantError: true}, + {name: "ungrouped_unchanged", failure: refused, peerError: types.ErrMetricsNotFound, wantError: true}, + } { + t.Run(tc.name, func(t *testing.T) { + stores := []*partialRenderBackend{{path: "m.ok", err: tc.failure, cluster: tc.cluster, dc: "dc1"}} + for range 2 { + stores = append(stores, &partialRenderBackend{path: "m.ok", err: tc.peerError, empty: tc.peerEmpty, cluster: tc.peerCluster, dc: tc.peerDC}) + } + // The canary had 94 replies: one refusal and 93 not-found, + // including the two peers in the refused store's replica group. + for i := range 91 { + stores = append(stores, &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: fmt.Sprintf("other-%d", i), dc: "dc1"}) + } + // Even in the same group, success for another target must not hide this + // failed fetch. Path routing excludes this peer from the m.ok request. + stores = append(stores, &partialRenderBackend{path: "m.ko", cluster: tc.cluster, dc: "dc1"}) + app := partialRenderApp(t, stores...) + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + if tc.wantError { + if w.Code != http.StatusInternalServerError { + t.Fatalf("HTTP %d, want 500: %s", w.Code, w.Body.String()) + } + } else { + want := 1.0 + if tc.wantData { + want = 100 + } + assertPartialRenderValue(t, w, want) + } + + // Exercise ordinary and grouped fetches separately; replies for another + // target, group or DC must not establish success for this request. + metrics, err := Render(app.TopLevelDomainCache, nil, false, app.Backends, app.ZipperConfig.RenderReplicaMismatchConfig, + context.Background(), "m.ok", 1794392400, 1794393000, app.ZipperMetrics, zap.NewNop()) + checkReplicaGroupFetch(t, err, len(metrics), tc.wantError, tc.wantData) + replies := sendGroupedRenderRequests(app, context.Background(), []string{"m.ok", "m.ko"}, 1794392400, 1794393000, &carbonapipb.AccessLogDetails{}, zap.NewNop()) + checkReplicaGroupFetch(t, replies[0].error, len(replies[0].data), tc.wantError, tc.wantData) + if replies[1].error != nil || len(replies[1].data) != 1 { + t.Fatalf("independent input changed: %+v", replies[1]) + } + }) + } +} + +func checkReplicaGroupFetch(t *testing.T, err error, nMetrics int, wantError, wantData bool) { + t.Helper() + var notFound types.ErrNotFound + fatal := err != nil && !errors.As(err, ¬Found) + if fatal != wantError { + t.Fatalf("fetch error = %v, want fatal %v", err, wantError) + } + if (nMetrics != 0) != wantData { + t.Fatalf("got %d metrics, want data %v", nMetrics, wantData) + } +} diff --git a/pkg/app/carbonapi/upstream.go b/pkg/app/carbonapi/upstream.go index cfce6cfc..29f48f24 100644 --- a/pkg/app/carbonapi/upstream.go +++ b/pkg/app/carbonapi/upstream.go @@ -107,8 +107,8 @@ func Info(cache *expirecache.Cache, TLDPrefixes []tldcache.TopLevelDomainPrefix, return infos, err } -// An empty successful replica reply cannot recover a failed series fetch. -// A replica that supplies data still wins over errors from other replicas. +// Backend fan-in has already recovered errors within answered replica groups. +// Remaining errors need data to recover; an unrelated empty reply is insufficient. func renderErrorsFanIn(metrics []types.Metric, errs []error, nBackends int) error { if len(metrics) == 0 { nBackends = len(errs) diff --git a/pkg/backend/render_fanin.go b/pkg/backend/render_fanin.go new file mode 100644 index 00000000..411d94e6 --- /dev/null +++ b/pkg/backend/render_fanin.go @@ -0,0 +1,71 @@ +package backend + +import ( + "errors" + + "github.com/bookingcom/carbonapi/pkg/types" +) + +// Keep replica replies together until errors have been classified. Only the +// backends actually selected for this request can establish that a group answered. +type renderGroup struct { + n int + cluster string + results chan []types.Metric + errors chan error +} + +func newRenderGroups(backends []Backend) ([]renderGroup, []int) { + groups := make([]renderGroup, 0) + membership := make([]int, len(backends)) + byGroup := make(map[[2]string]int) + for i, b := range backends { + _, cluster, dc := b.BackendInfo() + if cluster == "" { + // Ungrouped backends share channels, but never recover each other's errors. + dc = "" + } + key := [2]string{dc, cluster} + g, ok := byGroup[key] + if !ok { + g = len(groups) + byGroup[key] = g + groups = append(groups, renderGroup{cluster: cluster}) + } + groups[g].n++ + membership[i] = g + } + for i := range groups { + groups[i].results = make(chan []types.Metric, groups[i].n) + groups[i].errors = make(chan error, groups[i].n) + } + return groups, membership +} + +func collectRenderGroups(groups []renderGroup) ([][]types.Metric, []error) { + var messages [][]types.Metric + var errs []error + for _, g := range groups { + start := len(errs) + answered := false + for range g.n { + select { + case metrics := <-g.results: + messages = append(messages, metrics) + answered = true + case err := <-g.errors: + errs = append(errs, err) + var notFound types.ErrNotFound + if errors.As(err, ¬Found) { + answered = true + } + } + } + if g.cluster != "" && answered { + for i := start; i < len(errs); i++ { + errs[i] = types.ErrMetricsNotFound + } + } + } + return messages, errs +} diff --git a/pkg/backend/render_fanin_test.go b/pkg/backend/render_fanin_test.go new file mode 100644 index 00000000..5bea1106 --- /dev/null +++ b/pkg/backend/render_fanin_test.go @@ -0,0 +1,91 @@ +package backend + +import ( + "context" + "errors" + "syscall" + "testing" + "time" + + "github.com/bookingcom/carbonapi/pkg/cfg" + "github.com/bookingcom/carbonapi/pkg/types" + "go.uber.org/zap" +) + +func TestRenderGroupAnswersStayPerTarget(t *testing.T) { + for _, grouped := range []bool{false, true} { + name := "ordinary" + if grouped { + name = "coalesced" + } + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var backends []Backend + for replica := range 3 { + b := coalesceBackend(func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + if replica == 0 { + return nil, syscall.ECONNREFUSED + } + for _, path := range r.Targets { + if path == "failed" { + return nil, syscall.ECONNREFUSED + } + } + var metrics []types.Metric + for _, path := range r.Targets { + if path == "data" { + metrics = append(metrics, coalesceMetrics([]string{path})...) + } + } + if len(metrics) == 0 { + return nil, types.ErrMetricsNotFound + } + return metrics, nil + }, 4) + b.Proc() + backends = append(backends, b) + } + paths := []string{"missing", "failed", "data"} + requests := make([]types.RenderRequest, len(paths)) + choices := make([][]Backend, len(paths)) + for i, path := range paths { + requests[i] = types.NewRenderRequest([]string{path}, 1, 4) + choices[i] = backends + } + var replies []GroupedRenderReply + if grouped { + replies = RendersGrouped(ctx, choices, requests, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + } else { + for _, request := range requests { + metrics, stats, errs := Renders(ctx, backends, request, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + replies = append(replies, GroupedRenderReply{Metrics: metrics, Stats: stats, Errors: errs}) + } + } + for i, reply := range replies { + wantErrors := 3 + if paths[i] == "data" { + wantErrors = 1 + if len(reply.Metrics) != 1 || reply.Metrics[0].Name != "data" || reply.Metrics[0].Values[1] != 42 { + t.Fatalf("data lost: %+v", reply) + } + } else if len(reply.Metrics) != 0 { + t.Fatalf("%s acquired another target's data: %+v", paths[i], reply) + } + if len(reply.Errors) != wantErrors { + t.Fatalf("%s errors = %v, want %d", paths[i], reply.Errors, wantErrors) + } + for _, err := range reply.Errors { + var notFound types.ErrNotFound + if paths[i] == "failed" { + if !errors.Is(err, syscall.ECONNREFUSED) { + t.Fatalf("entirely failed group recovered: %v", reply.Errors) + } + } else if !errors.As(err, ¬Found) { + t.Fatalf("%s retained replica failure: %v", paths[i], reply.Errors) + } + } + } + }) + } +} diff --git a/pkg/backend/rpc.go b/pkg/backend/rpc.go index eb23e463..acb414e1 100644 --- a/pkg/backend/rpc.go +++ b/pkg/backend/rpc.go @@ -21,23 +21,13 @@ func Renders(ctx context.Context, backends []Backend, request types.RenderReques return nil, types.MetricRenderStats{}, nil } - msgCh := make(chan []types.Metric, len(backends)) - errCh := make(chan error, len(backends)) - for _, backend := range backends { - backend.SendRender(ctx, request, msgCh, errCh) - } - - msgs := make([][]types.Metric, 0, len(backends)) - errs := make([]error, 0, len(backends)) - for i := 0; i < len(backends); i++ { - select { - case msg := <-msgCh: - msgs = append(msgs, msg) - case err := <-errCh: - errs = append(errs, err) - } + groups, membership := newRenderGroups(backends) + for i, backend := range backends { + g := groups[membership[i]] + backend.SendRender(ctx, request, g.results, g.errors) } + msgs, errs := collectRenderGroups(groups) metrics, stats := types.MergeMetrics(msgs, replicaMismatchConfig, lg) return metrics, stats, errs } diff --git a/pkg/backend/stable_render.go b/pkg/backend/stable_render.go index a4c9bedd..8ca6c0aa 100644 --- a/pkg/backend/stable_render.go +++ b/pkg/backend/stable_render.go @@ -33,18 +33,18 @@ func RendersGrouped(ctx context.Context, choices [][]Backend, requests []types.R calls []renderDispatch } groups := make(map[*requestQueue[*renderReq]]*group) - results := make([]chan []types.Metric, len(requests)) - errors := make([]chan error, len(requests)) + replicaGroups := make([][]renderGroup, len(requests)) for i, request := range requests { - results[i] = make(chan []types.Metric, len(choices[i])) - errors[i] = make(chan error, len(choices[i])) - for _, b := range choices[i] { + var membership []int + replicaGroups[i], membership = newRenderGroups(choices[i]) + for j, b := range choices[i] { g := groups[b.fastRenderQ] if g == nil { g = &group{backend: b} groups[b.fastRenderQ] = g } - g.calls = append(g.calls, renderDispatch{request, results[i], errors[i]}) + replicas := replicaGroups[i][membership[j]] + g.calls = append(g.calls, renderDispatch{request, replicas.results, replicas.errors}) } } for _, g := range groups { @@ -55,15 +55,8 @@ func RendersGrouped(ctx context.Context, choices [][]Backend, requests []types.R if len(choices[i]) == 0 { continue } - messages := make([][]types.Metric, 0, len(choices[i])) - for range choices[i] { - select { - case metrics := <-results[i]: - messages = append(messages, metrics) - case err := <-errors[i]: - replies[i].Errors = append(replies[i].Errors, err) - } - } + var messages [][]types.Metric + messages, replies[i].Errors = collectRenderGroups(replicaGroups[i]) replies[i].Metrics, replies[i].Stats = types.MergeMetrics(messages, mismatch, lg) } return replies From 50182c38144a4e57d44eae70ddf58b0fef0865ad Mon Sep 17 00:00:00 2001 From: Anton Timofieiev Date: Fri, 13 Nov 2026 10:39:51 +0100 Subject: [PATCH 2/8] fix(backend): retry renders while a store refuses connections A store restart refuses connections for a few seconds. Since the required-fetch rule (#593) and the replica-group recovery, a refused fetch with no answering replica group member fails the render, so every store restart became a burst of 500s. Fetches that failed before the backend sent any response (refused, reset or dial failure inside http.Client.Do) are now repeated against the same backend with a 250 ms doubling backoff capped at 2 s, for up to timeouts.connectRetry (default 10s) and never past the request deadline. Groups that another member already answered are not retried. Failures after the backend answered are not retried. When the window runs out, the required-fetch and replica-group rules apply unchanged. New counters: backend_connect_retries_total and backend_connect_retry_recoveries_total. Co-Authored-By: Claude Opus 5.5 (1M context) --- config/carbonzipper.yaml | 3 + pkg/app/carbonapi/app.go | 5 + pkg/app/carbonapi/connect_retry_test.go | 83 ++++++++ pkg/app/carbonapi/metrics.go | 17 ++ pkg/app/carbonapi/partial_render_test.go | 9 + pkg/backend/connect_retry.go | 183 +++++++++++++++++ pkg/backend/connect_retry_test.go | 249 +++++++++++++++++++++++ pkg/backend/render_batch.go | 3 +- pkg/backend/render_fanin.go | 71 ++++--- pkg/backend/rpc.go | 8 +- pkg/backend/stable_render.go | 20 +- pkg/cfg/api_test.go | 2 + pkg/cfg/common.go | 5 + pkg/cfg/common_test.go | 3 + 14 files changed, 625 insertions(+), 36 deletions(-) create mode 100644 pkg/app/carbonapi/connect_retry_test.go create mode 100644 pkg/backend/connect_retry.go create mode 100644 pkg/backend/connect_retry_test.go diff --git a/config/carbonzipper.yaml b/config/carbonzipper.yaml index e86362d3..a5a4977a 100644 --- a/config/carbonzipper.yaml +++ b/config/carbonzipper.yaml @@ -21,6 +21,9 @@ timeouts: afterStarted: "200s" # Timeout to connect to the server connect: "200ms" + # How long a render keeps retrying a backend that refused, reset or timed + # out the connection before responding (a store restart). 0 disables. + connectRetry: "10s" # Number of concurrent requests to any given backend - default is no limit. # If set, you likely want >= MaxIdleConnsPerHost diff --git a/pkg/app/carbonapi/app.go b/pkg/app/carbonapi/app.go index 16170607..1b988c00 100644 --- a/pkg/app/carbonapi/app.go +++ b/pkg/app/carbonapi/app.go @@ -341,6 +341,11 @@ func newBackendHTTPTransport(config cfg.Zipper) http.RoundTripper { func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *PrometheusMetrics, lg *zap.Logger) ([]backend.Backend, error) { client := &http.Client{Transport: newBackendHTTPTransport(config)} + backend.SetConnectRetry(backend.ConnectRetry{ + Window: config.Timeouts.ConnectRetry, + Retries: zms.BackendConnectRetries, + Recoveries: zms.BackendConnectRecoveries, + }) configBackendList := config.GetBackends() backends := make([]backend.Backend, 0, len(configBackendList)) diff --git a/pkg/app/carbonapi/connect_retry_test.go b/pkg/app/carbonapi/connect_retry_test.go new file mode 100644 index 00000000..376ddbc7 --- /dev/null +++ b/pkg/app/carbonapi/connect_retry_test.go @@ -0,0 +1,83 @@ +package carbonapi + +import ( + "context" + "io" + "net" + "net/http" + "net/url" + "os" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/bookingcom/carbonapi/pkg/backend" + "github.com/bookingcom/carbonapi/pkg/types" +) + +// restartingStore fails its first refusals renders, then serves like its fixture. +type restartingStore struct { + *partialRenderBackend + refusals int32 + failure error + calls atomic.Int32 +} + +func (s *restartingStore) Render(ctx context.Context, r types.RenderRequest) ([]types.Metric, error) { + if s.calls.Add(1) <= s.refusals { + return nil, s.failure + } + return s.partialRenderBackend.Render(ctx, r) +} + +func TestRenderRetriesRestartingStore(t *testing.T) { + refused := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", + Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} + afterAccept := &url.Error{Op: "Get", URL: "http://store/render/", Err: io.ErrUnexpectedEOF} + const window = 300 * time.Millisecond + for _, tc := range []struct { + name string + refusals int32 + failure error + peer bool // another member of the refusing store's replica group answers + wantCalls int32 + wantError bool + }{ + {name: "refused_then_up", refusals: 3, failure: refused, wantCalls: 4}, + {name: "refused_past_window", refusals: 1 << 30, failure: refused, wantError: true}, + {name: "refused_peer_answers", refusals: 1 << 30, failure: refused, peer: true, wantCalls: 1}, + {name: "failure_after_accept", refusals: 1 << 30, failure: afterAccept, wantCalls: 1, wantError: true}, + } { + t.Run(tc.name, func(t *testing.T) { + backend.SetConnectRetry(backend.ConnectRetry{Window: window, InitialDelay: 10 * time.Millisecond, MaxDelay: 50 * time.Millisecond}) + t.Cleanup(func() { backend.SetConnectRetry(backend.ConnectRetry{}) }) + store := &restartingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc1"}, + refusals: tc.refusals, failure: tc.failure} + stores := []backend.BackendImpl{store, &partialRenderBackend{path: "m.ko", cluster: "cy", dc: "dc1"}} + if tc.peer { + stores = append(stores, &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc1"}) + } + app := partialRenderAppOf(t, stores...) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + elapsed := time.Since(start) + if tc.wantError { + if w.Code != http.StatusInternalServerError { + t.Fatalf("HTTP %d, want 500: %s", w.Code, w.Body.String()) + } + } else { + assertPartialRenderValue(t, w, 100) // 99 from the restarted store + 1 + } + if tc.wantCalls != 0 && store.calls.Load() != tc.wantCalls { + t.Fatalf("store rendered %d times, want %d", store.calls.Load(), tc.wantCalls) + } + if tc.name == "refused_past_window" && (elapsed < window-20*time.Millisecond || store.calls.Load() < 3) { + t.Fatalf("gave up after %v and %d attempts", elapsed, store.calls.Load()) + } + if elapsed > window+2*time.Second { + t.Fatalf("request took %v", elapsed) + } + }) + } +} diff --git a/pkg/app/carbonapi/metrics.go b/pkg/app/carbonapi/metrics.go index d49a01b2..5b2b8061 100644 --- a/pkg/app/carbonapi/metrics.go +++ b/pkg/app/carbonapi/metrics.go @@ -53,6 +53,9 @@ type ZipperPrometheusMetrics struct { BackendResponses *prometheus.CounterVec + BackendConnectRetries prometheus.Counter + BackendConnectRecoveries prometheus.Counter + RenderOutDurationExp *prometheus.HistogramVec FindOutDuration *prometheus.HistogramVec @@ -342,6 +345,8 @@ func registerPrometheusMetrics(ms *PrometheusMetrics, zms *ZipperPrometheusMetri prometheus.MustRegister(zms.RenderFixedMismatches) prometheus.MustRegister(zms.RenderMismatchedResponses) prometheus.MustRegister(zms.BackendResponses) + prometheus.MustRegister(zms.BackendConnectRetries) + prometheus.MustRegister(zms.BackendConnectRecoveries) prometheus.MustRegister(zms.RenderOutDurationExp) prometheus.MustRegister(zms.FindOutDuration) prometheus.MustRegister(zms.BackendEnqueuedRequests) @@ -462,5 +467,17 @@ func NewZipperPrometheusMetrics(config cfg.Zipper) *ZipperPrometheusMetrics { }, []string{"status", "request"}, ), + BackendConnectRetries: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "backend_connect_retries_total", + Help: "Render fetches repeated after a backend refused or dropped the connection before responding", + }, + ), + BackendConnectRecoveries: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "backend_connect_retry_recoveries_total", + Help: "Repeated render fetches that the backend answered with data or not-found", + }, + ), } } diff --git a/pkg/app/carbonapi/partial_render_test.go b/pkg/app/carbonapi/partial_render_test.go index 109f8343..2d93afcd 100644 --- a/pkg/app/carbonapi/partial_render_test.go +++ b/pkg/app/carbonapi/partial_render_test.go @@ -64,6 +64,15 @@ func (b *partialRenderBackend) Render(_ context.Context, r types.RenderRequest) } func partialRenderApp(t *testing.T, stores ...*partialRenderBackend) *App { + t.Helper() + impls := make([]backend.BackendImpl, len(stores)) + for i, store := range stores { + impls[i] = store + } + return partialRenderAppOf(t, impls...) +} + +func partialRenderAppOf(t *testing.T, stores ...backend.BackendImpl) *App { t.Helper() functions.New(nil, zap.NewNop()) c := cfg.DefaultAPIConfig() diff --git a/pkg/backend/connect_retry.go b/pkg/backend/connect_retry.go new file mode 100644 index 00000000..7d9a0a3f --- /dev/null +++ b/pkg/backend/connect_retry.go @@ -0,0 +1,183 @@ +package backend + +import ( + "context" + "errors" + "io" + "net" + "net/url" + "sync/atomic" + "syscall" + "time" + + "github.com/bookingcom/carbonapi/pkg/types" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +// ConnectRetry keeps render fetches alive across a short backend outage, such +// as a store restart. Only fetches that failed before the backend sent any +// response are repeated; a failure after the backend answered stays final. +type ConnectRetry struct { + // Window is the total backoff spent on one fetch. Zero disables retries. + Window time.Duration + // InitialDelay doubles after every attempt up to MaxDelay. + InitialDelay, MaxDelay time.Duration + + Retries prometheus.Counter // repeated fetches + Recoveries prometheus.Counter // repeated fetches that the backend answered +} + +var connectRetry atomic.Pointer[ConnectRetry] + +// SetConnectRetry replaces the process-wide render retry policy. +func SetConnectRetry(c ConnectRetry) { + if c.InitialDelay <= 0 { + c.InitialDelay = 250 * time.Millisecond + } + if c.MaxDelay < c.InitialDelay { + c.MaxDelay = max(2*time.Second, c.InitialDelay) + } + connectRetry.Store(&c) +} + +// unreachableError marks a fetch that never reached a backend response and +// remembers the backend to repeat it against. +type unreachableError struct { + error + backend Backend +} + +func (e unreachableError) Unwrap() error { return e.error } + +func (b *Backend) renderError(err error) error { + if connectFailure(err) { + return unreachableError{err, *b} + } + return err +} + +// connectFailure reports whether err is a refused, reset or timed-out +// connection seen before any response. http.Client.Do wraps exactly those +// failures in *url.Error; body reads and decoding happen after the backend +// answered and never produce one. +func connectFailure(err error) bool { + var urlErr *url.Error + if !errors.As(err, &urlErr) { + return false + } + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "dial" { + return true + } + return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, io.EOF) +} + +type renderFetch struct { + request types.RenderRequest + groups []renderGroup + messages [][]types.Metric +} + +// retryUnreachable repeats unreachable fetches that no replica group member +// has recovered, with one backoff schedule shared by all fetches of a request. +// It returns early when the request deadline cannot fit another attempt. +func retryUnreachable(ctx context.Context, fetches []renderFetch, lg *zap.Logger) { + policy := connectRetry.Load() + if policy == nil || policy.Window <= 0 || !anyUnreachable(fetches) { + return + } + start := time.Now() + delay := policy.InitialDelay + for attempt := 1; anyUnreachable(fetches); attempt++ { + wait := min(delay, policy.Window-time.Since(start)) + if wait <= 0 { + return + } + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) <= wait { + return + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + delay = min(2*delay, policy.MaxDelay) + + resent := make([][]int, len(fetches)) + for fi := range fetches { + f := &fetches[fi] + for gi := range f.groups { + g := &f.groups[gi] + if !g.unrecovered() { + continue + } + n := 0 + kept := g.errs[:0] + for _, err := range g.errs { + u, ok := err.(unreachableError) + if !ok { + kept = append(kept, err) + continue + } + lg.Debug("retrying render after backend connect failure", + zap.String("backend", u.backend.GetServerAddress()), + zap.Strings("targets", f.request.Targets), + zap.Int("attempt", attempt), + zap.Duration("elapsed", time.Since(start)), + zap.Error(u.error)) + u.backend.SendRender(ctx, f.request, g.results, g.errors) + n++ + } + clear(g.errs[len(kept):]) + g.errs = kept + if n != 0 { + if resent[fi] == nil { + resent[fi] = make([]int, len(f.groups)) + } + resent[fi][gi] = n + if policy.Retries != nil { + policy.Retries.Add(float64(n)) + } + } + } + } + for fi, counts := range resent { + for gi, n := range counts { + if n == 0 { + continue + } + var answered int + fetches[fi].messages, answered = fetches[fi].groups[gi].receive(n, fetches[fi].messages) + if answered != 0 { + if policy.Recoveries != nil { + policy.Recoveries.Add(float64(answered)) + } + lg.Debug("render recovered after backend connect retry", + zap.Strings("targets", fetches[fi].request.Targets), + zap.Int("attempt", attempt), + zap.Duration("elapsed", time.Since(start))) + } + } + } + } +} + +func anyUnreachable(fetches []renderFetch) bool { + for fi := range fetches { + for gi := range fetches[fi].groups { + g := &fetches[fi].groups[gi] + if !g.unrecovered() { + continue + } + for _, err := range g.errs { + if _, ok := err.(unreachableError); ok { + return true + } + } + } + } + return false +} diff --git a/pkg/backend/connect_retry_test.go b/pkg/backend/connect_retry_test.go new file mode 100644 index 00000000..42da88a1 --- /dev/null +++ b/pkg/backend/connect_retry_test.go @@ -0,0 +1,249 @@ +package backend + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "net/url" + "os" + "sync/atomic" + "syscall" + "testing" + "time" + + bnet "github.com/bookingcom/carbonapi/pkg/backend/net" + "github.com/bookingcom/carbonapi/pkg/cfg" + "github.com/bookingcom/carbonapi/pkg/types" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "go.uber.org/zap" +) + +var refusedDial = &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", + Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} + +type retryTestBackend struct { + coalesceTestBackend + cluster string +} + +func (b retryTestBackend) BackendInfo() (string, string, string) { return b.cluster, b.cluster, "dc1" } + +// retryStore fails its first refusals calls with failure, then answers. +func retryStore(cluster string, refusals int32, failure error, answer error) (Backend, *atomic.Int32) { + calls := new(atomic.Int32) + b := coalesceBackend(func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + if calls.Add(1) <= refusals { + return nil, failure + } + if answer != nil { + return nil, answer + } + return coalesceMetrics(r.Targets), nil + }, 4) + b.BackendImpl = retryTestBackend{coalesceTestBackend: b.BackendImpl.(coalesceTestBackend), cluster: cluster} + b.Proc() + return b, calls +} + +func counterValue(c prometheus.Counter) float64 { + m := &dto.Metric{} + if err := c.Write(m); err != nil { + panic(err) + } + return m.GetCounter().GetValue() +} + +func fastConnectRetry(t *testing.T, window time.Duration) (retries, recoveries prometheus.Counter) { + t.Helper() + retries = prometheus.NewCounter(prometheus.CounterOpts{Name: "test_retries"}) + recoveries = prometheus.NewCounter(prometheus.CounterOpts{Name: "test_recoveries"}) + SetConnectRetry(ConnectRetry{Window: window, InitialDelay: 10 * time.Millisecond, MaxDelay: 40 * time.Millisecond, + Retries: retries, Recoveries: recoveries}) + t.Cleanup(func() { connectRetry.Store(nil) }) + return retries, recoveries +} + +// fetchBoth runs the ordinary and the grouped (stable batch) fan-in. +func fetchBoth(t *testing.T, ctx context.Context, backends []Backend, path string, check func(name string, metrics []types.Metric, errs []error)) { + t.Helper() + request := types.NewRenderRequest([]string{path}, 1, 4) + metrics, _, errs := Renders(ctx, backends, request, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + check("ordinary", metrics, errs) + replies := RendersGrouped(ctx, [][]Backend{backends, backends}, []types.RenderRequest{request, types.NewRenderRequest([]string{path + ".2"}, 1, 4)}, + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + for _, reply := range replies { + check("grouped", reply.Metrics, reply.Errors) + } +} + +func TestConnectRetryRefusedThenUp(t *testing.T) { + for _, grouped := range []bool{false, true} { + retries, recoveries := fastConnectRetry(t, 2*time.Second) + // The store refuses its first three connections, then serves data. + b, _ := retryStore("cx", 3, refusedDial, nil) + start := time.Now() + request := types.NewRenderRequest([]string{"m"}, 1, 4) + replies := []GroupedRenderReply{{}} + if grouped { + replies = RendersGrouped(context.Background(), [][]Backend{{b}, {b}}, + []types.RenderRequest{request, types.NewRenderRequest([]string{"n"}, 1, 4)}, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + } else { + replies[0].Metrics, _, replies[0].Errors = Renders(context.Background(), []Backend{b}, request, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + } + for _, reply := range replies { + if len(reply.Errors) != 0 || len(reply.Metrics) != 1 || reply.Metrics[0].Values[1] != 42 { + t.Fatalf("grouped %v: %+v, want full data", grouped, reply) + } + } + if counterValue(retries) == 0 || counterValue(recoveries) == 0 || time.Since(start) > time.Second { + t.Fatalf("grouped %v: retries %v recoveries %v after %v", grouped, counterValue(retries), counterValue(recoveries), time.Since(start)) + } + } +} + +func TestConnectRetryWindowExhausted(t *testing.T) { + const window = 200 * time.Millisecond + retries, recoveries := fastConnectRetry(t, window) + b, _ := retryStore("cx", 1<<30, refusedDial, nil) + start := time.Now() + metrics, _, errs := Renders(context.Background(), []Backend{b}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + elapsed := time.Since(start) + if len(metrics) != 0 || len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("metrics %+v errs %v, want the refusal", metrics, errs) + } + var u unreachableError + if errors.As(errs[0], &u) { + t.Fatal("internal retry marker leaked") + } + if elapsed < window-20*time.Millisecond || elapsed > window+time.Second { + t.Fatalf("gave up after %v, want about %v", elapsed, window) + } + if counterValue(retries) == 0 || counterValue(recoveries) != 0 { + t.Fatalf("retries %v recoveries %v", counterValue(retries), counterValue(recoveries)) + } +} + +func TestConnectRetryRespectsDeadline(t *testing.T) { + fastConnectRetry(t, 10*time.Second) + b, _ := retryStore("cx", 1<<30, refusedDial, nil) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + _, _, errs := Renders(ctx, []Backend{b}, types.NewRenderRequest([]string{"m"}, 1, 4), cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("errs %v, want the refusal", errs) + } + if ctx.Err() != nil { + t.Fatal("retries ran past the request deadline") + } +} + +func TestConnectRetrySkipsAnsweredGroup(t *testing.T) { + for _, peer := range []error{nil, types.ErrMetricsNotFound} { + retries, _ := fastConnectRetry(t, 2*time.Second) + refused, calls := retryStore("cy", 1<<30, refusedDial, nil) + answering, _ := retryStore("cy", 0, nil, peer) + start := time.Now() + fetchBoth(t, context.Background(), []Backend{refused, answering}, "m", func(name string, metrics []types.Metric, errs []error) { + wantErrs := 1 + if peer != nil { + wantErrs = 2 + } + if len(errs) != wantErrs || (peer == nil) != (len(metrics) == 1) { + t.Fatalf("%s peer %v: metrics %+v errs %v, want group recovery", name, peer, metrics, errs) + } + for _, err := range errs { + if !errors.Is(err, types.ErrMetricsNotFound) { + t.Fatalf("%s peer %v: unrecovered %v", name, peer, errs) + } + } + }) + if calls.Load() == 0 || counterValue(retries) != 0 || time.Since(start) > time.Second { + t.Fatalf("peer %v: answered group was retried: retries %v after %v", peer, counterValue(retries), time.Since(start)) + } + } +} + +func TestConnectRetryKeepsFailuresAfterAccept(t *testing.T) { + for _, failure := range []error{ + bnet.ErrHTTPCode(http.StatusServiceUnavailable), + io.ErrUnexpectedEOF, + &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNRESET}, // body read, no *url.Error + &url.Error{Op: "Get", URL: "http://store/render/", Err: context.DeadlineExceeded}, + } { + retries, _ := fastConnectRetry(t, 2*time.Second) + b, calls := retryStore("cx", 1, failure, nil) + _, _, errs := Renders(context.Background(), []Backend{b}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if len(errs) != 1 || errs[0] != failure || calls.Load() != 1 || counterValue(retries) != 0 { + t.Fatalf("%v: errs %v calls %d retries %v, want one final failure", failure, errs, calls.Load(), counterValue(retries)) + } + } +} + +// Real transports: a closed port and a peer that drops the connection before +// any response are connect failures; failures after a response started are not. +func TestConnectFailureClassification(t *testing.T) { + closed, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + refusedAddr := closed.Addr().String() + closed.Close() + + serve := func(handle func(net.Conn)) string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) + go func() { + for { + c, err := l.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + buf := make([]byte, 4096) + _, _ = c.Read(buf) + handle(c) + }() + } + }() + return l.Addr().String() + } + dropped := serve(func(c net.Conn) { + if tc, ok := c.(*net.TCPConn); ok { + _ = tc.SetLinger(0) // RST instead of FIN + } + }) + truncated := serve(func(c net.Conn) { + _, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: 1000\r\n\r\npartial") + }) + failed := serve(func(c net.Conn) { + _, _ = io.WriteString(c, "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n") + }) + + for _, transport := range []http.RoundTripper{ + &http.Transport{DisableCompression: true}, + bnet.NewSyncTransport(&http.Transport{DisableCompression: true}), + } { + for _, tc := range []struct { + addr string + want bool + }{{refusedAddr, true}, {dropped, true}, {truncated, false}, {failed, false}} { + nb, err := bnet.New(bnet.Config{Address: tc.addr, Cluster: "c", Client: &http.Client{Transport: transport}, Timeout: 2 * time.Second}) + if err != nil { + t.Fatal(err) + } + _, err = nb.Render(context.Background(), types.NewRenderRequest([]string{"m"}, 1, 4)) + if err == nil || connectFailure(err) != tc.want { + t.Errorf("%T %s: %v classified %v, want %v", transport, tc.addr, err, err != nil && connectFailure(err), tc.want) + } + } + } +} diff --git a/pkg/backend/render_batch.go b/pkg/backend/render_batch.go index a49d32ea..a41d392c 100644 --- a/pkg/backend/render_batch.go +++ b/pkg/backend/render_batch.go @@ -86,7 +86,7 @@ func (b *Backend) renderOne(r *renderReq) { res, err := b.renderMetrics(r.Ctx, r.RenderRequest) t.ObserveDuration() if err != nil { - r.Errors <- err + r.Errors <- b.renderError(err) return } b.addSourceMetaToMetrics(res) @@ -116,6 +116,7 @@ func (b *Backend) renderBatch(batch []*renderReq) { } // Each original render keeps its own replica fan-in. In particular, a // slow or failed replica must not trigger another whole-query retry. + err = b.renderError(err) for _, r := range batch { r.Errors <- err } diff --git a/pkg/backend/render_fanin.go b/pkg/backend/render_fanin.go index 411d94e6..f917e6f6 100644 --- a/pkg/backend/render_fanin.go +++ b/pkg/backend/render_fanin.go @@ -9,10 +9,12 @@ import ( // Keep replica replies together until errors have been classified. Only the // backends actually selected for this request can establish that a group answered. type renderGroup struct { - n int - cluster string - results chan []types.Metric - errors chan error + n int + cluster string + results chan []types.Metric + errors chan error + answered bool + errs []error } func newRenderGroups(backends []Backend) ([]renderGroup, []int) { @@ -42,30 +44,49 @@ func newRenderGroups(backends []Backend) ([]renderGroup, []int) { return groups, membership } -func collectRenderGroups(groups []renderGroup) ([][]types.Metric, []error) { - var messages [][]types.Metric - var errs []error - for _, g := range groups { - start := len(errs) - answered := false - for range g.n { - select { - case metrics := <-g.results: - messages = append(messages, metrics) - answered = true - case err := <-g.errors: - errs = append(errs, err) - var notFound types.ErrNotFound - if errors.As(err, ¬Found) { - answered = true - } +// receive collects n replies and returns how many of them were answers: data +// or a backend's own not-found. +func (g *renderGroup) receive(n int, messages [][]types.Metric) ([][]types.Metric, int) { + answers := 0 + for range n { + select { + case metrics := <-g.results: + messages = append(messages, metrics) + answers++ + case err := <-g.errors: + g.errs = append(g.errs, err) + var notFound types.ErrNotFound + if errors.As(err, ¬Found) { + answers++ } } - if g.cluster != "" && answered { - for i := start; i < len(errs); i++ { - errs[i] = types.ErrMetricsNotFound + } + if answers != 0 { + g.answered = true + } + return messages, answers +} + +// unrecovered reports whether this group's errors still fail the fetch. +func (g *renderGroup) unrecovered() bool { return g.cluster == "" || !g.answered } + +func collectRenderGroups(f *renderFetch) { + for i := range f.groups { + f.messages, _ = f.groups[i].receive(f.groups[i].n, f.messages) + } +} + +func renderGroupErrors(groups []renderGroup) []error { + var errs []error + for _, g := range groups { + for _, err := range g.errs { + if !g.unrecovered() { + err = types.ErrMetricsNotFound + } else if u, ok := err.(unreachableError); ok { + err = u.error } + errs = append(errs, err) } } - return messages, errs + return errs } diff --git a/pkg/backend/rpc.go b/pkg/backend/rpc.go index acb414e1..9b2d35ee 100644 --- a/pkg/backend/rpc.go +++ b/pkg/backend/rpc.go @@ -27,9 +27,11 @@ func Renders(ctx context.Context, backends []Backend, request types.RenderReques backend.SendRender(ctx, request, g.results, g.errors) } - msgs, errs := collectRenderGroups(groups) - metrics, stats := types.MergeMetrics(msgs, replicaMismatchConfig, lg) - return metrics, stats, errs + fetches := []renderFetch{{request: request, groups: groups}} + collectRenderGroups(&fetches[0]) + retryUnreachable(ctx, fetches, lg) + metrics, stats := types.MergeMetrics(fetches[0].messages, replicaMismatchConfig, lg) + return metrics, stats, renderGroupErrors(groups) } // Infos makes Info calls to multiple backends. diff --git a/pkg/backend/stable_render.go b/pkg/backend/stable_render.go index 8ca6c0aa..1eef8ee3 100644 --- a/pkg/backend/stable_render.go +++ b/pkg/backend/stable_render.go @@ -33,31 +33,37 @@ func RendersGrouped(ctx context.Context, choices [][]Backend, requests []types.R calls []renderDispatch } groups := make(map[*requestQueue[*renderReq]]*group) - replicaGroups := make([][]renderGroup, len(requests)) + fetches := make([]renderFetch, len(requests)) for i, request := range requests { var membership []int - replicaGroups[i], membership = newRenderGroups(choices[i]) + fetches[i].request = request + fetches[i].groups, membership = newRenderGroups(choices[i]) for j, b := range choices[i] { g := groups[b.fastRenderQ] if g == nil { g = &group{backend: b} groups[b.fastRenderQ] = g } - replicas := replicaGroups[i][membership[j]] + replicas := fetches[i].groups[membership[j]] g.calls = append(g.calls, renderDispatch{request, replicas.results, replicas.errors}) } } for _, g := range groups { g.backend.sendRenderGroup(ctx, g.calls) } + for i := range fetches { + collectRenderGroups(&fetches[i]) + } + // One backoff schedule covers every target, so a restarting store delays + // the request once rather than once per target. + retryUnreachable(ctx, fetches, lg) replies := make([]GroupedRenderReply, len(requests)) - for i := range requests { + for i := range fetches { if len(choices[i]) == 0 { continue } - var messages [][]types.Metric - messages, replies[i].Errors = collectRenderGroups(replicaGroups[i]) - replies[i].Metrics, replies[i].Stats = types.MergeMetrics(messages, mismatch, lg) + replies[i].Errors = renderGroupErrors(fetches[i].groups) + replies[i].Metrics, replies[i].Stats = types.MergeMetrics(fetches[i].messages, mismatch, lg) } return replies } diff --git a/pkg/cfg/api_test.go b/pkg/cfg/api_test.go index 06b61eb6..ad0f8a64 100644 --- a/pkg/cfg/api_test.go +++ b/pkg/cfg/api_test.go @@ -69,6 +69,7 @@ loggerConfig: Global: 10 * time.Minute, AfterStarted: 10 * time.Minute, Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, }, ConcurrencyLimitPerServer: 1024, KeepAliveInterval: 30 * time.Second, @@ -170,6 +171,7 @@ loggerConfig: Global: 10 * time.Minute, AfterStarted: 10 * time.Minute, Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, }, ConcurrencyLimitPerServer: 1024, KeepAliveInterval: 30 * time.Second, diff --git a/pkg/cfg/common.go b/pkg/cfg/common.go index 4a637195..4734414e 100644 --- a/pkg/cfg/common.go +++ b/pkg/cfg/common.go @@ -50,6 +50,7 @@ func DefaultCommonConfig() Common { Global: 10000 * time.Millisecond, AfterStarted: 2 * time.Second, Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, }, ConcurrencyLimitPerServer: 20, KeepAliveInterval: 30 * time.Second, @@ -373,6 +374,10 @@ type Timeouts struct { Global time.Duration `yaml:"global"` AfterStarted time.Duration `yaml:"afterStarted"` Connect time.Duration `yaml:"connect"` + // ConnectRetry is how long a render keeps retrying a backend that refused, + // reset or timed out the connection before sending any response, as during + // a store restart. Zero disables the retry. + ConnectRetry time.Duration `yaml:"connectRetry"` } type ProtocolBackend struct { diff --git a/pkg/cfg/common_test.go b/pkg/cfg/common_test.go index 9b17a1f0..6ec0ee0d 100644 --- a/pkg/cfg/common_test.go +++ b/pkg/cfg/common_test.go @@ -86,6 +86,7 @@ monitoring: Global: 20 * time.Second, AfterStarted: 15 * time.Second, Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, @@ -265,6 +266,7 @@ monitoring: Global: 20 * time.Second, AfterStarted: 15 * time.Second, Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, @@ -511,6 +513,7 @@ monitoring: Global: 20 * time.Second, AfterStarted: 15 * time.Second, Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, From 9b86b87d8a16a04069a7eda0a5051bf53800b47b Mon Sep 17 00:00:00 2001 From: Anton Timofieiev Date: Fri, 13 Nov 2026 11:25:14 +0100 Subject: [PATCH 3/8] fix(zipper): cancel connect retries once another replica answered The connect retry retried a refused store's render fetch for up to timeouts.connectRetry whenever no member of that store's own replica group answered. Renders query every DC's group and fill missing points from any of them, so while one store refused, every render touching it waited the whole window even though the other DCs had already returned the same series. A production canary (30 s refusal of a one-store group) added p50 7.9 s and max 10.1 s per render and turned client give-ups into 133 HTTP 500s and 95 499s. A refused fetch is now given up as soon as a replica group of the same cluster has returned data for the target: before the first retry, and during an attempt, where repeats still in flight to other groups of that cluster are no longer awaited. Retries continue only while no other group has data (all copies briefly unreachable), which is what the connect retry was for. Not-found answers, data from another cluster and ungrouped backends never cancel a retry. A cancelled fetch keeps its plain connect error, so the required-fetch and replica-group rules decide the render exactly as before. New counter: backend_connect_retries_cancelled_total. Co-Authored-By: Claude Opus 5.5 (1M context) --- config/carbonzipper.yaml | 3 +- pkg/app/carbonapi/app.go | 7 +- pkg/app/carbonapi/connect_retry_test.go | 33 ++++- pkg/app/carbonapi/metrics.go | 8 ++ pkg/backend/connect_retry.go | 176 ++++++++++++++++++++---- pkg/backend/connect_retry_test.go | 145 +++++++++++++++++-- pkg/backend/render_fanin.go | 50 +++++-- pkg/cfg/common.go | 4 +- 8 files changed, 363 insertions(+), 63 deletions(-) diff --git a/config/carbonzipper.yaml b/config/carbonzipper.yaml index a5a4977a..d42c981a 100644 --- a/config/carbonzipper.yaml +++ b/config/carbonzipper.yaml @@ -22,7 +22,8 @@ timeouts: # Timeout to connect to the server connect: "200ms" # How long a render keeps retrying a backend that refused, reset or timed - # out the connection before responding (a store restart). 0 disables. + # out the connection before responding (a store restart), while no replica + # group of the same cluster in another DC has returned data. 0 disables. connectRetry: "10s" # Number of concurrent requests to any given backend - default is no limit. diff --git a/pkg/app/carbonapi/app.go b/pkg/app/carbonapi/app.go index 1b988c00..e7994f83 100644 --- a/pkg/app/carbonapi/app.go +++ b/pkg/app/carbonapi/app.go @@ -342,9 +342,10 @@ func newBackendHTTPTransport(config cfg.Zipper) http.RoundTripper { func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *PrometheusMetrics, lg *zap.Logger) ([]backend.Backend, error) { client := &http.Client{Transport: newBackendHTTPTransport(config)} backend.SetConnectRetry(backend.ConnectRetry{ - Window: config.Timeouts.ConnectRetry, - Retries: zms.BackendConnectRetries, - Recoveries: zms.BackendConnectRecoveries, + Window: config.Timeouts.ConnectRetry, + Retries: zms.BackendConnectRetries, + Recoveries: zms.BackendConnectRecoveries, + Cancellations: zms.BackendConnectCancels, }) configBackendList := config.GetBackends() diff --git a/pkg/app/carbonapi/connect_retry_test.go b/pkg/app/carbonapi/connect_retry_test.go index 376ddbc7..a854d7f0 100644 --- a/pkg/app/carbonapi/connect_retry_test.go +++ b/pkg/app/carbonapi/connect_retry_test.go @@ -35,21 +35,35 @@ func TestRenderRetriesRestartingStore(t *testing.T) { refused := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} afterAccept := &url.Error{Op: "Get", URL: "http://store/render/", Err: io.ErrUnexpectedEOF} - const window = 300 * time.Millisecond + const forever = 1 << 30 for _, tc := range []struct { name string + window time.Duration refusals int32 failure error peer bool // another member of the refusing store's replica group answers + otherDC bool // a replica group in another DC holds the same series + otherDown int32 wantCalls int32 wantError bool + maxTime time.Duration }{ {name: "refused_then_up", refusals: 3, failure: refused, wantCalls: 4}, - {name: "refused_past_window", refusals: 1 << 30, failure: refused, wantError: true}, - {name: "refused_peer_answers", refusals: 1 << 30, failure: refused, peer: true, wantCalls: 1}, - {name: "failure_after_accept", refusals: 1 << 30, failure: afterAccept, wantCalls: 1, wantError: true}, + {name: "refused_past_window", refusals: forever, failure: refused, wantError: true}, + {name: "refused_peer_answers", refusals: forever, failure: refused, peer: true, wantCalls: 1}, + {name: "failure_after_accept", refusals: forever, failure: afterAccept, wantCalls: 1, wantError: true}, + // The other DCs already answered: no retry, no wait, full data. + {name: "refused_other_group_answers", window: 5 * time.Second, refusals: forever, failure: refused, + otherDC: true, wantCalls: 1, maxTime: 200 * time.Millisecond}, + {name: "all_groups_refused_then_up", refusals: 3, failure: refused, otherDC: true, otherDown: 3}, + {name: "all_groups_refused_past_window", refusals: forever, failure: refused, otherDC: true, otherDown: forever, + wantError: true}, } { t.Run(tc.name, func(t *testing.T) { + window := tc.window + if window == 0 { + window = 300 * time.Millisecond + } backend.SetConnectRetry(backend.ConnectRetry{Window: window, InitialDelay: 10 * time.Millisecond, MaxDelay: 50 * time.Millisecond}) t.Cleanup(func() { backend.SetConnectRetry(backend.ConnectRetry{}) }) store := &restartingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc1"}, @@ -58,6 +72,10 @@ func TestRenderRetriesRestartingStore(t *testing.T) { if tc.peer { stores = append(stores, &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc1"}) } + if tc.otherDC { + stores = append(stores, &restartingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc3"}, + refusals: tc.otherDown, failure: refused}) + } app := partialRenderAppOf(t, stores...) start := time.Now() w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") @@ -67,14 +85,17 @@ func TestRenderRetriesRestartingStore(t *testing.T) { t.Fatalf("HTTP %d, want 500: %s", w.Code, w.Body.String()) } } else { - assertPartialRenderValue(t, w, 100) // 99 from the restarted store + 1 + assertPartialRenderValue(t, w, 100) // 99 from a replica of m.ok + 1 } if tc.wantCalls != 0 && store.calls.Load() != tc.wantCalls { t.Fatalf("store rendered %d times, want %d", store.calls.Load(), tc.wantCalls) } - if tc.name == "refused_past_window" && (elapsed < window-20*time.Millisecond || store.calls.Load() < 3) { + if tc.refusals == forever && tc.failure == refused && tc.wantError && (elapsed < window-20*time.Millisecond || store.calls.Load() < 3) { t.Fatalf("gave up after %v and %d attempts", elapsed, store.calls.Load()) } + if tc.maxTime != 0 && elapsed > tc.maxTime { + t.Fatalf("request took %v, want under %v", elapsed, tc.maxTime) + } if elapsed > window+2*time.Second { t.Fatalf("request took %v", elapsed) } diff --git a/pkg/app/carbonapi/metrics.go b/pkg/app/carbonapi/metrics.go index 5b2b8061..031800c8 100644 --- a/pkg/app/carbonapi/metrics.go +++ b/pkg/app/carbonapi/metrics.go @@ -55,6 +55,7 @@ type ZipperPrometheusMetrics struct { BackendConnectRetries prometheus.Counter BackendConnectRecoveries prometheus.Counter + BackendConnectCancels prometheus.Counter RenderOutDurationExp *prometheus.HistogramVec FindOutDuration *prometheus.HistogramVec @@ -347,6 +348,7 @@ func registerPrometheusMetrics(ms *PrometheusMetrics, zms *ZipperPrometheusMetri prometheus.MustRegister(zms.BackendResponses) prometheus.MustRegister(zms.BackendConnectRetries) prometheus.MustRegister(zms.BackendConnectRecoveries) + prometheus.MustRegister(zms.BackendConnectCancels) prometheus.MustRegister(zms.RenderOutDurationExp) prometheus.MustRegister(zms.FindOutDuration) prometheus.MustRegister(zms.BackendEnqueuedRequests) @@ -479,5 +481,11 @@ func NewZipperPrometheusMetrics(config cfg.Zipper) *ZipperPrometheusMetrics { Help: "Repeated render fetches that the backend answered with data or not-found", }, ), + BackendConnectCancels: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "backend_connect_retries_cancelled_total", + Help: "Render fetches of an unreachable backend given up without waiting for the retry window because another replica group of the same cluster returned data", + }, + ), } } diff --git a/pkg/backend/connect_retry.go b/pkg/backend/connect_retry.go index 7d9a0a3f..31cc8952 100644 --- a/pkg/backend/connect_retry.go +++ b/pkg/backend/connect_retry.go @@ -26,6 +26,9 @@ type ConnectRetry struct { Retries prometheus.Counter // repeated fetches Recoveries prometheus.Counter // repeated fetches that the backend answered + // Cancellations counts unreachable fetches given up because another + // replica group of the same cluster returned data for the target. + Cancellations prometheus.Counter } var connectRetry atomic.Pointer[ConnectRetry] @@ -79,12 +82,20 @@ type renderFetch struct { messages [][]types.Metric } -// retryUnreachable repeats unreachable fetches that no replica group member -// has recovered, with one backoff schedule shared by all fetches of a request. -// It returns early when the request deadline cannot fit another attempt. +// retryUnreachable repeats unreachable fetches that no replica group has +// answered, with one backoff schedule shared by all fetches of a request. +// A fetch is repeated only while no other replica group of its cluster has +// returned data for the target: once one has, the merged result is complete +// without the unreachable store, and waiting for it would only delay the +// response. It returns early when the request deadline cannot fit another +// attempt. func retryUnreachable(ctx context.Context, fetches []renderFetch, lg *zap.Logger) { policy := connectRetry.Load() - if policy == nil || policy.Window <= 0 || !anyUnreachable(fetches) { + if policy == nil || policy.Window <= 0 { + return + } + cancelCovered(fetches, policy, lg) + if !anyUnreachable(fetches) { return } start := time.Now() @@ -106,7 +117,7 @@ func retryUnreachable(ctx context.Context, fetches []renderFetch, lg *zap.Logger } delay = min(2*delay, policy.MaxDelay) - resent := make([][]int, len(fetches)) + var flights []retryFlight for fi := range fetches { f := &fetches[fi] for gi := range f.groups { @@ -114,7 +125,7 @@ func retryUnreachable(ctx context.Context, fetches []renderFetch, lg *zap.Logger if !g.unrecovered() { continue } - n := 0 + var sent []unreachableError kept := g.errs[:0] for _, err := range g.errs { u, ok := err.(unreachableError) @@ -129,42 +140,151 @@ func retryUnreachable(ctx context.Context, fetches []renderFetch, lg *zap.Logger zap.Duration("elapsed", time.Since(start)), zap.Error(u.error)) u.backend.SendRender(ctx, f.request, g.results, g.errors) - n++ + sent = append(sent, u) } clear(g.errs[len(kept):]) g.errs = kept - if n != 0 { - if resent[fi] == nil { - resent[fi] = make([]int, len(f.groups)) - } - resent[fi][gi] = n + if len(sent) != 0 { + flights = append(flights, retryFlight{fetch: fi, group: gi, sent: sent}) if policy.Retries != nil { - policy.Retries.Add(float64(n)) + policy.Retries.Add(float64(len(sent))) } } } } - for fi, counts := range resent { - for gi, n := range counts { - if n == 0 { - continue - } - var answered int - fetches[fi].messages, answered = fetches[fi].groups[gi].receive(n, fetches[fi].messages) - if answered != 0 { - if policy.Recoveries != nil { - policy.Recoveries.Add(float64(answered)) - } - lg.Debug("render recovered after backend connect retry", - zap.Strings("targets", fetches[fi].request.Targets), - zap.Int("attempt", attempt), - zap.Duration("elapsed", time.Since(start))) + awaitRetries(fetches, flights) + for _, fl := range flights { + if fl.answered == 0 { + continue + } + if policy.Recoveries != nil { + policy.Recoveries.Add(float64(fl.answered)) + } + lg.Debug("render recovered after backend connect retry", + zap.Strings("targets", fetches[fl.fetch].request.Targets), + zap.Int("attempt", attempt), + zap.Duration("elapsed", time.Since(start))) + } + cancelCovered(fetches, policy, lg) + } +} + +// retryFlight is one replica group's repeated fetches within an attempt. +type retryFlight struct { + fetch, group int + sent []unreachableError + pending int // replies still awaited + answered int +} + +type retryReply struct { + flight int + metrics []types.Metric + err error +} + +// awaitRetries collects the replies of one attempt. As soon as a replica group +// returns data, the same fetch's repeats to other groups of that cluster are no +// longer awaited: they keep their unreachable error for cancelCovered, and +// their late replies are dropped. +func awaitRetries(fetches []renderFetch, flights []retryFlight) { + outstanding := 0 + for i := range flights { + flights[i].pending = len(flights[i].sent) + outstanding += flights[i].pending + } + // Buffered for every reply, so abandoned forwarders never block. + replies := make(chan retryReply, outstanding) + for i := range flights { + g := &fetches[flights[i].fetch].groups[flights[i].group] + go func(flight, n int, results <-chan []types.Metric, errs <-chan error) { + for range n { + metrics, err := nextReply(results, errs) + replies <- retryReply{flight, metrics, err} + } + }(i, flights[i].pending, g.results, g.errors) + } + for outstanding > 0 { + r := <-replies + fl := &flights[r.flight] + if fl.pending == 0 { + continue // abandoned + } + fl.pending-- + outstanding-- + f := &fetches[fl.fetch] + g := &f.groups[fl.group] + var answer bool + if f.messages, answer = g.take(r.metrics, r.err, f.messages); !answer { + continue + } + fl.answered++ + if !g.data || g.cluster == "" { + continue + } + for j := range flights { + o := &flights[j] + if j == r.flight || o.fetch != fl.fetch || o.pending == 0 || f.groups[o.group].cluster != g.cluster { + continue + } + og := &f.groups[o.group] + for _, u := range o.sent[len(o.sent)-o.pending:] { + og.errs = append(og.errs, u) + } + outstanding -= o.pending + o.pending = 0 + } + } +} + +// cancelCovered gives up the unreachable fetches of every replica group whose +// target another group of the same cluster has returned data for. Their +// errors become final, so the required-fetch and replica-group rules decide the render exactly as without +// retries. Ungrouped backends are never covered: they never recover each +// other's errors. +func cancelCovered(fetches []renderFetch, policy *ConnectRetry, lg *zap.Logger) { + for fi := range fetches { + f := &fetches[fi] + for gi := range f.groups { + g := &f.groups[gi] + if !g.unrecovered() || !f.covered(g.cluster) { + continue + } + n := 0 + for i, err := range g.errs { + if u, ok := err.(unreachableError); ok { + g.errs[i] = u.error + n++ } } + if n == 0 { + continue + } + if policy.Cancellations != nil { + policy.Cancellations.Add(float64(n)) + } + lg.Debug("render connect retry cancelled: another replica group answered", + zap.Strings("targets", f.request.Targets), + zap.String("cluster", g.cluster), + zap.Int("fetches", n)) } } } +// covered reports whether a replica group of cluster returned data for this +// fetch. +func (f *renderFetch) covered(cluster string) bool { + if cluster == "" { + return false + } + for gi := range f.groups { + if f.groups[gi].cluster == cluster && f.groups[gi].data { + return true + } + } + return false +} + func anyUnreachable(fetches []renderFetch) bool { for fi := range fetches { for gi := range fetches[fi].groups { diff --git a/pkg/backend/connect_retry_test.go b/pkg/backend/connect_retry_test.go index 42da88a1..a4164212 100644 --- a/pkg/backend/connect_retry_test.go +++ b/pkg/backend/connect_retry_test.go @@ -26,24 +26,36 @@ var refusedDial = &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.O type retryTestBackend struct { coalesceTestBackend - cluster string + cluster, dc string } -func (b retryTestBackend) BackendInfo() (string, string, string) { return b.cluster, b.cluster, "dc1" } +func (b retryTestBackend) BackendInfo() (string, string, string) { return b.cluster, b.cluster, b.dc } // retryStore fails its first refusals calls with failure, then answers. func retryStore(cluster string, refusals int32, failure error, answer error) (Backend, *atomic.Int32) { - calls := new(atomic.Int32) - b := coalesceBackend(func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { - if calls.Add(1) <= refusals { - return nil, failure - } + return retryStoreIn("dc1", cluster, refusals, failure, answer) +} + +// retryStoreIn is retryStore in the replica group of dc. +func retryStoreIn(dc, cluster string, refusals int32, failure error, answer error) (Backend, *atomic.Int32) { + return retryStoreFunc(dc, cluster, refusals, failure, func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { if answer != nil { return nil, answer } return coalesceMetrics(r.Targets), nil + }) +} + +func retryStoreFunc(dc, cluster string, refusals int32, failure error, + serve func(context.Context, types.RenderRequest) ([]types.Metric, error)) (Backend, *atomic.Int32) { + calls := new(atomic.Int32) + b := coalesceBackend(func(ctx context.Context, r types.RenderRequest) ([]types.Metric, error) { + if calls.Add(1) <= refusals { + return nil, failure + } + return serve(ctx, r) }, 4) - b.BackendImpl = retryTestBackend{coalesceTestBackend: b.BackendImpl.(coalesceTestBackend), cluster: cluster} + b.BackendImpl = retryTestBackend{coalesceTestBackend: b.BackendImpl.(coalesceTestBackend), cluster: cluster, dc: dc} b.Proc() return b, calls } @@ -57,13 +69,20 @@ func counterValue(c prometheus.Counter) float64 { } func fastConnectRetry(t *testing.T, window time.Duration) (retries, recoveries prometheus.Counter) { + t.Helper() + retries, recoveries, _ = fastConnectRetryCancels(t, window) + return retries, recoveries +} + +func fastConnectRetryCancels(t *testing.T, window time.Duration) (retries, recoveries, cancels prometheus.Counter) { t.Helper() retries = prometheus.NewCounter(prometheus.CounterOpts{Name: "test_retries"}) recoveries = prometheus.NewCounter(prometheus.CounterOpts{Name: "test_recoveries"}) + cancels = prometheus.NewCounter(prometheus.CounterOpts{Name: "test_cancels"}) SetConnectRetry(ConnectRetry{Window: window, InitialDelay: 10 * time.Millisecond, MaxDelay: 40 * time.Millisecond, - Retries: retries, Recoveries: recoveries}) + Retries: retries, Recoveries: recoveries, Cancellations: cancels}) t.Cleanup(func() { connectRetry.Store(nil) }) - return retries, recoveries + return retries, recoveries, cancels } // fetchBoth runs the ordinary and the grouped (stable batch) fan-in. @@ -167,6 +186,112 @@ func TestConnectRetrySkipsAnsweredGroup(t *testing.T) { } } +// A store whose whole replica group is unreachable is not retried once another +// DC's group of the same cluster returned data: the render returns at the +// healthy groups' speed with their full data, and the refusal stays a plain +// error that the required-fetch rule recovers from the other group's data. +func TestConnectRetryCancelledByOtherGroup(t *testing.T) { + retries, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + refused, _ := retryStoreIn("dc1", "cy", 1<<30, refusedDial, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fra, _ := retryStoreIn("dc2", "cy", 0, nil, nil) + start := time.Now() + fetchBoth(t, context.Background(), []Backend{refused, lhr, fra}, "m", func(name string, metrics []types.Metric, errs []error) { + if len(metrics) != 1 || metrics[0].Values[1] != 42 { + t.Fatalf("%s: metrics %+v, want the other groups' data", name, metrics) + } + if len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("%s: errs %v, want the refusal", name, errs) + } + var u unreachableError + if errors.As(errs[0], &u) { + t.Fatalf("%s: internal retry marker leaked", name) + } + }) + if elapsed := time.Since(start); elapsed > 200*time.Millisecond { + t.Fatalf("waited %v for a store other groups had already covered", elapsed) + } + // One ordinary and two grouped fetches. + if counterValue(retries) != 0 || counterValue(cancels) != 3 { + t.Fatalf("retries %v cancels %v, want 0/3", counterValue(retries), counterValue(cancels)) + } +} + +// Only data from a replica group of the same cluster makes a retry redundant. +func TestConnectRetryNotCancelledWithoutReplicaData(t *testing.T) { + const window = 150 * time.Millisecond + for _, tc := range []struct { + name string + dc, cluster string + answer error + ungroupedPeer bool + }{ + {name: "other_cluster_data", dc: "dc3", cluster: "carbon"}, + {name: "other_group_not_found", dc: "dc3", cluster: "cy", answer: types.ErrMetricsNotFound}, + {name: "ungrouped", ungroupedPeer: true}, + } { + t.Run(tc.name, func(t *testing.T) { + retries, _, cancels := fastConnectRetryCancels(t, window) + cluster := "cy" + if tc.ungroupedPeer { + cluster = "" + } + refused, _ := retryStoreIn("dc1", cluster, 1<<30, refusedDial, nil) + peer, _ := retryStoreIn(tc.dc, tc.cluster, 0, nil, tc.answer) + start := time.Now() + Renders(context.Background(), []Backend{refused, peer}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if elapsed := time.Since(start); elapsed < window-20*time.Millisecond || counterValue(retries) == 0 || counterValue(cancels) != 0 { + t.Fatalf("gave up after %v: retries %v cancels %v, want the full window", elapsed, counterValue(retries), counterValue(cancels)) + } + }) + } +} + +// When every group was unreachable and one recovers, the others' repeats are +// not awaited, even if a just-restarted store accepts but answers slowly. +func TestConnectRetryStopsWaitingOnceAGroupRecovers(t *testing.T) { + _, recoveries, cancels := fastConnectRetryCancels(t, 5*time.Second) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + slow, _ := retryStoreFunc("dc1", "cy", 1, refusedDial, func(context.Context, types.RenderRequest) ([]types.Metric, error) { + <-release + return nil, refusedDial + }) + lhr, _ := retryStoreIn("dc3", "cy", 1, refusedDial, nil) + start := time.Now() + metrics, _, errs := Renders(context.Background(), []Backend{slow, lhr}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if elapsed := time.Since(start); elapsed > 300*time.Millisecond { + t.Fatalf("waited %v on a slow repeat after another group recovered", elapsed) + } + if len(metrics) != 1 || len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("metrics %+v errs %v, want lhr data and the ams refusal", metrics, errs) + } + var u unreachableError + if errors.As(errs[0], &u) { + t.Fatal("internal retry marker leaked") + } + if counterValue(recoveries) != 1 || counterValue(cancels) != 1 { + t.Fatalf("recoveries %v cancels %v, want 1/1", counterValue(recoveries), counterValue(cancels)) + } +} + +// All replica groups refuse briefly: retries continue until one is back. +func TestConnectRetryAllGroupsRefusedThenUp(t *testing.T) { + retries, recoveries, cancels := fastConnectRetryCancels(t, 2*time.Second) + ams, _ := retryStoreIn("dc1", "cy", 1<<30, refusedDial, nil) + lhr, _ := retryStoreIn("dc3", "cy", 3, refusedDial, nil) + metrics, _, errs := Renders(context.Background(), []Backend{ams, lhr}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if len(metrics) != 1 || metrics[0].Values[1] != 42 || len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("metrics %+v errs %v, want lhr data after its restart", metrics, errs) + } + if counterValue(retries) < 6 || counterValue(recoveries) != 1 || counterValue(cancels) != 1 { + t.Fatalf("retries %v recoveries %v cancels %v", counterValue(retries), counterValue(recoveries), counterValue(cancels)) + } +} + func TestConnectRetryKeepsFailuresAfterAccept(t *testing.T) { for _, failure := range []error{ bnet.ErrHTTPCode(http.StatusServiceUnavailable), diff --git a/pkg/backend/render_fanin.go b/pkg/backend/render_fanin.go index f917e6f6..cbe1106c 100644 --- a/pkg/backend/render_fanin.go +++ b/pkg/backend/render_fanin.go @@ -14,6 +14,7 @@ type renderGroup struct { results chan []types.Metric errors chan error answered bool + data bool // a member returned at least one series errs []error } @@ -44,26 +45,47 @@ func newRenderGroups(backends []Backend) ([]renderGroup, []int) { return groups, membership } -// receive collects n replies and returns how many of them were answers: data -// or a backend's own not-found. +// next waits for one member's reply. +func (g *renderGroup) next() ([]types.Metric, error) { + return nextReply(g.results, g.errors) +} + +func nextReply(results <-chan []types.Metric, errs <-chan error) ([]types.Metric, error) { + select { + case metrics := <-results: + return metrics, nil + case err := <-errs: + return nil, err + } +} + +// take records one member's reply and reports whether it was an answer: data +// or the backend's own not-found. +func (g *renderGroup) take(metrics []types.Metric, err error, messages [][]types.Metric) ([][]types.Metric, bool) { + if err != nil { + g.errs = append(g.errs, err) + var notFound types.ErrNotFound + if !errors.As(err, ¬Found) { + return messages, false + } + } else { + messages = append(messages, metrics) + g.data = g.data || len(metrics) != 0 + } + g.answered = true + return messages, true +} + +// receive collects n replies and returns how many of them were answers. func (g *renderGroup) receive(n int, messages [][]types.Metric) ([][]types.Metric, int) { answers := 0 for range n { - select { - case metrics := <-g.results: - messages = append(messages, metrics) + metrics, err := g.next() + var answer bool + if messages, answer = g.take(metrics, err, messages); answer { answers++ - case err := <-g.errors: - g.errs = append(g.errs, err) - var notFound types.ErrNotFound - if errors.As(err, ¬Found) { - answers++ - } } } - if answers != 0 { - g.answered = true - } return messages, answers } diff --git a/pkg/cfg/common.go b/pkg/cfg/common.go index 4734414e..76456bcc 100644 --- a/pkg/cfg/common.go +++ b/pkg/cfg/common.go @@ -376,7 +376,9 @@ type Timeouts struct { Connect time.Duration `yaml:"connect"` // ConnectRetry is how long a render keeps retrying a backend that refused, // reset or timed out the connection before sending any response, as during - // a store restart. Zero disables the retry. + // a store restart. The retry stops as soon as a replica group of the same + // cluster in another DC has returned data for the target. Zero disables + // the retry. ConnectRetry time.Duration `yaml:"connectRetry"` } From a15501c44e05c4cf891cd38761030014f4afa8de Mon Sep 17 00:00:00 2001 From: Anton Timofieiev Date: Fri, 13 Nov 2026 11:27:29 +0100 Subject: [PATCH 4/8] fix(types): merge replicas from the freshest reply and keep filling past mismatches mergeMetrics used the first reply as the base and stopped healing at the first reply whose step or point count differed from it. A store that answered first with a shorter series (for example while replaying after a restart) became the base and disabled healing for every reply after it, so renders showed nulls that other replicas held. When the finest-step replies differ in point count, the freshest one (latest present point, then most present points, then reply order) now decides the shape, and the base is the first reply with that shape. Replies of another shape are skipped one by one instead of ending the list. Replies of one shape still merge in reply order, so values, majority voting and mismatch counts are bit-identical to before; a differential test against the old algorithm checks this. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/types/merge_replicas_test.go | 524 +++++++++++++++++++++++++++++++ pkg/types/packed_points.go | 46 +++ pkg/types/types.go | 96 +++++- 3 files changed, 649 insertions(+), 17 deletions(-) create mode 100644 pkg/types/merge_replicas_test.go diff --git a/pkg/types/merge_replicas_test.go b/pkg/types/merge_replicas_test.go new file mode 100644 index 00000000..9dec9928 --- /dev/null +++ b/pkg/types/merge_replicas_test.go @@ -0,0 +1,524 @@ +package types + +import ( + "math" + "math/rand" + "reflect" + "sort" + "testing" + + "github.com/bookingcom/carbonapi/pkg/cfg" +) + +// legacyMergeMetrics is mergeMetrics as of bk-master 89d6f213, before the +// freshest-shape base and per-reply skipping. The differential tests below use +// it as the oracle. +func legacyMergeMetrics(metrics []Metric, replicaMismatchConfig cfg.RenderReplicaMismatchConfig) (metric Metric, stats MetricRenderStats) { + if len(metrics) == 0 { + return Metric{}, MetricRenderStats{} + } + + if len(metrics) == 1 { + m := metrics[0] + m.materializePoints() + return m, MetricRenderStats{} + } + + var equalityFunc floatEqualityFunc + if replicaMismatchConfig.RenderReplicaMismatchApproximateCheck { + equalityFunc = AreFloatsApproximatelyEqual + } + + replicaMatchMode := replicaMismatchConfig.RenderReplicaMatchMode + + var mismatches, fixedMismatches int + + sort.Sort(byStepTime(metrics)) + healed := 0 + + clusterMap := make(map[string]bool) + for _, m := range metrics { + for _, cl := range m.SourceClusters { + clusterMap[cl] = true + } + } + clusters := make([]string, 0, len(clusterMap)) + for cl := range clusterMap { + clusters = append(clusters, cl) + } + sort.Strings(clusters) + // metrics[0] has the highest resolution of metrics + metric = metrics[0] + metric.materializePoints() + var cursors []packedPointCursor + for _, replica := range metrics[1:] { + if replica.packed != nil { + cursors = make([]packedPointCursor, len(metrics)) + for i := range cursors { + cursors[i].block = -1 + } + break + } + } + metric.SourceClusters = clusters + // Schema compatibility is constant across all points. Preserve the old + // first-incompatible-replica boundary without rechecking it per point. + replicaLimit := 1 + for replicaLimit < len(metrics) && metrics[replicaLimit].StepTime == metric.StepTime && metrics[replicaLimit].pointCount() == len(metric.Values) { + replicaLimit++ + } + valuesForPoint := make([]float64, 0, len(metrics)) + isMismatchFindConfig := replicaMatchMode != cfg.ReplicaMatchModeNormal + for i := range metric.Values { + if cursors != nil && i&63 == 0 { + for j := 1; j < replicaLimit; j++ { + if p := metrics[j].packed; p != nil { + cursors[j].advanceToBlock(p, i>>6) + } + } + } + pointExists := !metric.IsAbsent[i] + shouldLookForMismatch := isMismatchFindConfig + mismatchObserved := false + valuesForPoint = valuesForPoint[:0] + if pointExists { + valuesForPoint = append(valuesForPoint, metric.Values[i]) + } + for j := 1; j < replicaLimit; j++ { + if pointExists && !shouldLookForMismatch { + break + } + m := metrics[j] + + var value float64 + if m.packed != nil { + var absent bool + value, absent = cursors[j].pointAt(m.packed, i) + if absent { + continue + } + } else { + if m.IsAbsent[i] { + continue + } + value = m.Values[i] + } + + valuesForPoint = append(valuesForPoint, value) + + if !pointExists { + metric.IsAbsent[i] = false + metric.Values[i] = value + healed++ + pointExists = true + } + + if !mismatchObserved { + mismatchObserved = (equalityFunc == nil && metric.Values[i] != value) || + (equalityFunc != nil && !equalityFunc(metric.Values[i], value)) + if mismatchObserved && replicaMatchMode == cfg.ReplicaMatchModeCheck { + // mismatch exists, enough for check mode + shouldLookForMismatch = false + } + } + } + if !mismatchObserved { + continue + } + + mismatches++ + if replicaMatchMode == cfg.ReplicaMatchModeMajority { + majorityValue, isMajority, err := getPointMajorityValue(valuesForPoint, equalityFunc) + if err == nil && isMajority { + metric.Values[i] = majorityValue + fixedMismatches++ + } + } + } + + return metric, MetricRenderStats{ + DataPointCount: len(metric.Values), + MismatchCount: mismatches, + FixedMismatchCount: fixedMismatches, + } +} + +var mergeTestConfigs = func() []cfg.RenderReplicaMismatchConfig { + var configs []cfg.RenderReplicaMismatchConfig + for _, mode := range []cfg.ReplicaMatchMode{cfg.ReplicaMatchModeNormal, cfg.ReplicaMatchModeCheck, cfg.ReplicaMatchModeMajority} { + for _, approx := range []bool{false, true} { + configs = append(configs, cfg.RenderReplicaMismatchConfig{RenderReplicaMatchMode: mode, RenderReplicaMismatchApproximateCheck: approx}) + } + } + return configs +}() + +// mergeTestInput returns an independent copy of replies, packing the flagged +// ones, because mergeMetrics sorts its input and heals the base in place. +func mergeTestInput(ms []Metric, packed []bool) []Metric { + out := make([]Metric, len(ms)) + for i, m := range ms { + m.Values = append([]float64(nil), m.Values...) + m.IsAbsent = append([]bool(nil), m.IsAbsent...) + m.SourceClusters = append([]string(nil), m.SourceClusters...) + if packed != nil && packed[i] { + m.PackPoints() + } + out[i] = m + } + return out +} + +func assertSameMerge(t *testing.T, got Metric, gotStats MetricRenderStats, want Metric, wantStats MetricRenderStats) { + t.Helper() + if gotStats != wantStats { + t.Fatalf("stats differ: got %+v, want %+v", gotStats, wantStats) + } + if got.packed != nil || want.packed != nil || len(got.Values) != len(want.Values) { + t.Fatalf("unmaterialized or different length output: got %d points, want %d", len(got.Values), len(want.Values)) + } + for i := range got.Values { + if math.Float64bits(got.Values[i]) != math.Float64bits(want.Values[i]) { + t.Fatalf("value bits differ at %d: got %x, want %x", i, math.Float64bits(got.Values[i]), math.Float64bits(want.Values[i])) + } + } + got.Values, want.Values = nil, nil + if !reflect.DeepEqual(got, want) { + t.Fatalf("metadata or absence differs:\ngot %+v\nwant %+v", got, want) + } +} + +// mergeTestReply builds a 60 s reply starting at start; NaN marks an absent point. +func mergeTestReply(cluster string, start int32, values ...float64) Metric { + m := Metric{ + Name: "metric", + StartTime: start, + StopTime: start + 60*int32(len(values)), + StepTime: 60, + Values: make([]float64, len(values)), + IsAbsent: make([]bool, len(values)), + SourceClusters: []string{cluster}, + } + for i, v := range values { + if math.IsNaN(v) { + m.IsAbsent[i] = true + } else { + m.Values[i] = v + } + } + return m +} + +func assertMergedPoints(t *testing.T, got Metric, want Metric) { + t.Helper() + if got.StartTime != want.StartTime || got.StopTime != want.StopTime || !reflect.DeepEqual(got.IsAbsent, want.IsAbsent) || !reflect.DeepEqual(got.Values, want.Values) { + t.Fatalf("got %d-%d %v %v, want %d-%d %v %v", got.StartTime, got.StopTime, got.Values, got.IsAbsent, want.StartTime, want.StopTime, want.Values, want.IsAbsent) + } +} + +var nan = math.NaN() + +func TestMergeBaseTieBreaks(t *testing.T) { + for _, tc := range []struct { + name string + replies []Metric + want int + }{ + {"one shape keeps the first reply", []Metric{mergeTestReply("A", 600, 1, nan), mergeTestReply("B", 600, 1, 2)}, 0}, + {"latest present point decides the shape", []Metric{mergeTestReply("A", 600, 1, 2, 3), mergeTestReply("B", 600, 1, 2, 3, 4)}, 1}, + {"then most present points", []Metric{mergeTestReply("A", 600, nan, nan, 3, 4, 5), mergeTestReply("B", 660, 2, 3, 4, 5)}, 1}, + {"then reply order", []Metric{mergeTestReply("A", 600, nan, 2, 3, 4, 5), mergeTestReply("B", 660, 2, 3, 4, 5)}, 0}, + {"base is the first reply of the freshest shape", []Metric{mergeTestReply("A", 600, 1, 2, 3, nan, nan), mergeTestReply("B", 600, 1, 2, 3), mergeTestReply("C", 600, 1, 2, 3, 4, 5)}, 0}, + {"all absent keeps the first reply", []Metric{mergeTestReply("A", 600, nan, nan), mergeTestReply("B", 600, nan)}, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := mergeBase(tc.replies); got != tc.want { + t.Fatalf("mergeBase = %d, want %d", got, tc.want) + } + packed := mergeTestInput(tc.replies, nil) + for i := range packed { + packed[i].packed = &packedPoints{n: len(packed[i].Values), values: make([]uint64, (len(packed[i].Values)+63)/64), absent: make([]uint64, (len(packed[i].Values)+63)/64)} + for j, absent := range packed[i].IsAbsent { + if absent { + packed[i].packed.absent[j/64] |= uint64(1) << (uint(j) & 63) + } + } + packed[i].Values, packed[i].IsAbsent = nil, nil + } + if got := mergeBase(packed); got != tc.want { + t.Fatalf("packed mergeBase = %d, want %d", got, tc.want) + } + }) + } +} + +// A store that answers first but still lacks its newest points must not +// leave nulls that another reply can fill. +func TestMergeMetricsStaleFirstReplyHealedFromFresherReply(t *testing.T) { + for _, tc := range []struct { + name string + replies []Metric + want Metric + legacyBroken bool + }{ + { + name: "trailing absent points", + replies: []Metric{mergeTestReply("A", 600, 1, 2, 3, nan, nan), mergeTestReply("B", 600, 1, 2, 3, 4, 5)}, + want: mergeTestReply("", 600, 1, 2, 3, 4, 5), + }, + { + name: "shorter series", + replies: []Metric{mergeTestReply("A", 600, 1, 2, 3), mergeTestReply("B", 600, 1, 2, 3, 4, 5)}, + want: mergeTestReply("", 600, 1, 2, 3, 4, 5), + legacyBroken: true, + }, + { + name: "shorter, stale and complete replies", + replies: []Metric{ + mergeTestReply("A", 600, 1, 2, 3), + mergeTestReply("B", 600, 1, 2, 3, nan, nan), + mergeTestReply("C", 600, 1, nan, 3, 4, 5), + }, + want: mergeTestReply("", 600, 1, 2, 3, 4, 5), + legacyBroken: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + for _, c := range mergeTestConfigs { + got, stats := mergeMetrics(mergeTestInput(tc.replies, nil), c) + assertMergedPoints(t, got, tc.want) + if stats.MismatchCount != 0 { + t.Fatalf("%s: %d mismatches, want 0", c.String(), stats.MismatchCount) + } + legacy, _ := legacyMergeMetrics(mergeTestInput(tc.replies, nil), c) + if broken := !reflect.DeepEqual(legacy.Values, tc.want.Values) || !reflect.DeepEqual(legacy.IsAbsent, tc.want.IsAbsent); broken != tc.legacyBroken { + t.Fatalf("%s: legacy merge broken=%v, want %v (%v %v)", c.String(), broken, tc.legacyBroken, legacy.Values, legacy.IsAbsent) + } + } + }) + } +} + +// A reply with another point count in the middle of the list is skipped on +// its own; the replies after it still fill the base's gaps. +func TestMergeMetricsSkipsOnlyMismatchedReply(t *testing.T) { + replies := []Metric{ + mergeTestReply("A", 600, 1, nan, 3, 4, 5), + mergeTestReply("B", 600, 1, 2, 3, 4), + mergeTestReply("C", 600, 1, 2, 3, 4, nan), + } + want := mergeTestReply("", 600, 1, 2, 3, 4, 5) + for _, c := range mergeTestConfigs { + got, stats := mergeMetrics(mergeTestInput(replies, nil), c) + assertMergedPoints(t, got, want) + if stats.MismatchCount != 0 || stats.DataPointCount != 5 { + t.Fatalf("%s: stats %+v", c.String(), stats) + } + if legacy, _ := legacyMergeMetrics(mergeTestInput(replies, nil), c); !legacy.IsAbsent[1] { + t.Fatalf("%s: legacy merge filled the gap; the test no longer covers the old limit", c.String()) + } + } +} + +func TestMergeMetricsMajorityUnchangedOnDisagreement(t *testing.T) { + majority := cfg.RenderReplicaMismatchConfig{RenderReplicaMatchMode: cfg.ReplicaMatchModeMajority} + + // One shape, first reply stale: the base stays the first reply, so a + // strict majority wins and an unresolved conflict keeps the base value, + // exactly as before. + replies := []Metric{ + mergeTestReply("A", 600, 1, 2, 7, nan), + mergeTestReply("B", 600, 1, 5, 8, 10), + mergeTestReply("C", 600, 1, 5, 9, 11), + } + got, stats := mergeMetrics(mergeTestInput(replies, nil), majority) + assertMergedPoints(t, got, mergeTestReply("", 600, 1, 5, 7, 10)) + if stats != (MetricRenderStats{DataPointCount: 4, MismatchCount: 3, FixedMismatchCount: 1}) { + t.Fatalf("stats %+v", stats) + } + for _, c := range mergeTestConfigs { + got, gs := mergeMetrics(mergeTestInput(replies, nil), c) + want, ws := legacyMergeMetrics(mergeTestInput(replies, nil), c) + assertSameMerge(t, got, gs, want, ws) + } + + // A shorter reply in the middle no longer removes the replies after it + // from the vote. + replies = []Metric{ + mergeTestReply("A", 600, 1, 2, 3, nan), + mergeTestReply("B", 600, 1, 5, 3), + mergeTestReply("C", 600, 1, 5, 3, 4), + mergeTestReply("D", 600, 1, 5, 9, 4), + } + got, stats = mergeMetrics(mergeTestInput(replies, nil), majority) + assertMergedPoints(t, got, mergeTestReply("", 600, 1, 5, 3, 4)) + if stats != (MetricRenderStats{DataPointCount: 4, MismatchCount: 2, FixedMismatchCount: 2}) { + t.Fatalf("stats %+v", stats) + } +} + +var mergeTestValueBits = []uint64{ + 0, 1 << 63, // +0 and -0 + 0x7ff0000000000000, 0xfff0000000000000, // +Inf and -Inf + 0x7ff8000000000000, 0x7ff8000000000123, 0xfff8000000000001, // NaN payloads + 0x3fd3333333333333, 0x3fd3333333333334, // 0.3 and 0.1+0.2 + 1, +} + +func mergeTestValue(rng *rand.Rand) float64 { + if rng.Intn(3) == 0 { + return math.Float64frombits(mergeTestValueBits[rng.Intn(len(mergeTestValueBits))]) + } + // A small value range so replies often agree and majorities form. + return float64(rng.Intn(4)) +} + +func mergeTestAbsence(rng *rand.Rand, n int) []bool { + absent := make([]bool, n) + density := []float64{0, 0.05, 0.5, 0.95, 1}[rng.Intn(5)] + for i := range absent { + absent[i] = rng.Float64() < density + } + if rng.Intn(3) == 0 { // stale tail, as from a store still replaying + for i := n - rng.Intn(n+1); i < n; i++ { + absent[i] = true + } + } + if rng.Intn(4) == 0 { // leading gap + for i, end := 0, rng.Intn(n+1); i < end; i++ { + absent[i] = true + } + } + return absent +} + +var mergeTestSizes = []int{1, 2, 5, 64, 129, 300} + +// Replies of one shape whose present points agree must merge bit-identically +// to the old algorithm, including NaN payloads, absent-slot payloads and the +// mismatch counts, in every match mode. +func TestMergeMetricsAgreeingRepliesMatchLegacy(t *testing.T) { + rng := rand.New(rand.NewSource(20261113)) + for iter := 0; iter < 300; iter++ { + n := mergeTestSizes[iter%len(mergeTestSizes)] + values, absentValues := make([]float64, n), make([]float64, n) + for i := range values { + values[i] = mergeTestValue(rng) + // Decoders write 0 or NaN into absent slots. + absentValues[i] = math.Float64frombits([]uint64{0, 0x7ff8000000000001}[rng.Intn(2)]) + } + var replies []Metric + var packed []bool + for r := 0; r < 2+rng.Intn(4); r++ { + m := Metric{Name: "a", StartTime: 600, StopTime: 600 + 60*int32(n), StepTime: 60, Values: make([]float64, n), IsAbsent: mergeTestAbsence(rng, n), SourceClusters: []string{string(rune('A' + r))}} + for i := range m.Values { + m.Values[i] = values[i] + if m.IsAbsent[i] { + m.Values[i] = absentValues[i] + } + } + replies = append(replies, m) + packed = append(packed, rng.Intn(2) == 0) + } + if rng.Intn(3) == 0 { // a coarser archive, ignored by both + coarse := Metric{Name: "a", StartTime: 600, StopTime: 600 + 120*int32(n/2+1), StepTime: 120, Values: make([]float64, n/2+1), IsAbsent: mergeTestAbsence(rng, n/2+1), SourceClusters: []string{"Z"}} + for i := range coarse.Values { + coarse.Values[i] = mergeTestValue(rng) + } + at := rng.Intn(len(replies) + 1) + replies = append(replies[:at], append([]Metric{coarse}, replies[at:]...)...) + packed = append(packed[:at], append([]bool{false}, packed[at:]...)...) + } + for _, c := range mergeTestConfigs { + want, ws := legacyMergeMetrics(mergeTestInput(replies, nil), c) + got, gs := mergeMetrics(mergeTestInput(replies, packed), c) + assertSameMerge(t, got, gs, want, ws) + } + } +} + +// mergeTestFreshest is a direct statement of the base rule: among the +// finest-step replies, the latest present point, then the most present +// points, then reply order. +func mergeTestFreshest(sorted []Metric) int { + key := func(m Metric) (int64, int) { + last, present := int64(math.MinInt64), 0 + for i, absent := range m.IsAbsent { + if !absent { + last = int64(m.StartTime) + int64(i)*int64(m.StepTime) + present++ + } + } + return last, present + } + best := 0 + for j := 1; j < len(sorted) && sorted[j].StepTime == sorted[0].StepTime; j++ { + lj, pj := key(sorted[j]) + lb, pb := key(sorted[best]) + if lj > lb || (lj == lb && pj > pb) { + best = j + } + } + return best +} + +// For any mix of shapes and values, including disagreeing values in majority +// mode, the merge equals the old algorithm run on the replies of the freshest +// shape (in reply order) followed by the others, which it then ignores. +func TestMergeMetricsMixedShapesMatchLegacyOnFreshestShape(t *testing.T) { + rng := rand.New(rand.NewSource(20261114)) + for iter := 0; iter < 300; iter++ { + n := mergeTestSizes[iter%len(mergeTestSizes)] + shared := make([]float64, n+4) + for i := range shared { + shared[i] = mergeTestValue(rng) + } + var replies []Metric + var packed []bool + for r := 0; r < 2+rng.Intn(5); r++ { + count, start, step := n, int32(600), int32(60) + switch rng.Intn(5) { + case 0: // shorter: missing its newest or oldest points + count = max(1, n-1-rng.Intn(3)) + start += 60 * int32(rng.Intn(2)*(n-count)) + case 1: // longer + count = n + 1 + rng.Intn(3) + start -= 60 * int32(rng.Intn(2)*(count-n)) + case 2: + if rng.Intn(3) == 0 { + step, count = 120, n/2+1 + } + } + m := Metric{Name: "a", StartTime: start, StopTime: start + step*int32(count), StepTime: step, Values: make([]float64, count), IsAbsent: mergeTestAbsence(rng, count), SourceClusters: []string{string(rune('A' + r))}} + agree := rng.Intn(2) == 0 + for i := range m.Values { + if agree { + m.Values[i] = shared[i] + } else { + m.Values[i] = mergeTestValue(rng) + } + } + replies = append(replies, m) + packed = append(packed, rng.Intn(2) == 0) + } + + sorted := mergeTestInput(replies, nil) + sort.Stable(byStepTime(sorted)) + freshest := sorted[mergeTestFreshest(sorted)] + var ordered, rest []Metric + for _, m := range sorted { + if m.StepTime == freshest.StepTime && len(m.Values) == len(freshest.Values) { + ordered = append(ordered, m) + } else { + rest = append(rest, m) + } + } + ordered = append(ordered, rest...) + + for _, c := range mergeTestConfigs { + want, ws := legacyMergeMetrics(mergeTestInput(ordered, nil), c) + got, gs := mergeMetrics(mergeTestInput(replies, packed), c) + assertSameMerge(t, got, gs, want, ws) + } + } +} diff --git a/pkg/types/packed_points.go b/pkg/types/packed_points.go index 68c03d07..85c1de14 100644 --- a/pkg/types/packed_points.go +++ b/pkg/types/packed_points.go @@ -89,6 +89,52 @@ func (m Metric) pointCount() int { return len(m.Values) } +// lastPresentTime returns the timestamp of the newest present point, or +// math.MinInt64 when every point is absent. +func (m Metric) lastPresentTime() int64 { + last := -1 + if p := m.packed; p != nil { + for block := len(p.absent) - 1; block >= 0 && last < 0; block-- { + present := ^p.absent[block] + if valid := p.n - block*64; valid < 64 { + present &= uint64(1)<= 0; i-- { + if !m.IsAbsent[i] { + last = i + break + } + } + } + if last < 0 { + return math.MinInt64 + } + return int64(m.StartTime) + int64(last)*int64(m.StepTime) +} + +// presentCount returns the number of present points. +func (m Metric) presentCount() int { + if p := m.packed; p != nil { + absent := 0 + for _, block := range p.absent { + absent += bits.OnesCount64(block) + } + return p.n - absent + } + present := 0 + for _, absent := range m.IsAbsent { + if !absent { + present++ + } + } + return present +} + func (c *packedPointCursor) advanceToBlock(p *packedPoints, block int) { for c.block < block { c.offset = c.next diff --git a/pkg/types/types.go b/pkg/types/types.go index 2ff38da6..f6da66f5 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -289,6 +289,63 @@ func getPointMajorityValue(values []float64, equalityFunc floatEqualityFunc) (ma return m, majorityCount > valuesCount/2, nil } +// mergeBase returns the index of the reply that mergeMetrics starts from. +// metrics must be sorted by step, finest first. +// +// When all finest-step replies have the same point count, the base is the +// first reply, as it always was: replies of one shape heal each other's +// absent points in reply order whichever of them is the base. +// +// When their point counts differ, the freshest finest-step reply decides the +// shape: the one with the latest present point, then the most present points, +// then the earliest in reply order. The base is the first reply with that +// shape. A reply that lacks its newest points, for example from a store still +// replaying after a restart, therefore cannot cut the merged series short. +func mergeBase(metrics []Metric) int { + step, count := metrics[0].StepTime, metrics[0].pointCount() + sameShape := true + for _, m := range metrics[1:] { + if m.StepTime != step { + break + } + if m.pointCount() != count { + sameShape = false + break + } + } + if sameShape { + return 0 + } + + freshest := 0 + freshestLast, freshestPresent := metrics[0].lastPresentTime(), -1 + for j := 1; j < len(metrics) && metrics[j].StepTime == step; j++ { + last := metrics[j].lastPresentTime() + if last < freshestLast { + continue + } + present := -1 + if last == freshestLast { + if freshestPresent < 0 { + freshestPresent = metrics[freshest].presentCount() + } + present = metrics[j].presentCount() + if present <= freshestPresent { + continue + } + } + freshest, freshestLast, freshestPresent = j, last, present + } + + count = metrics[freshest].pointCount() + for j := 0; j < freshest; j++ { + if metrics[j].pointCount() == count { + return j + } + } + return freshest +} + func mergeMetrics(metrics []Metric, replicaMismatchConfig cfg.RenderReplicaMismatchConfig) (metric Metric, stats MetricRenderStats) { if len(metrics) == 0 { return Metric{}, MetricRenderStats{} @@ -323,31 +380,36 @@ func mergeMetrics(metrics []Metric, replicaMismatchConfig cfg.RenderReplicaMisma clusters = append(clusters, cl) } sort.Strings(clusters) - // metrics[0] has the highest resolution of metrics - metric = metrics[0] + base := mergeBase(metrics) + metric = metrics[base] metric.materializePoints() + // Every reply with the base's step and point count heals the base, in reply + // order; the base is the first reply of its shape, so they all follow it. + // A reply of another shape is skipped on its own: it no longer hides the + // replies after it. Schema compatibility is constant across all points, so + // it is checked once here and not per point. + var peerBuf [8]int + peers := peerBuf[:0] + packedPeer := false + for j := base + 1; j < len(metrics); j++ { + if metrics[j].StepTime == metric.StepTime && metrics[j].pointCount() == len(metric.Values) { + peers = append(peers, j) + packedPeer = packedPeer || metrics[j].packed != nil + } + } var cursors []packedPointCursor - for _, replica := range metrics[1:] { - if replica.packed != nil { - cursors = make([]packedPointCursor, len(metrics)) - for i := range cursors { - cursors[i].block = -1 - } - break + if packedPeer { + cursors = make([]packedPointCursor, len(metrics)) + for i := range cursors { + cursors[i].block = -1 } } metric.SourceClusters = clusters - // Schema compatibility is constant across all points. Preserve the old - // first-incompatible-replica boundary without rechecking it per point. - replicaLimit := 1 - for replicaLimit < len(metrics) && metrics[replicaLimit].StepTime == metric.StepTime && metrics[replicaLimit].pointCount() == len(metric.Values) { - replicaLimit++ - } valuesForPoint := make([]float64, 0, len(metrics)) isMismatchFindConfig := replicaMatchMode != cfg.ReplicaMatchModeNormal for i := range metric.Values { if cursors != nil && i&63 == 0 { - for j := 1; j < replicaLimit; j++ { + for _, j := range peers { if p := metrics[j].packed; p != nil { cursors[j].advanceToBlock(p, i>>6) } @@ -360,7 +422,7 @@ func mergeMetrics(metrics []Metric, replicaMismatchConfig cfg.RenderReplicaMisma if pointExists { valuesForPoint = append(valuesForPoint, metric.Values[i]) } - for j := 1; j < replicaLimit; j++ { + for _, j := range peers { if pointExists && !shouldLookForMismatch { break } From d0b08f366882785b46c9637cd9a51775c0007c7f Mon Sep 17 00:00:00 2001 From: Anton Timofieiev Date: Fri, 13 Nov 2026 11:49:52 +0100 Subject: [PATCH 5/8] fix(zipper): cancel connect retries once the cluster answered not-found Connect retries were given up only when another replica group of the same cluster returned data. Renders fan out to every cluster, so most fetches to a refused store are for names held in other clusters; there the refused cluster's other groups can only answer not-found, the fetch was never covered, and it still waited the whole connectRetry window. A production canary (30s refusal of a one-store replica group) got 41 HTTP 500 and 67 499 in 30s against 1 and 1 on a peer instance. Replica groups of one cluster hold the same names, so any definitive answer from another group of the cluster, data or not-found, now covers the target: the refused fetch is given up before the first retry, and repeats in flight to other groups of that cluster stop being awaited once one group answers. Retries continue only while no other group of the cluster has answered at all (every copy unreachable), the restart case the connect retry was for. Server errors, answers from another cluster and ungrouped backends still never cancel a retry. A cancelled fetch keeps its plain connect error, so the required-fetch and replica-group rules decide the render as before: a required fetch no group has data for still fails. backend_connect_retries_cancelled_total now also counts not-found cover. Co-Authored-By: Claude Opus 5.5 (1M context) --- config/carbonzipper.yaml | 3 +- pkg/app/carbonapi/connect_retry_test.go | 55 ++++++++++++++ pkg/app/carbonapi/metrics.go | 2 +- pkg/backend/connect_retry.go | 43 +++++++---- pkg/backend/connect_retry_test.go | 97 ++++++++++++++++++++++++- pkg/cfg/common.go | 4 +- 6 files changed, 181 insertions(+), 23 deletions(-) diff --git a/config/carbonzipper.yaml b/config/carbonzipper.yaml index d42c981a..69fe23a0 100644 --- a/config/carbonzipper.yaml +++ b/config/carbonzipper.yaml @@ -23,7 +23,8 @@ timeouts: connect: "200ms" # How long a render keeps retrying a backend that refused, reset or timed # out the connection before responding (a store restart), while no replica - # group of the same cluster in another DC has returned data. 0 disables. + # group of the same cluster in another DC has answered (data or not-found). + # 0 disables. connectRetry: "10s" # Number of concurrent requests to any given backend - default is no limit. diff --git a/pkg/app/carbonapi/connect_retry_test.go b/pkg/app/carbonapi/connect_retry_test.go index a854d7f0..2f2a8fb8 100644 --- a/pkg/app/carbonapi/connect_retry_test.go +++ b/pkg/app/carbonapi/connect_retry_test.go @@ -5,6 +5,7 @@ import ( "io" "net" "net/http" + "net/http/httptest" "net/url" "os" "sync/atomic" @@ -102,3 +103,57 @@ func TestRenderRetriesRestartingStore(t *testing.T) { }) } } + +// The fe-112 canary scenario end to end: one cluster with three replica groups, +// the one-store dc1 group refused, the dc3 and dc2 groups answering +// not-found. The render returns without the retry wait. With the series held +// by another cluster it is complete; with the series nowhere, the response is +// the one the same stores give with retries disabled (the required-fetch and replica-group rules unchanged). +func TestRenderRefusedStoreOtherGroupsNotFound(t *testing.T) { + refused := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", + Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} + for _, tc := range []struct { + name string + dataCluster bool + }{ + {name: "series_in_other_cluster", dataCluster: true}, + {name: "series_nowhere"}, + } { + t.Run(tc.name, func(t *testing.T) { + run := func(window time.Duration) (*restartingStore, int, string, time.Duration) { + backend.SetConnectRetry(backend.ConnectRetry{Window: window, InitialDelay: 10 * time.Millisecond, MaxDelay: 50 * time.Millisecond}) + t.Cleanup(func() { backend.SetConnectRetry(backend.ConnectRetry{}) }) + store := &restartingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "carbon", dc: "dc1"}, + refusals: 1 << 30, failure: refused} + stores := []backend.BackendImpl{store, + &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: "carbon", dc: "dc3"}, + &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: "carbon", dc: "dc2"}, + &partialRenderBackend{path: "m.ko", cluster: "cy", dc: "dc1"}} + if tc.dataCluster { + stores = append(stores, &partialRenderBackend{path: "m.ok", cluster: "cy", dc: "dc2"}) + } + app := partialRenderAppOf(t, stores...) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + return store, w.Code, w.Body.String(), time.Since(start) + } + store, code, body, elapsed := run(5 * time.Second) + if elapsed > 200*time.Millisecond || store.calls.Load() != 1 { + t.Fatalf("took %v and %d store calls, want no retry wait", elapsed, store.calls.Load()) + } + _, baseCode, baseBody, _ := run(0) + if code != baseCode || body != baseBody { + t.Fatalf("HTTP %d %s, without retries HTTP %d %s", code, body, baseCode, baseBody) + } + if tc.dataCluster { + w := httptest.NewRecorder() + w.Code = code + w.Body.WriteString(body) + assertPartialRenderValue(t, w, 100) + } else if code != http.StatusInternalServerError { + // A required series no group has data for still fails (the required-fetch and replica-group rules). + t.Fatalf("HTTP %d, want 500 as without retries: %s", code, body) + } + }) + } +} diff --git a/pkg/app/carbonapi/metrics.go b/pkg/app/carbonapi/metrics.go index 031800c8..fda87089 100644 --- a/pkg/app/carbonapi/metrics.go +++ b/pkg/app/carbonapi/metrics.go @@ -484,7 +484,7 @@ func NewZipperPrometheusMetrics(config cfg.Zipper) *ZipperPrometheusMetrics { BackendConnectCancels: prometheus.NewCounter( prometheus.CounterOpts{ Name: "backend_connect_retries_cancelled_total", - Help: "Render fetches of an unreachable backend given up without waiting for the retry window because another replica group of the same cluster returned data", + Help: "Render fetches of an unreachable backend given up without waiting for the retry window because another replica group of the same cluster answered (data or not-found)", }, ), } diff --git a/pkg/backend/connect_retry.go b/pkg/backend/connect_retry.go index 31cc8952..462ce3aa 100644 --- a/pkg/backend/connect_retry.go +++ b/pkg/backend/connect_retry.go @@ -27,7 +27,8 @@ type ConnectRetry struct { Retries prometheus.Counter // repeated fetches Recoveries prometheus.Counter // repeated fetches that the backend answered // Cancellations counts unreachable fetches given up because another - // replica group of the same cluster returned data for the target. + // replica group of the same cluster answered for the target (data or + // not-found). Cancellations prometheus.Counter } @@ -85,10 +86,14 @@ type renderFetch struct { // retryUnreachable repeats unreachable fetches that no replica group has // answered, with one backoff schedule shared by all fetches of a request. // A fetch is repeated only while no other replica group of its cluster has -// returned data for the target: once one has, the merged result is complete -// without the unreachable store, and waiting for it would only delay the -// response. It returns early when the request deadline cannot fit another -// attempt. +// answered for the target at all. Replica groups of one cluster hold the same +// names, so a definitive answer from any of them, data or not-found, already +// says what the unreachable store would return: with data the merged result is +// complete without it, and with not-found the names live in another cluster +// (renders fan out to every cluster) or nowhere. Waiting for the store would +// only delay the response. Retries are for the case where every copy of the +// cluster is unreachable, such as a one-group cluster restarting. It returns +// early when the request deadline cannot fit another attempt. func retryUnreachable(ctx context.Context, fetches []renderFetch, lg *zap.Logger) { policy := connectRetry.Load() if policy == nil || policy.Window <= 0 { @@ -184,9 +189,9 @@ type retryReply struct { } // awaitRetries collects the replies of one attempt. As soon as a replica group -// returns data, the same fetch's repeats to other groups of that cluster are no -// longer awaited: they keep their unreachable error for cancelCovered, and -// their late replies are dropped. +// answers (data or not-found), the same fetch's repeats to other groups of that +// cluster are no longer awaited: they keep their unreachable error for +// cancelCovered, and their late replies are dropped. func awaitRetries(fetches []renderFetch, flights []retryFlight) { outstanding := 0 for i := range flights { @@ -219,7 +224,7 @@ func awaitRetries(fetches []renderFetch, flights []retryFlight) { continue } fl.answered++ - if !g.data || g.cluster == "" { + if g.cluster == "" { continue } for j := range flights { @@ -238,9 +243,11 @@ func awaitRetries(fetches []renderFetch, flights []retryFlight) { } // cancelCovered gives up the unreachable fetches of every replica group whose -// target another group of the same cluster has returned data for. Their -// errors become final, so the required-fetch and replica-group rules decide the render exactly as without -// retries. Ungrouped backends are never covered: they never recover each +// target another group of the same cluster has answered for, with data or +// not-found. Their errors become final plain connect errors, so the required-fetch and replica-group rules +// decide the render exactly as without retries: another group's data still +// does not recover them, and a required fetch that no group has data for +// still fails. Ungrouped backends are never covered: they never recover each // other's errors. func cancelCovered(fetches []renderFetch, policy *ConnectRetry, lg *zap.Logger) { for fi := range fetches { @@ -263,7 +270,7 @@ func cancelCovered(fetches []renderFetch, policy *ConnectRetry, lg *zap.Logger) if policy.Cancellations != nil { policy.Cancellations.Add(float64(n)) } - lg.Debug("render connect retry cancelled: another replica group answered", + lg.Debug("render connect retry cancelled: another replica group of the cluster answered", zap.Strings("targets", f.request.Targets), zap.String("cluster", g.cluster), zap.Int("fetches", n)) @@ -271,14 +278,18 @@ func cancelCovered(fetches []renderFetch, policy *ConnectRetry, lg *zap.Logger) } } -// covered reports whether a replica group of cluster returned data for this -// fetch. +// covered reports whether a replica group of cluster gave a definitive answer +// for this fetch: data or the backend's own not-found. A not-found counts as +// much as data. The canary of the data-only rule (30s refusal of a one-store +// group) still waited the whole window for every glob held in another +// cluster, because the refused cluster's other groups could only answer +// not-found: 41 HTTP 500 and 67 499 in 30s on the refused frontend. func (f *renderFetch) covered(cluster string) bool { if cluster == "" { return false } for gi := range f.groups { - if f.groups[gi].cluster == cluster && f.groups[gi].data { + if f.groups[gi].cluster == cluster && f.groups[gi].answered { return true } } diff --git a/pkg/backend/connect_retry_test.go b/pkg/backend/connect_retry_test.go index a4164212..f9ba8973 100644 --- a/pkg/backend/connect_retry_test.go +++ b/pkg/backend/connect_retry_test.go @@ -217,8 +217,9 @@ func TestConnectRetryCancelledByOtherGroup(t *testing.T) { } } -// Only data from a replica group of the same cluster makes a retry redundant. -func TestConnectRetryNotCancelledWithoutReplicaData(t *testing.T) { +// Only an answer from a replica group of the same cluster makes a retry +// redundant. +func TestConnectRetryNotCancelledWithoutReplicaAnswer(t *testing.T) { const window = 150 * time.Millisecond for _, tc := range []struct { name string @@ -227,7 +228,8 @@ func TestConnectRetryNotCancelledWithoutReplicaData(t *testing.T) { ungroupedPeer bool }{ {name: "other_cluster_data", dc: "dc3", cluster: "carbon"}, - {name: "other_group_not_found", dc: "dc3", cluster: "cy", answer: types.ErrMetricsNotFound}, + {name: "other_cluster_not_found", dc: "dc3", cluster: "carbon", answer: types.ErrMetricsNotFound}, + {name: "other_group_server_error", dc: "dc3", cluster: "cy", answer: bnet.ErrHTTPCode(http.StatusServiceUnavailable)}, {name: "ungrouped", ungroupedPeer: true}, } { t.Run(tc.name, func(t *testing.T) { @@ -248,6 +250,95 @@ func TestConnectRetryNotCancelledWithoutReplicaData(t *testing.T) { } } +// The fe-112 canary: the refused store is the only member of one of its +// cluster's replica groups, the other groups of that cluster answer not-found, +// and the series lives in another cluster. Replicas of one cluster hold the +// same names, so their not-found already tells what the refused store would +// say: the fetch is given up at once instead of after the whole retry window. +// The refusal stays a plain connect error, so the required-fetch and replica-group rules decide the render as +// without retries: another cluster's data completes it, and with no data +// anywhere the fetch fails as a required fetch did before. +func TestConnectRetryCancelledByOtherGroupsNotFound(t *testing.T) { + for _, tc := range []struct { + name string + dataCluster bool // a group of another cluster holds the series + lhrRefused bool // a second group of the refused cluster is unreachable too + wantCancels float64 + }{ + {name: "series_in_other_cluster", dataCluster: true, wantCancels: 3}, + {name: "series_nowhere", wantCancels: 3}, + {name: "two_groups_refused_one_not_found", dataCluster: true, lhrRefused: true, wantCancels: 6}, + } { + t.Run(tc.name, func(t *testing.T) { + retries, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + refused, _ := retryStoreIn("dc1", "carbon", 1<<30, refusedDial, nil) + lhrRefusals := int32(0) + if tc.lhrRefused { + lhrRefusals = 1 << 30 + } + lhr, _ := retryStoreIn("dc3", "carbon", lhrRefusals, refusedDial, types.ErrMetricsNotFound) + fra, _ := retryStoreIn("dc2", "carbon", 0, nil, types.ErrMetricsNotFound) + backends := []Backend{refused, lhr, fra} + if tc.dataCluster { + cy, _ := retryStoreIn("dc2", "cy", 0, nil, nil) + backends = append(backends, cy) + } + start := time.Now() + fetchBoth(t, context.Background(), backends, "m", func(name string, metrics []types.Metric, errs []error) { + if (len(metrics) == 1) != tc.dataCluster { + t.Fatalf("%s: metrics %+v, want data %v", name, metrics, tc.dataCluster) + } + refusals, notFound := 0, 0 + for _, err := range errs { + var u unreachableError + switch { + case errors.As(err, &u): + t.Fatalf("%s: internal retry marker leaked: %v", name, errs) + case errors.Is(err, syscall.ECONNREFUSED): + refusals++ + case errors.Is(err, types.ErrMetricsNotFound): + notFound++ + } + } + wantRefusals := 1 + if tc.lhrRefused { + wantRefusals = 2 + } + if refusals != wantRefusals || notFound != 3-wantRefusals { + t.Fatalf("%s: errs %v, want %d refusals as plain errors and the not-founds", name, errs, wantRefusals) + } + }) + if elapsed := time.Since(start); elapsed > 200*time.Millisecond { + t.Fatalf("waited %v although the refused cluster's other groups answered not-found", elapsed) + } + if counterValue(retries) != 0 || counterValue(cancels) != tc.wantCancels { + t.Fatalf("retries %v cancels %v, want 0/%v", counterValue(retries), counterValue(cancels), tc.wantCancels) + } + }) + } +} + +// Every group of the cluster refuses, then one comes back and answers +// not-found: the others' retries stop there. +func TestConnectRetryAllGroupsRefusedThenNotFound(t *testing.T) { + retries, recoveries, cancels := fastConnectRetryCancels(t, 2*time.Second) + ams, _ := retryStoreIn("dc1", "carbon", 1<<30, refusedDial, nil) + lhr, _ := retryStoreIn("dc3", "carbon", 1<<30, refusedDial, nil) + fra, _ := retryStoreIn("dc2", "carbon", 3, refusedDial, types.ErrMetricsNotFound) + start := time.Now() + metrics, _, errs := Renders(context.Background(), []Backend{ams, lhr, fra}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("waited %v after a group answered not-found", elapsed) + } + if len(metrics) != 0 || len(errs) != 3 { + t.Fatalf("metrics %+v errs %v", metrics, errs) + } + if counterValue(retries) < 9 || counterValue(recoveries) != 1 || counterValue(cancels) != 2 { + t.Fatalf("retries %v recoveries %v cancels %v", counterValue(retries), counterValue(recoveries), counterValue(cancels)) + } +} + // When every group was unreachable and one recovers, the others' repeats are // not awaited, even if a just-restarted store accepts but answers slowly. func TestConnectRetryStopsWaitingOnceAGroupRecovers(t *testing.T) { diff --git a/pkg/cfg/common.go b/pkg/cfg/common.go index 76456bcc..a4316d7e 100644 --- a/pkg/cfg/common.go +++ b/pkg/cfg/common.go @@ -377,8 +377,8 @@ type Timeouts struct { // ConnectRetry is how long a render keeps retrying a backend that refused, // reset or timed out the connection before sending any response, as during // a store restart. The retry stops as soon as a replica group of the same - // cluster in another DC has returned data for the target. Zero disables - // the retry. + // cluster in another DC has answered for the target, with data or + // not-found. Zero disables the retry. ConnectRetry time.Duration `yaml:"connectRetry"` } From 1ed6de3e24c1339e799423ccb6c4a4ed78128af0 Mon Sep 17 00:00:00 2001 From: Anton Timofieiev Date: Fri, 13 Nov 2026 12:16:45 +0100 Subject: [PATCH 6/8] fix(zipper): treat a refused store as not-found when the rest of its cluster answered not-found A production canary returned HTTP 500 for renders such as sumSeries(x, *) while the only store of a one-store replica group refused connections (and again while it timed out at dial). The literal target "*" got not-found from 93 stores and a connect error from one. Since the required-fetch rule the fan-in returns not-found only when every error is a not-found, and the not-found retry cancellation kept the plain connect error on the fetch it gave up, so the whole render failed. A peer instance on the previous release returned 200 with the other series. Replica groups of one cluster hold the same names. When every other replica group of the failed store's cluster gave a definitive not-found (some member answered, none returned data), the store's connect failures (refused, reset or timed out before any response) now count as the cluster's not-found, with or without connect retries. The connect error still fails the fetch when the group is its cluster's only group, when another group of the cluster is unreachable too or failed (HTTP 5xx), or when one returned data (the render has that data anyway). Server errors, failures after the backend accepted the request, slow-replica cuts and ungrouped backends are unchanged, so the required-fetch rule still rejects a fetch whose data could exist only on the unreachable stores. Tests: the canary fetch of "*" (94 stores, 93 not-found, one single-store group refused or timing out at dial) is not-found, and sumSeries(a.b.*, *) returns 200 with the a.b.* series; every carbon group unreachable still returns 500. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/app/carbonapi/connect_retry_test.go | 118 ++++++++++++++++-- .../carbonapi/replica_group_render_test.go | 3 + pkg/backend/connect_retry.go | 10 +- pkg/backend/connect_retry_test.go | 104 ++++++++++++++- pkg/backend/render_fanin.go | 53 +++++++- 5 files changed, 264 insertions(+), 24 deletions(-) diff --git a/pkg/app/carbonapi/connect_retry_test.go b/pkg/app/carbonapi/connect_retry_test.go index 2f2a8fb8..f4fcc4c2 100644 --- a/pkg/app/carbonapi/connect_retry_test.go +++ b/pkg/app/carbonapi/connect_retry_test.go @@ -2,6 +2,8 @@ package carbonapi import ( "context" + "errors" + "fmt" "io" "net" "net/http" @@ -15,6 +17,7 @@ import ( "github.com/bookingcom/carbonapi/pkg/backend" "github.com/bookingcom/carbonapi/pkg/types" + "go.uber.org/zap" ) // restartingStore fails its first refusals renders, then serves like its fixture. @@ -106,9 +109,10 @@ func TestRenderRetriesRestartingStore(t *testing.T) { // The fe-112 canary scenario end to end: one cluster with three replica groups, // the one-store dc1 group refused, the dc3 and dc2 groups answering -// not-found. The render returns without the retry wait. With the series held -// by another cluster it is complete; with the series nowhere, the response is -// the one the same stores give with retries disabled (the required-fetch and replica-group rules unchanged). +// not-found. The render returns without the retry wait, and the response is +// the one the same stores give with retries disabled. With the series held by +// another cluster it is complete; with the series nowhere, the refusal counts +// as the cluster's not-found, so the render returns the remaining input. func TestRenderRefusedStoreOtherGroupsNotFound(t *testing.T) { refused := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} @@ -145,14 +149,110 @@ func TestRenderRefusedStoreOtherGroupsNotFound(t *testing.T) { if code != baseCode || body != baseBody { t.Fatalf("HTTP %d %s, without retries HTTP %d %s", code, body, baseCode, baseBody) } + w := httptest.NewRecorder() + w.Code = code + w.Body.WriteString(body) if tc.dataCluster { - w := httptest.NewRecorder() - w.Code = code - w.Body.WriteString(body) assertPartialRenderValue(t, w, 100) - } else if code != http.StatusInternalServerError { - // A required series no group has data for still fails (the required-fetch and replica-group rules). - t.Fatalf("HTTP %d, want 500 as without retries: %s", code, body) + } else { + assertPartialRenderValue(t, w, 1) // m.ok not found, m.ko alone + } + }) + } +} + +// namespaceStore answers the glob a.b.* with its series, if it has one, and +// every other target with not-found. +type namespaceStore struct { + *partialRenderBackend + series string + failure error +} + +func (s *namespaceStore) Render(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + if s.failure != nil { + return nil, s.failure + } + if s.series == "" || len(r.Targets) != 1 || r.Targets[0] != "a.b.*" { + return nil, types.ErrMetricsNotFound + } + m := types.Metric{Name: s.series, StartTime: r.From, StopTime: r.Until, StepTime: 60} + for ts := r.From; ts < r.Until; ts += 60 { + m.Values = append(m.Values, 42) + m.IsAbsent = append(m.IsAbsent, false) + } + return []types.Metric{m}, nil +} + +// The fe-112 canary 500s: dashboards pass the literal target "*" as an extra +// argument, as in sumSeries(a.b.*, *). With the one store of the dc1 carbon +// group refused or timing out at dial and the 93 other stores answering +// not-found for "*", the render returns 200 with the a.b.* series instead of +// failing on the unreachable store. If every carbon group is unreachable, the +// failed "*" fetch could hide data, and the render still fails (the required-fetch rule). +func TestRenderStarWithUnreachableSingleStoreGroup(t *testing.T) { + refused := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", + Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} + dialTimeout := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.ErrDeadlineExceeded}} + for _, tc := range []struct { + name string + failure error + allCarbonDown bool + }{ + {name: "refused", failure: refused}, + {name: "dial_timeout", failure: dialTimeout}, + {name: "every_carbon_group_unreachable", failure: refused, allCarbonDown: true}, + } { + t.Run(tc.name, func(t *testing.T) { + // connectRetry 10s as in production: no retry wait either way. + backend.SetConnectRetry(backend.ConnectRetry{Window: 10 * time.Second, InitialDelay: 10 * time.Millisecond, MaxDelay: 50 * time.Millisecond}) + t.Cleanup(func() { backend.SetConnectRetry(backend.ConnectRetry{}) }) + if tc.allCarbonDown { + backend.SetConnectRetry(backend.ConnectRetry{}) + } + store := func(name, cluster, dc string) *namespaceStore { + return &namespaceStore{partialRenderBackend: &partialRenderBackend{path: name, cluster: cluster, dc: dc}} + } + stores := []backend.BackendImpl{&namespaceStore{partialRenderBackend: &partialRenderBackend{path: "store-1", cluster: "carbon", dc: "dc1"}, failure: tc.failure}} + for _, dc := range []string{"dc3", "dc2"} { + for i := range 3 { + s := store(fmt.Sprintf("carbon-%s-%d", dc, i), "carbon", dc) + if tc.allCarbonDown { + s.failure = tc.failure + } + stores = append(stores, s) + } + } + for c := range 29 { + for _, dc := range []string{"dc1", "dc3", "dc2"} { + s := store(fmt.Sprintf("other-%d-%s", c, dc), fmt.Sprintf("other-%d", c), dc) + if c == 0 { + s.series = "a.b.c" // a.b.* lives in another cluster + } + stores = append(stores, s) + } + } + if len(stores) != 94 { + t.Fatalf("%d stores", len(stores)) + } + app := partialRenderAppOf(t, stores...) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(a.b.*, *)") + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("render took %v", elapsed) + } + if tc.allCarbonDown { + if w.Code != http.StatusInternalServerError { + t.Fatalf("HTTP %d, want 500 while every carbon copy is unreachable: %s", w.Code, w.Body.String()) + } + return + } + assertPartialRenderValue(t, w, 42) + _, err := Render(app.TopLevelDomainCache, nil, false, app.Backends, app.ZipperConfig.RenderReplicaMismatchConfig, + context.Background(), "*", 1794392400, 1794393000, app.ZipperMetrics, zap.NewNop()) + var notFound types.ErrNotFound + if !errors.As(err, ¬Found) { + t.Fatalf("fetch of * = %v, want not-found", err) } }) } diff --git a/pkg/app/carbonapi/replica_group_render_test.go b/pkg/app/carbonapi/replica_group_render_test.go index 8392593a..732c1017 100644 --- a/pkg/app/carbonapi/replica_group_render_test.go +++ b/pkg/app/carbonapi/replica_group_render_test.go @@ -37,6 +37,9 @@ func TestRenderReplicaGroupFailure(t *testing.T) { {name: "all_group_members_refused", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: refused, wantError: true}, {name: "data_replica_wins", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", wantData: true}, {name: "other_group_cannot_recover", failure: refused, cluster: "cy", peerCluster: "infra", peerDC: "dc1", peerError: types.ErrMetricsNotFound, wantError: true}, + // A bare dial error is not an http.Client connect failure (no *url.Error), + // so the cluster not-found rule does not apply; real refusals are in + // TestRenderStarWithUnreachableSingleStoreGroup. {name: "other_dc_cannot_recover", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc4", peerError: types.ErrMetricsNotFound, wantError: true}, {name: "ungrouped_unchanged", failure: refused, peerError: types.ErrMetricsNotFound, wantError: true}, } { diff --git a/pkg/backend/connect_retry.go b/pkg/backend/connect_retry.go index 462ce3aa..2e8c3945 100644 --- a/pkg/backend/connect_retry.go +++ b/pkg/backend/connect_retry.go @@ -244,11 +244,11 @@ func awaitRetries(fetches []renderFetch, flights []retryFlight) { // cancelCovered gives up the unreachable fetches of every replica group whose // target another group of the same cluster has answered for, with data or -// not-found. Their errors become final plain connect errors, so the required-fetch and replica-group rules -// decide the render exactly as without retries: another group's data still -// does not recover them, and a required fetch that no group has data for -// still fails. Ungrouped backends are never covered: they never recover each -// other's errors. +// not-found. Their errors become final plain connect errors and the fan-in +// decides the render exactly as without retries (renderGroupErrors): they +// count as the cluster's not-found when every other group of the cluster +// answered not-found, and otherwise stay errors for the required-fetch and replica-group rules. Ungrouped +// backends are never covered: they never recover each other's errors. func cancelCovered(fetches []renderFetch, policy *ConnectRetry, lg *zap.Logger) { for fi := range fetches { f := &fetches[fi] diff --git a/pkg/backend/connect_retry_test.go b/pkg/backend/connect_retry_test.go index f9ba8973..118ab811 100644 --- a/pkg/backend/connect_retry_test.go +++ b/pkg/backend/connect_retry_test.go @@ -3,6 +3,7 @@ package backend import ( "context" "errors" + "fmt" "io" "net" "net/http" @@ -254,10 +255,10 @@ func TestConnectRetryNotCancelledWithoutReplicaAnswer(t *testing.T) { // cluster's replica groups, the other groups of that cluster answer not-found, // and the series lives in another cluster. Replicas of one cluster hold the // same names, so their not-found already tells what the refused store would -// say: the fetch is given up at once instead of after the whole retry window. -// The refusal stays a plain connect error, so the required-fetch and replica-group rules decide the render as -// without retries: another cluster's data completes it, and with no data -// anywhere the fetch fails as a required fetch did before. +// say: the fetch is given up at once instead of after the whole retry window, +// and the refusal counts as the cluster's not-found (renderGroupErrors). With a +// second group of the cluster unreachable too, not every other group answered, +// so both refusals stay plain connect errors. func TestConnectRetryCancelledByOtherGroupsNotFound(t *testing.T) { for _, tc := range []struct { name string @@ -300,12 +301,12 @@ func TestConnectRetryCancelledByOtherGroupsNotFound(t *testing.T) { notFound++ } } - wantRefusals := 1 + wantRefusals := 0 if tc.lhrRefused { wantRefusals = 2 } if refusals != wantRefusals || notFound != 3-wantRefusals { - t.Fatalf("%s: errs %v, want %d refusals as plain errors and the not-founds", name, errs, wantRefusals) + t.Fatalf("%s: errs %v, want %d refusals as plain errors and the rest not-found", name, errs, wantRefusals) } }) if elapsed := time.Since(start); elapsed > 200*time.Millisecond { @@ -463,3 +464,94 @@ func TestConnectFailureClassification(t *testing.T) { } } } + +// The fe-112 canary fetch of the literal target "*": 94 stores, 93 answer +// not-found and the one store of the dc1 carbon group is refused or times out +// at dial. The carbon cluster's other groups answered not-found, so the failed +// store counts as not-found too and the fetch is the cluster's not-found, with +// or without connect retries. Only when no other group of its cluster gave a +// definitive not-found does the connect error stay and fail the fetch (the required-fetch rule). +func TestRenderUnreachableGroupClusterNotFound(t *testing.T) { + dialTimeout := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.ErrDeadlineExceeded}} + serverError := bnet.ErrHTTPCode(http.StatusServiceUnavailable) + for _, tc := range []struct { + name string + failure error + carbon map[string]error // answer of the other carbon groups; nil: not-found + data bool // a carbon group returns data + wantFails int // connect failures still returned as errors + }{ + {name: "refused", failure: refusedDial}, + {name: "dial_timeout", failure: dialTimeout}, + {name: "every_copy_unreachable", failure: refusedDial, carbon: map[string]error{"dc3": refusedDial, "dc2": refusedDial}, wantFails: 7}, + {name: "another_group_unreachable", failure: refusedDial, carbon: map[string]error{"dc3": refusedDial}, wantFails: 4}, + {name: "another_group_server_error", failure: refusedDial, carbon: map[string]error{"dc3": serverError}, wantFails: 1}, + {name: "another_group_has_data", failure: refusedDial, data: true, wantFails: 1}, + } { + for _, window := range []time.Duration{0, 400 * time.Millisecond} { + t.Run(fmt.Sprintf("%s/retry_%v", tc.name, window), func(t *testing.T) { + fastConnectRetryCancels(t, window) + if window == 0 { + connectRetry.Store(nil) + } + // carbon: the one-store dc1 group plus three stores each in dc3 and dc2. + failed, _ := retryStoreIn("dc1", "carbon", 1<<30, tc.failure, nil) + backends := []Backend{failed} + for _, dc := range []string{"dc3", "dc2"} { + for i := range 3 { + var answer error = types.ErrMetricsNotFound + if tc.data && dc == "dc2" && i == 0 { + answer = nil + } + refusals, failure := int32(0), error(nil) + if err := tc.carbon[dc]; errors.Is(err, syscall.ECONNREFUSED) { + refusals, failure = 1<<30, err + } else if err != nil { + answer = err + } + b, _ := retryStoreIn(dc, "carbon", refusals, failure, answer) + backends = append(backends, b) + } + } + // 87 more stores in 29 other clusters, one per DC, all not-found. + for c := range 29 { + for _, dc := range []string{"dc1", "dc3", "dc2"} { + b, _ := retryStoreIn(dc, fmt.Sprintf("other-%d", c), 0, nil, types.ErrMetricsNotFound) + backends = append(backends, b) + } + } + if len(backends) != 94 { + t.Fatalf("%d backends", len(backends)) + } + start := time.Now() + fetchBoth(t, context.Background(), backends, "*", func(name string, metrics []types.Metric, errs []error) { + if (len(metrics) != 0) != tc.data || len(errs) != 94-len(metrics) { + t.Fatalf("%s: %d metrics, %d errors", name, len(metrics), len(errs)) + } + fails, others := 0, 0 + for _, err := range errs { + var u unreachableError + var notFound types.ErrNotFound + switch { + case errors.As(err, &u): + t.Fatalf("%s: internal retry marker leaked: %v", name, err) + case connectFailure(err): + fails++ + case !errors.As(err, ¬Found): + others++ + } + } + if fails != tc.wantFails { + t.Fatalf("%s: %d connect failures returned, want %d: %v", name, fails, tc.wantFails, errs) + } + if (others != 0) != (tc.carbon["dc3"] == serverError) { + t.Fatalf("%s: %d other errors: %v", name, others, errs) + } + }) + if tc.wantFails == 0 && time.Since(start) > 200*time.Millisecond { + t.Fatalf("waited %v although every other carbon group answered not-found", time.Since(start)) + } + }) + } + } +} diff --git a/pkg/backend/render_fanin.go b/pkg/backend/render_fanin.go index cbe1106c..dd862556 100644 --- a/pkg/backend/render_fanin.go +++ b/pkg/backend/render_fanin.go @@ -98,17 +98,62 @@ func collectRenderGroups(f *renderFetch) { } } +// renderGroupErrors returns the errors that still decide the fetch: an +// answered group's errors become not-found, and so do the connect failures of +// a group whose cluster's other replica groups all answered not-found +// (clusterNotFound). Every other error is returned as the backend reported it. func renderGroupErrors(groups []renderGroup) []error { var errs []error - for _, g := range groups { + for gi := range groups { + g := &groups[gi] + notFound := !g.unrecovered() + unreachableNotFound := !notFound && clusterNotFound(groups, gi) for _, err := range g.errs { - if !g.unrecovered() { + switch { + case notFound, unreachableNotFound && connectFailure(err): err = types.ErrMetricsNotFound - } else if u, ok := err.(unreachableError); ok { - err = u.error + default: + if u, ok := err.(unreachableError); ok { + err = u.error + } } errs = append(errs, err) } } return errs } + +// clusterNotFound reports whether the cluster of group gi has at least one +// other replica group and every one of them gave a definitive not-found: some +// member answered, and none returned data. Replica groups of one cluster hold +// the same names, so that is what gi's refused or timed-out stores would have +// answered too; their connect failures then count as the cluster's not-found +// instead of failing the fetch. The fe-112 canary needed this: the literal +// target "*" (as in sumSeries(x, *)) got not-found from 93 stores and a refusal +// from the one store of the dc1 carbon group, and the required-fetch rule (not-found only +// when every error is a not-found) turned the whole render into HTTP 500. +// +// The connect failure stays an error, so the required-fetch rule still fails a fetch no store has +// data for, when gi is its cluster's only group, when any other group is +// unreachable too or failed (for example HTTP 5xx), or when another group +// returned data (the render then has that data anyway). Server errors and +// failures after the backend accepted the request are never converted, and +// ungrouped backends never recover each other. +func clusterNotFound(groups []renderGroup, gi int) bool { + cluster := groups[gi].cluster + if cluster == "" { + return false + } + others := 0 + for oi := range groups { + o := &groups[oi] + if oi == gi || o.cluster != cluster { + continue + } + if !o.answered || o.data { + return false + } + others++ + } + return others != 0 +} From 51745d4cd3510f5d5ef3aaf9848d1ad405e14047 Mon Sep 17 00:00:00 2001 From: Anton Timofieiev Date: Fri, 13 Nov 2026 11:38:48 +0100 Subject: [PATCH 7/8] fix(zipper): stop waiting on a slow replica once the others answered A go-carbon store that restarts accepts connections while it warms up but answers renders only after seconds. Renders query every DC's replica group and merge them, yet the fan-in waited for every backend up to afterStarted (15s in production). During one store restart about 300 renders per minute per instance took 10-15s although the other DCs had answered. The render fan-in now collects all replica groups concurrently. Once every other replica group of the same cluster has received all its replies and given a definitive answer (data or not-found), and at least one of them returned data, a group still silent after timeouts.slowReplicaGrace (default 1s, 0 keeps waiting) is no longer awaited: its outstanding fetches become a plain error, recovered as not-found when a member of that group answered, and the merged result of the other groups is returned. backend_slow_replica_cancelled_total counts the abandoned fetches. If every other group of the cluster said not-found, the slow group may be the only holder of the series, so it is awaited in full. This differs from the connect retry, which also gives up on a not-found cover: a refused store cannot answer until it is back, a slow one will deliver its copy. A group is also never cut while another group of its cluster is still pending or only refused or failed, data from another cluster never counts, and ungrouped backends are always awaited. The in-flight request is not cancelled because coalesced batches share one context with other targets. Message order stays by group, so mergeMetrics sees the same input order as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- config/carbonzipper.yaml | 6 + pkg/app/carbonapi/app.go | 4 + pkg/app/carbonapi/metrics.go | 8 + pkg/app/carbonapi/slow_replica_test.go | 69 +++++ pkg/backend/rpc.go | 2 +- pkg/backend/slow_replica.go | 236 ++++++++++++++++ pkg/backend/slow_replica_test.go | 376 +++++++++++++++++++++++++ pkg/backend/stable_render.go | 6 +- pkg/cfg/api_test.go | 18 +- pkg/cfg/common.go | 18 +- pkg/cfg/common_test.go | 27 +- 11 files changed, 742 insertions(+), 28 deletions(-) create mode 100644 pkg/app/carbonapi/slow_replica_test.go create mode 100644 pkg/backend/slow_replica.go create mode 100644 pkg/backend/slow_replica_test.go diff --git a/config/carbonzipper.yaml b/config/carbonzipper.yaml index 69fe23a0..d66c436c 100644 --- a/config/carbonzipper.yaml +++ b/config/carbonzipper.yaml @@ -26,6 +26,12 @@ timeouts: # group of the same cluster in another DC has answered (data or not-found). # 0 disables. connectRetry: "10s" + # How long a render still waits for a replica group once every other + # replica group of the same cluster answered (data or not-found) and at + # least one returned data (a store that accepts connections but answers + # slowly while it warms up). If all the others said not-found, the slow + # group may hold the only copy and is awaited. 0 waits up to afterStarted. + slowReplicaGrace: "1s" # Number of concurrent requests to any given backend - default is no limit. # If set, you likely want >= MaxIdleConnsPerHost diff --git a/pkg/app/carbonapi/app.go b/pkg/app/carbonapi/app.go index e7994f83..04e403e5 100644 --- a/pkg/app/carbonapi/app.go +++ b/pkg/app/carbonapi/app.go @@ -347,6 +347,10 @@ func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *Prometheu Recoveries: zms.BackendConnectRecoveries, Cancellations: zms.BackendConnectCancels, }) + backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{ + Grace: config.Timeouts.SlowReplicaGrace, + Cancelled: zms.BackendSlowReplicaCuts, + }) configBackendList := config.GetBackends() backends := make([]backend.Backend, 0, len(configBackendList)) diff --git a/pkg/app/carbonapi/metrics.go b/pkg/app/carbonapi/metrics.go index fda87089..4242893d 100644 --- a/pkg/app/carbonapi/metrics.go +++ b/pkg/app/carbonapi/metrics.go @@ -56,6 +56,7 @@ type ZipperPrometheusMetrics struct { BackendConnectRetries prometheus.Counter BackendConnectRecoveries prometheus.Counter BackendConnectCancels prometheus.Counter + BackendSlowReplicaCuts prometheus.Counter RenderOutDurationExp *prometheus.HistogramVec FindOutDuration *prometheus.HistogramVec @@ -349,6 +350,7 @@ func registerPrometheusMetrics(ms *PrometheusMetrics, zms *ZipperPrometheusMetri prometheus.MustRegister(zms.BackendConnectRetries) prometheus.MustRegister(zms.BackendConnectRecoveries) prometheus.MustRegister(zms.BackendConnectCancels) + prometheus.MustRegister(zms.BackendSlowReplicaCuts) prometheus.MustRegister(zms.RenderOutDurationExp) prometheus.MustRegister(zms.FindOutDuration) prometheus.MustRegister(zms.BackendEnqueuedRequests) @@ -487,5 +489,11 @@ func NewZipperPrometheusMetrics(config cfg.Zipper) *ZipperPrometheusMetrics { Help: "Render fetches of an unreachable backend given up without waiting for the retry window because another replica group of the same cluster answered (data or not-found)", }, ), + BackendSlowReplicaCuts: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "backend_slow_replica_cancelled_total", + Help: "Render fetches of a backend no longer awaited because every other replica group of the same cluster had answered, at least one with data, and this backend stayed silent for timeouts.slowReplicaGrace", + }, + ), } } diff --git a/pkg/app/carbonapi/slow_replica_test.go b/pkg/app/carbonapi/slow_replica_test.go new file mode 100644 index 00000000..e271fef1 --- /dev/null +++ b/pkg/app/carbonapi/slow_replica_test.go @@ -0,0 +1,69 @@ +package carbonapi + +import ( + "context" + "testing" + "time" + + "github.com/bookingcom/carbonapi/pkg/backend" + "github.com/bookingcom/carbonapi/pkg/types" +) + +// stallingStore accepts renders and answers only after stall, like a store +// warming up after a restart. +type stallingStore struct { + *partialRenderBackend + stall time.Duration + release chan struct{} +} + +func (s *stallingStore) Render(ctx context.Context, r types.RenderRequest) ([]types.Metric, error) { + select { + case <-s.release: + case <-time.After(s.stall): + } + return s.partialRenderBackend.Render(ctx, r) +} + +// One store of a three-DC cluster stalls for 10 s: the render answers 200 with +// the other DCs' data after the grace instead of waiting for the store. +func TestRenderDoesNotWaitForSlowReplica(t *testing.T) { + const grace = time.Second + backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{Grace: grace}) + t.Cleanup(func() { backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{}) }) + slow := &stallingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc2"}, + stall: 10 * time.Second, release: make(chan struct{})} + t.Cleanup(func() { close(slow.release) }) + app := partialRenderAppOf(t, slow, + &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc1"}, + &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc3"}, + &partialRenderBackend{path: "m.ko", cluster: "cy", dc: "dc1"}) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + if elapsed := time.Since(start); elapsed > 1500*time.Millisecond { + t.Fatalf("render took %v, want about the %v grace", elapsed, grace) + } + assertPartialRenderValue(t, w, 100) // 99 from a replica of m.ok + 1 +} + +// One store stalls and the other DCs of its cluster answer not-found: the +// slow store may hold the only copy, so the render waits for it and returns +// its data rather than cutting it after the grace. +func TestRenderWaitsForSlowPossibleOnlyHolder(t *testing.T) { + const stall = 300 * time.Millisecond + backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{Grace: 20 * time.Millisecond}) + t.Cleanup(func() { backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{}) }) + slow := &stallingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc2"}, + stall: stall, release: make(chan struct{})} + t.Cleanup(func() { close(slow.release) }) + app := partialRenderAppOf(t, slow, + &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: "cx", dc: "dc1"}, + &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: "cx", dc: "dc3"}, + &partialRenderBackend{path: "m.ko", cluster: "cy", dc: "dc1"}) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + if elapsed := time.Since(start); elapsed < stall-20*time.Millisecond { + t.Fatalf("render returned after %v without the possible only holder", elapsed) + } + assertPartialRenderValue(t, w, 100) // 99 from the slow store + 1 +} diff --git a/pkg/backend/rpc.go b/pkg/backend/rpc.go index 9b2d35ee..95275652 100644 --- a/pkg/backend/rpc.go +++ b/pkg/backend/rpc.go @@ -28,7 +28,7 @@ func Renders(ctx context.Context, backends []Backend, request types.RenderReques } fetches := []renderFetch{{request: request, groups: groups}} - collectRenderGroups(&fetches[0]) + collectRenderFetches(fetches, lg) retryUnreachable(ctx, fetches, lg) metrics, stats := types.MergeMetrics(fetches[0].messages, replicaMismatchConfig, lg) return metrics, stats, renderGroupErrors(groups) diff --git a/pkg/backend/slow_replica.go b/pkg/backend/slow_replica.go new file mode 100644 index 00000000..1b8231b1 --- /dev/null +++ b/pkg/backend/slow_replica.go @@ -0,0 +1,236 @@ +package backend + +import ( + "errors" + "sync/atomic" + "time" + + "github.com/bookingcom/carbonapi/pkg/types" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +// SlowReplicaGrace bounds how long a render waits for a replica group that is +// still silent after every other replica group of the same cluster gave a +// definitive answer for the target (data or not-found) and at least one of +// them returned data, as a store warming up after a restart does: it accepts +// connections but answers only at the backend timeout. +type SlowReplicaGrace struct { + // Grace is the extra wait granted to the slow group. Zero waits for every + // backend as before. + Grace time.Duration + // Cancelled counts backend fetches no longer awaited after the grace. + Cancelled prometheus.Counter +} + +var slowReplicaGrace atomic.Pointer[SlowReplicaGrace] + +// SetSlowReplicaGrace replaces the process-wide slow replica policy. +func SetSlowReplicaGrace(s SlowReplicaGrace) { slowReplicaGrace.Store(&s) } + +// errSlowReplica is the error of a fetch abandoned after the grace. Like any +// other failure, it stays an error only when no member of its own group +// answered; the render already has the other groups' data. +var errSlowReplica = errors.New("render not awaited: every other replica group answered, one with data, and this backend did not answer within the slow replica grace") + +// cuttable reports whether some cluster of this fetch has several replica +// groups, so that one of them can ever be left behind. +func (f *renderFetch) cuttable() bool { + seen := make(map[string]struct{}, len(f.groups)) + for i := range f.groups { + c := f.groups[i].cluster + if c == "" { + continue + } + if _, ok := seen[c]; ok { + return true + } + seen[c] = struct{}{} + } + return false +} + +// coveredByOthers reports whether the slow group g may be cut after the grace. +// That needs both: +// +// - every other replica group of g's cluster has received all its replies +// and gave a definitive answer: data or the backend's own not-found. A +// group that is still pending, or only refused or failed, says nothing +// about the target, so g is awaited. +// - at least one of those groups returned data. If every other group said +// not-found, g may be the only holder of the series (replicas can diverge, +// for example for names created while a DC was down), so g is awaited in +// full. This is where the slow grace differs from the connect retry +// (connect_retry.go), which also gives up on a not-found cover: a refused +// store cannot answer at all until it is back, while a slow one is +// reachable and will deliver its copy. +func (f *renderFetch) coveredByOthers(gi int, pending []int) bool { + g := &f.groups[gi] + data := false + for oi := range f.groups { + o := &f.groups[oi] + if oi == gi || o.cluster != g.cluster { + continue + } + if pending[oi] != 0 || !o.answered { + return false + } + data = data || o.data + } + return data +} + +type fanInReply struct { + fetch, group int + metrics []types.Metric + err error +} + +// collectRenderFetches waits for every backend reply of every fetch, except +// that a replica group still silent policy.Grace after all other groups of its +// cluster answered, at least one with data (coveredByOthers), is no longer +// awaited. Its outstanding fetches get +// errSlowReplica; their late replies are dropped. The in-flight requests are +// not cancelled: coalesced batches share one context across targets, and +// another target may still need the same backend. +func collectRenderFetches(fetches []renderFetch, lg *zap.Logger) { + policy := slowReplicaGrace.Load() + if policy == nil || policy.Grace <= 0 { + for i := range fetches { + collectRenderGroups(&fetches[i]) + } + return + } + var watched []int + for i := range fetches { + if fetches[i].cuttable() { + watched = append(watched, i) + } + } + if len(watched) != 0 { + collectWithGrace(fetches, watched, policy, lg) + } + // Fetches that cannot be cut keep the ordinary fan-in. Their replies are + // buffered, so collecting them afterwards adds no wait. + for i := range fetches { + if !fetches[i].cuttable() { + collectRenderGroups(&fetches[i]) + } + } +} + +func collectWithGrace(fetches []renderFetch, watched []int, policy *SlowReplicaGrace, lg *zap.Logger) { + pending := make(map[int][]int, len(watched)) // fetch -> group -> replies awaited + cutAt := make(map[int][]time.Time, len(watched)) // fetch -> group -> grace deadline + byGroup := make(map[int][][][]types.Metric, len(watched)) + outstanding := 0 + for _, fi := range watched { + f := &fetches[fi] + pending[fi] = make([]int, len(f.groups)) + cutAt[fi] = make([]time.Time, len(f.groups)) + byGroup[fi] = make([][][]types.Metric, len(f.groups)) + for gi := range f.groups { + pending[fi][gi] = f.groups[gi].n + outstanding += f.groups[gi].n + } + } + // Buffered for every reply, so forwarders of abandoned groups never block. + replies := make(chan fanInReply, outstanding) + for _, fi := range watched { + for gi := range fetches[fi].groups { + g := &fetches[fi].groups[gi] + go func(fi, gi, n int, results <-chan []types.Metric, errs <-chan error) { + for range n { + metrics, err := nextReply(results, errs) + replies <- fanInReply{fi, gi, metrics, err} + } + }(fi, gi, g.n, g.results, g.errors) + } + } + + var timer *time.Timer + var next time.Time + arm := func() { + next = time.Time{} + for _, fi := range watched { + for gi, at := range cutAt[fi] { + if !at.IsZero() && pending[fi][gi] != 0 && (next.IsZero() || at.Before(next)) { + next = at + } + } + } + if timer != nil { + timer.Stop() + timer = nil + } + if !next.IsZero() { + timer = time.NewTimer(time.Until(next)) + } + } + defer func() { + if timer != nil { + timer.Stop() + } + }() + + for outstanding > 0 { + var expired <-chan time.Time + if timer != nil { + expired = timer.C + } + select { + case r := <-replies: + if pending[r.fetch][r.group] == 0 { + continue // abandoned group + } + pending[r.fetch][r.group]-- + outstanding-- + f := &fetches[r.fetch] + byGroup[r.fetch][r.group], _ = f.groups[r.group].take(r.metrics, r.err, byGroup[r.fetch][r.group]) + // Deadlines only appear; a group's coverage never goes away. + changed := false + for gi := range f.groups { + if pending[r.fetch][gi] != 0 && cutAt[r.fetch][gi].IsZero() && f.groups[gi].cluster != "" && f.coveredByOthers(gi, pending[r.fetch]) { + cutAt[r.fetch][gi] = time.Now().Add(policy.Grace) + changed = true + } + } + if changed { + arm() + } + case <-expired: + now := time.Now() + for _, fi := range watched { + f := &fetches[fi] + for gi := range f.groups { + n := pending[fi][gi] + if n == 0 || cutAt[fi][gi].IsZero() || cutAt[fi][gi].After(now) { + continue + } + g := &f.groups[gi] + for range n { + g.errs = append(g.errs, errSlowReplica) + } + pending[fi][gi] = 0 + outstanding -= n + if policy.Cancelled != nil { + policy.Cancelled.Add(float64(n)) + } + lg.Debug("render stopped waiting for a slow replica group: every other group answered, one with data", + zap.Strings("targets", f.request.Targets), + zap.String("cluster", g.cluster), + zap.Int("fetches", n), + zap.Duration("grace", policy.Grace)) + } + } + timer = nil + arm() + } + } + // Keep the ordinary fan-in's message order: by group, then by arrival. + for _, fi := range watched { + for _, msgs := range byGroup[fi] { + fetches[fi].messages = append(fetches[fi].messages, msgs...) + } + } +} diff --git a/pkg/backend/slow_replica_test.go b/pkg/backend/slow_replica_test.go new file mode 100644 index 00000000..58b4059c --- /dev/null +++ b/pkg/backend/slow_replica_test.go @@ -0,0 +1,376 @@ +package backend + +import ( + "context" + "errors" + "syscall" + "testing" + "time" + + "github.com/bookingcom/carbonapi/pkg/cfg" + "github.com/bookingcom/carbonapi/pkg/types" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +func slowGrace(t *testing.T, grace time.Duration) prometheus.Counter { + t.Helper() + cuts := prometheus.NewCounter(prometheus.CounterOpts{Name: "test_slow_replica_cuts"}) + SetSlowReplicaGrace(SlowReplicaGrace{Grace: grace, Cancelled: cuts}) + t.Cleanup(func() { slowReplicaGrace.Store(nil) }) + return cuts +} + +// stallingStore accepts the render, as a warming-up store does, and answers +// with data after stall (or when the test ends). +func stallingStore(t *testing.T, dc, cluster string, stall time.Duration) Backend { + t.Helper() + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + b, _ := retryStoreFunc(dc, cluster, 0, nil, func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + select { + case <-release: + case <-time.After(stall): + } + return coalesceMetrics(r.Targets), nil + }) + return b +} + +// renderTimed runs the ordinary and the grouped fan-in and reports each one's +// replies and duration. +func renderTimed(t *testing.T, backends []Backend, mismatch cfg.RenderReplicaMismatchConfig, + check func(name string, elapsed time.Duration, replies []GroupedRenderReply)) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + request := types.NewRenderRequest([]string{"m"}, 1, 4) + start := time.Now() + reply := GroupedRenderReply{} + reply.Metrics, reply.Stats, reply.Errors = Renders(ctx, backends, request, mismatch, zap.NewNop()) + check("ordinary", time.Since(start), []GroupedRenderReply{reply}) + start = time.Now() + replies := RendersGrouped(ctx, [][]Backend{backends, backends}, + []types.RenderRequest{request, types.NewRenderRequest([]string{"m.2"}, 1, 4)}, mismatch, zap.NewNop()) + check("grouped", time.Since(start), replies) +} + +func assertFullData(t *testing.T, name string, reply GroupedRenderReply) { + t.Helper() + if len(reply.Metrics) != 1 || reply.Metrics[0].Values[1] != 42 || reply.Metrics[0].IsAbsent[1] { + t.Fatalf("%s: metrics %+v, want full data", name, reply.Metrics) + } +} + +// Two DCs answer at once and the third accepts and stalls for 10 s, as a +// restarting store does: the render returns the merged data after the grace, +// and the abandoned fetch is a plain error that no longer fails the render. +func TestSlowReplicaCutAfterGrace(t *testing.T) { + const grace = time.Second + for _, mode := range []cfg.ReplicaMatchMode{cfg.ReplicaMatchModeNormal, cfg.ReplicaMatchModeMajority} { + t.Run(string(mode), func(t *testing.T) { + cuts := slowGrace(t, grace) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fra := stallingStore(t, "dc2", "cy", 10*time.Second) + backends := []Backend{fra, ams, lhr} + renderTimed(t, backends, cfg.RenderReplicaMismatchConfig{RenderReplicaMatchMode: mode}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < grace-50*time.Millisecond || elapsed > 1500*time.Millisecond { + t.Fatalf("%s: took %v, want the %v grace", name, elapsed, grace) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 1 || !errors.Is(reply.Errors[0], errSlowReplica) { + t.Fatalf("%s: errs %v, want the abandoned fetch", name, reply.Errors) + } + // renderErrorsFanIn: fewer errors than backends never fails a render with data. + if len(reply.Errors) >= len(backends) { + t.Fatalf("%s: %d errors for %d backends would fail the render", name, len(reply.Errors), len(backends)) + } + } + }) + // One ordinary and two grouped fetches. + if got := counterValue(cuts); got != 3 { + t.Fatalf("cancelled %v slow fetches, want 3", got) + } + }) + } +} + +// A slow store whose group-mates answered not-found leaves its group answered: +// the abandoned fetch is recovered as not-found, like the group's other errors. +func TestSlowReplicaCutInAnsweredGroup(t *testing.T) { + cuts := slowGrace(t, 100*time.Millisecond) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fraPeer, _ := retryStoreIn("dc2", "cy", 0, nil, types.ErrMetricsNotFound) + fra := stallingStore(t, "dc2", "cy", 10*time.Second) + renderTimed(t, []Backend{fraPeer, fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed > 600*time.Millisecond { + t.Fatalf("%s: took %v", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + var notFound types.ErrNotFound + for _, err := range reply.Errors { + if !errors.As(err, ¬Found) { + t.Fatalf("%s: errs %v, want only not-found", name, reply.Errors) + } + } + } + }) + if counterValue(cuts) != 3 { + t.Fatalf("cancelled %v slow fetches, want 3", counterValue(cuts)) + } +} + +// A not-found counts as a definitive answer once another group has data: the +// slow group is cut after the grace with one group holding the data and the +// other answering not-found. +func TestSlowReplicaCutWithNotFoundGroup(t *testing.T) { + const grace = 100 * time.Millisecond + cuts := slowGrace(t, grace) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, types.ErrMetricsNotFound) + fra := stallingStore(t, "dc2", "cy", 10*time.Second) + renderTimed(t, []Backend{fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < grace-20*time.Millisecond || elapsed > grace+500*time.Millisecond { + t.Fatalf("%s: took %v, want the %v grace", name, elapsed, grace) + } + for _, reply := range replies { + assertFullData(t, name, reply) + var slow bool + for _, err := range reply.Errors { + slow = slow || errors.Is(err, errSlowReplica) + } + if len(reply.Errors) != 2 || !slow { + t.Fatalf("%s: errs %v, want the not-found and the abandoned fetch", name, reply.Errors) + } + } + }) + if counterValue(cuts) != 3 { + t.Fatalf("cancelled %v slow fetches, want 3", counterValue(cuts)) + } +} + +// A slow group is awaited in full unless every other group of its cluster gave +// a definitive answer and one of them returned data. If all the others said +// not-found, it may hold the only copy of the series. +func TestSlowReplicaAwaitedWithoutFullCoverage(t *testing.T) { + const stall = 300 * time.Millisecond + for _, tc := range []struct { + name string + others func() []Backend + }{ + {name: "only_copy", others: func() []Backend { + ams, _ := retryStoreIn("dc1", "cy", 0, nil, types.ErrMetricsNotFound) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, types.ErrMetricsNotFound) + return []Backend{ams, lhr} + }}, + {name: "other_group_server_error", others: func() []Backend { + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, errors.New("upstream HTTP 503")) + return []Backend{ams, lhr} + }}, + {name: "other_cluster_data", others: func() []Backend { + ams, _ := retryStoreIn("dc1", "carbon", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "carbon", 0, nil, nil) + return []Backend{ams, lhr} + }}, + {name: "ungrouped", others: func() []Backend { + a, _ := retryStoreIn("dc1", "", 0, nil, nil) + b, _ := retryStoreIn("dc3", "", 0, nil, nil) + return []Backend{a, b} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + cuts := slowGrace(t, 20*time.Millisecond) + cluster := "cy" + if tc.name == "ungrouped" { + cluster = "" + } + fra := stallingStore(t, "dc2", cluster, stall) + renderTimed(t, append([]Backend{fra}, tc.others()...), cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < stall-20*time.Millisecond { + t.Fatalf("%s: returned after %v without the slow group", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + for _, err := range reply.Errors { + if errors.Is(err, errSlowReplica) { + t.Fatalf("%s: slow group cut: %v", name, reply.Errors) + } + } + } + }) + if counterValue(cuts) != 0 { + t.Fatalf("cancelled %v slow fetches, want 0", counterValue(cuts)) + } + }) + } +} + +// Healthy groups are unaffected: all fast is unchanged, and a group that is +// slower but answers within the grace still heals the merged series. +func TestSlowReplicaWithinGraceStillMerged(t *testing.T) { + cuts := slowGrace(t, time.Second) + holey := func(dc string, delay time.Duration, value float64, absent bool) Backend { + b, _ := retryStoreFunc(dc, "cy", 0, nil, func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + time.Sleep(delay) + ms := coalesceMetrics(r.Targets) + for i := range ms { + ms[i].Values[2], ms[i].IsAbsent[2] = value, absent + } + return ms, nil + }) + return b + } + for _, tc := range []struct { + name string + delay time.Duration + max time.Duration + }{ + {name: "all_fast", max: 200 * time.Millisecond}, + {name: "slower_within_grace", delay: 150 * time.Millisecond, max: 600 * time.Millisecond}, + } { + t.Run(tc.name, func(t *testing.T) { + backends := []Backend{holey("dc1", 0, 0, true), holey("dc3", 0, 0, true), holey("dc2", tc.delay, 7, false)} + renderTimed(t, backends, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < tc.delay || elapsed > tc.max { + t.Fatalf("%s: took %v", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 0 || reply.Metrics[0].IsAbsent[2] || reply.Metrics[0].Values[2] != 7 { + t.Fatalf("%s: %+v, want fra's point healed in and no errors", name, reply) + } + } + }) + }) + } + if counterValue(cuts) != 0 { + t.Fatalf("cancelled %v fetches, want 0", counterValue(cuts)) + } +} + +// Grace 0, or no policy, waits for the slow store as before. +func TestSlowReplicaGraceZeroWaits(t *testing.T) { + const stall = 300 * time.Millisecond + for _, name := range []string{"zero", "unset"} { + t.Run(name, func(t *testing.T) { + var cuts prometheus.Counter + if name == "zero" { + cuts = slowGrace(t, 0) + } + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fra := stallingStore(t, "dc2", "cy", stall) + renderTimed(t, []Backend{fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < stall-20*time.Millisecond { + t.Fatalf("%s: returned after %v without the slow store", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 0 { + t.Fatalf("%s: errs %v", name, reply.Errors) + } + } + }) + if cuts != nil && counterValue(cuts) != 0 { + t.Fatalf("cancelled %v fetches with grace 0", counterValue(cuts)) + } + }) + } +} + +// The refused-store path (connect-retry cancellation) and the slow-store grace compose: a refusal is +// cancelled at once, a stall after the grace, and neither counts the other. +func TestSlowReplicaWithRefusedStore(t *testing.T) { + const grace = 200 * time.Millisecond + t.Run("refused_only", func(t *testing.T) { + _, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + cuts := slowGrace(t, grace) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fra, _ := retryStoreIn("dc2", "cy", 1<<30, refusedDial, nil) + renderTimed(t, []Backend{fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed > 150*time.Millisecond { + t.Fatalf("%s: took %v for a refused store", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + } + }) + if counterValue(cancels) != 3 || counterValue(cuts) != 0 { + t.Fatalf("connect cancels %v slow cuts %v, want 3/0", counterValue(cancels), counterValue(cuts)) + } + }) + t.Run("refused_and_stalled_in_one_group", func(t *testing.T) { + retries, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + cuts := slowGrace(t, grace) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + refused, _ := retryStoreIn("dc2", "cy", 1<<30, refusedDial, nil) + stalled := stallingStore(t, "dc2", "cy", 10*time.Second) + renderTimed(t, []Backend{refused, stalled, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < grace-20*time.Millisecond || elapsed > grace+500*time.Millisecond { + t.Fatalf("%s: took %v, want the %v grace", name, elapsed, grace) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 2 { + t.Fatalf("%s: errs %v, want the refusal and the abandoned fetch", name, reply.Errors) + } + var refusal, slow bool + for _, err := range reply.Errors { + var u unreachableError + if errors.As(err, &u) { + t.Fatalf("%s: internal retry marker leaked", name) + } + refusal = refusal || errors.Is(err, syscall.ECONNREFUSED) + slow = slow || errors.Is(err, errSlowReplica) + } + if !refusal || !slow { + t.Fatalf("%s: errs %v", name, reply.Errors) + } + } + }) + if counterValue(retries) != 0 || counterValue(cancels) != 3 || counterValue(cuts) != 3 { + t.Fatalf("retries %v connect cancels %v slow cuts %v, want 0/3/3", counterValue(retries), counterValue(cancels), counterValue(cuts)) + } + }) + // A refusing group gave no definitive answer, so the stalled group is not + // cut: two bad groups at once wait for the slow one, and then the refusal + // is cancelled. + t.Run("refused_group_and_stalled_group", func(t *testing.T) { + const stall = 300 * time.Millisecond + _, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + cuts := slowGrace(t, 20*time.Millisecond) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 1<<30, refusedDial, nil) + fra := stallingStore(t, "dc2", "cy", stall) + renderTimed(t, []Backend{fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < stall-20*time.Millisecond || elapsed > stall+500*time.Millisecond { + t.Fatalf("%s: took %v", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 1 || !errors.Is(reply.Errors[0], syscall.ECONNREFUSED) { + t.Fatalf("%s: errs %v, want the refusal only", name, reply.Errors) + } + } + }) + if counterValue(cancels) != 3 || counterValue(cuts) != 0 { + t.Fatalf("connect cancels %v slow cuts %v, want 3/0", counterValue(cancels), counterValue(cuts)) + } + }) +} diff --git a/pkg/backend/stable_render.go b/pkg/backend/stable_render.go index 1eef8ee3..005821c3 100644 --- a/pkg/backend/stable_render.go +++ b/pkg/backend/stable_render.go @@ -51,9 +51,9 @@ func RendersGrouped(ctx context.Context, choices [][]Backend, requests []types.R for _, g := range groups { g.backend.sendRenderGroup(ctx, g.calls) } - for i := range fetches { - collectRenderGroups(&fetches[i]) - } + // One fan-in covers every target, so the slow replica grace of each runs + // from when its other groups answered, not after the previous target. + collectRenderFetches(fetches, lg) // One backoff schedule covers every target, so a restarting store delays // the request once rather than once per target. retryUnreachable(ctx, fetches, lg) diff --git a/pkg/cfg/api_test.go b/pkg/cfg/api_test.go index ad0f8a64..5164e2d5 100644 --- a/pkg/cfg/api_test.go +++ b/pkg/cfg/api_test.go @@ -66,10 +66,11 @@ loggerConfig: MaxProcs: 16, Timeouts: Timeouts{ - Global: 10 * time.Minute, - AfterStarted: 10 * time.Minute, - Connect: 200 * time.Millisecond, - ConnectRetry: 10 * time.Second, + Global: 10 * time.Minute, + AfterStarted: 10 * time.Minute, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 1024, KeepAliveInterval: 30 * time.Second, @@ -168,10 +169,11 @@ loggerConfig: MaxProcs: 16, Timeouts: Timeouts{ - Global: 10 * time.Minute, - AfterStarted: 10 * time.Minute, - Connect: 200 * time.Millisecond, - ConnectRetry: 10 * time.Second, + Global: 10 * time.Minute, + AfterStarted: 10 * time.Minute, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 1024, KeepAliveInterval: 30 * time.Second, diff --git a/pkg/cfg/common.go b/pkg/cfg/common.go index a4316d7e..d42b3db8 100644 --- a/pkg/cfg/common.go +++ b/pkg/cfg/common.go @@ -47,10 +47,11 @@ func DefaultCommonConfig() Common { MaxProcs: 1, Timeouts: Timeouts{ - Global: 10000 * time.Millisecond, - AfterStarted: 2 * time.Second, - Connect: 200 * time.Millisecond, - ConnectRetry: 10 * time.Second, + Global: 10000 * time.Millisecond, + AfterStarted: 2 * time.Second, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 20, KeepAliveInterval: 30 * time.Second, @@ -380,6 +381,15 @@ type Timeouts struct { // cluster in another DC has answered for the target, with data or // not-found. Zero disables the retry. ConnectRetry time.Duration `yaml:"connectRetry"` + // SlowReplicaGrace is how long a render still waits for a replica group + // once every other replica group of the same cluster answered for the + // target (data or not-found) and at least one of them returned data, as + // when a store warming up after a restart accepts connections but answers + // slowly. The slow group's fetches are then dropped and the other groups' + // merged data returned. If every other group said not-found, the slow + // group may hold the only copy and is awaited. Zero waits for every + // backend up to afterStarted. + SlowReplicaGrace time.Duration `yaml:"slowReplicaGrace"` } type ProtocolBackend struct { diff --git a/pkg/cfg/common_test.go b/pkg/cfg/common_test.go index 6ec0ee0d..70690d43 100644 --- a/pkg/cfg/common_test.go +++ b/pkg/cfg/common_test.go @@ -83,10 +83,11 @@ monitoring: }, MaxProcs: 32, Timeouts: Timeouts{ - Global: 20 * time.Second, - AfterStarted: 15 * time.Second, - Connect: 200 * time.Millisecond, - ConnectRetry: 10 * time.Second, + Global: 20 * time.Second, + AfterStarted: 15 * time.Second, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, @@ -263,10 +264,11 @@ monitoring: }, MaxProcs: 32, Timeouts: Timeouts{ - Global: 20 * time.Second, - AfterStarted: 15 * time.Second, - Connect: 200 * time.Millisecond, - ConnectRetry: 10 * time.Second, + Global: 20 * time.Second, + AfterStarted: 15 * time.Second, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, @@ -510,10 +512,11 @@ monitoring: }, MaxProcs: 32, Timeouts: Timeouts{ - Global: 20 * time.Second, - AfterStarted: 15 * time.Second, - Connect: 200 * time.Millisecond, - ConnectRetry: 10 * time.Second, + Global: 20 * time.Second, + AfterStarted: 15 * time.Second, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, From f73acdaa5c1711cb0d90afeb1121a6672098517c Mon Sep 17 00:00:00 2001 From: Anton Timofieiev Date: Fri, 13 Nov 2026 15:35:54 +0100 Subject: [PATCH 8/8] fix(backend): prewarm store connections before a reload takes traffic A release reload (systemctl reload -> SIGUSR2) re-execs carbonapi through gracehttp: the new process starts with an empty backend connection pool and takes full traffic at once. Every render then dials, thousands of store connects start within a second under the 200 ms connect timeout, and the ones that miss it fail renders after the connect-retry window. On one production instance that gave ~270 render 500s at a release and 46 on a repeat reload. The old process keeps serving until the new one calls gracehttp.Serve, so dial ahead of that: SetupZipper now dials prewarmConnsPerBackend (default 64) connections to every HTTP backend, round by round with at most 256 in flight and a 2 s connect budget, bounded by prewarmTimeout (default 5 s). The transport's DialContext hands those out before dialing; unused ones close after 10 s. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/app/carbonapi/app.go | 100 +++++++++++++-- pkg/app/carbonapi/backend_conn_test.go | 90 +++++++++++++- pkg/backend/net/warm_dialer.go | 164 +++++++++++++++++++++++++ pkg/backend/net/warm_dialer_test.go | 159 ++++++++++++++++++++++++ pkg/cfg/common.go | 12 +- 5 files changed, 512 insertions(+), 13 deletions(-) create mode 100644 pkg/backend/net/warm_dialer.go create mode 100644 pkg/backend/net/warm_dialer_test.go diff --git a/pkg/app/carbonapi/app.go b/pkg/app/carbonapi/app.go index 04e403e5..a677d413 100644 --- a/pkg/app/carbonapi/app.go +++ b/pkg/app/carbonapi/app.go @@ -1,10 +1,12 @@ package carbonapi import ( + "context" "fmt" "log" "net" "net/http" + "net/url" "os" "runtime" "strconv" @@ -324,14 +326,30 @@ func (app *App) deferredAccessLogging(accessLogger *zap.Logger, r *http.Request, } } -func newBackendHTTPTransport(config cfg.Zipper) http.RoundTripper { +// prewarmDialTimeout is the connect budget of a prewarm dial. It is longer +// than the request-path connect timeout because nobody waits on it, and long +// enough to ride out one dropped SYN (1 s initial retransmission). +const prewarmDialTimeout = 2 * time.Second + +// prewarmConnTTL is how long a prewarmed connection may wait for its first +// request before it is closed instead of handed out. +const prewarmConnTTL = 10 * time.Second + +// prewarmParallel caps concurrent prewarm dials across all backends. +const prewarmParallel = 256 + +func newBackendHTTPTransport(config cfg.Zipper, warm *bnet.WarmDialer) http.RoundTripper { + dial := (&net.Dialer{ + Timeout: config.Timeouts.Connect, + KeepAlive: config.KeepAliveInterval, + }).DialContext + if warm != nil { + dial = warm.DialContext + } transport := &http.Transport{ MaxIdleConnsPerHost: config.MaxIdleConnsPerHost, IdleConnTimeout: config.IdleConnTimeout, - DialContext: (&net.Dialer{ - Timeout: config.Timeouts.Connect, - KeepAlive: config.KeepAliveInterval, - }).DialContext, + DialContext: dial, } if os.Getenv("CARBONAPI_SYNC_BACKEND_HTTP") == "1" { return bnet.NewSyncTransport(transport) @@ -339,8 +357,67 @@ func newBackendHTTPTransport(config cfg.Zipper) http.RoundTripper { return transport } +func newWarmDialer(config cfg.Zipper) *bnet.WarmDialer { + return bnet.NewWarmDialer((&net.Dialer{ + Timeout: config.Timeouts.Connect, + KeepAlive: config.KeepAliveInterval, + }).DialContext, prewarmConnTTL) +} + +// prewarmAddrs returns the unique host:port of every HTTP backend. +func prewarmAddrs(config cfg.Zipper) []string { + seen := make(map[string]struct{}) + var addrs []string + for _, b := range config.GetBackends() { + u, err := url.Parse(b.Http) + if err != nil || u.Scheme != "http" || u.Hostname() == "" { + continue + } + addr := u.Host + if u.Port() == "" { + addr = net.JoinHostPort(u.Hostname(), "80") + } + if _, dup := seen[addr]; dup { + continue + } + seen[addr] = struct{}{} + addrs = append(addrs, addr) + } + return addrs +} + +// prewarmBackends dials config.PrewarmConnsPerBackend connections to every +// HTTP backend before the process serves. It runs before gracehttp.Serve, so +// on a reload the old process still serves while this one warms up. +func prewarmBackends(config cfg.Zipper, warm *bnet.WarmDialer, lg *zap.Logger) { + if warm == nil || config.PrewarmConnsPerBackend <= 0 || config.PrewarmTimeout <= 0 { + return + } + addrs := prewarmAddrs(config) + ctx, cancel := context.WithTimeout(context.Background(), config.PrewarmTimeout) + defer cancel() + dial := (&net.Dialer{ + Timeout: max(config.Timeouts.Connect, prewarmDialTimeout), + KeepAlive: config.KeepAliveInterval, + }).DialContext + res := warm.Prewarm(ctx, dial, addrs, config.PrewarmConnsPerBackend, prewarmParallel) + lg.Info("backend prewarm done", + zap.Int("backends", len(addrs)), + zap.Int("conns_per_backend", config.PrewarmConnsPerBackend), + zap.Int("dialed", res.Dialed), + zap.Int("failed", res.Failed), + zap.Duration("elapsed", res.Elapsed), + ) +} + func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *PrometheusMetrics, lg *zap.Logger) ([]backend.Backend, error) { - client := &http.Client{Transport: newBackendHTTPTransport(config)} + bs, _, err := initBackends(config, zms, ms, lg) + return bs, err +} + +func initBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *PrometheusMetrics, lg *zap.Logger) ([]backend.Backend, *bnet.WarmDialer, error) { + warm := newWarmDialer(config) + client := &http.Client{Transport: newBackendHTTPTransport(config, warm)} backend.SetConnectRetry(backend.ConnectRetry{ Window: config.Timeouts.ConnectRetry, Retries: zms.BackendConnectRetries, @@ -356,7 +433,7 @@ func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *Prometheu backends := make([]backend.Backend, 0, len(configBackendList)) for _, host := range configBackendList { if host.Http == "" { - return nil, fmt.Errorf("backend without http address was provided: %+v", host) + return nil, nil, fmt.Errorf("backend without http address was provided: %+v", host) } dc, cluster, _ := config.InfoOfBackend(host.Http) var b backend.Backend @@ -384,12 +461,12 @@ func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *Prometheu be, err = bnet.New(bConf) } if err != nil { - return nil, errors.Wrapf(err, "could not create backend for host: %s", host) + return nil, nil, errors.Wrapf(err, "could not create backend for host: %s", host) } beDuration, err := ms.BackendDuration.CurryWith(prometheus.Labels{"dc": dc, "cluster": cluster}) if err != nil { - return nil, errors.Wrap(err, "could not curry backend duration metric") + return nil, nil, errors.Wrap(err, "could not curry backend duration metric") } b = backend.NewBackend(be, config.BackendQueueSize, @@ -403,7 +480,7 @@ func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *Prometheu backends = append(backends, b) } - return backends, nil + return backends, warm, nil } // Setup sets up the zipper for future lanuch. @@ -438,10 +515,11 @@ func SetupZipper(configFile string, BuildVersion string, ms *PrometheusMetrics, ) zms := NewZipperPrometheusMetrics(config) - bs, err := InitBackends(config, zms, ms, lg) + bs, warm, err := initBackends(config, zms, ms, lg) if err != nil { lg.Fatal("failed to init backends", zap.Error(err)) } + prewarmBackends(config, warm, lg) return config, bs, expirecache.New(0), tldcache.InitTLDPrefixes(lg, config.TLDCacheExtraPrefixes), zms } diff --git a/pkg/app/carbonapi/backend_conn_test.go b/pkg/app/carbonapi/backend_conn_test.go index 58c22e83..6bd94944 100644 --- a/pkg/app/carbonapi/backend_conn_test.go +++ b/pkg/app/carbonapi/backend_conn_test.go @@ -42,7 +42,7 @@ func TestBackendHTTPTransportSelection(t *testing.T) { config := cfg.DefaultZipperConfig() config.MaxIdleConnsPerHost = 23 config.IdleConnTimeout = 43 * time.Second - transport := newBackendHTTPTransport(config) + transport := newBackendHTTPTransport(config, nil) t.Cleanup((&http.Client{Transport: transport}).CloseIdleConnections) if tc.sync { if _, ok := transport.(*bnet.SyncTransport); !ok { @@ -126,3 +126,91 @@ func testBackendIdleConnTimeoutFromConfig(t *testing.T) { t.Fatalf("%d connections after idleConnTimeout, want 2 (idle connection not closed)", n) } } + +// A reloaded process must take traffic with connections it dialed before +// serving, not dial every store under full load. +func TestBackendPrewarmServesFirstRendersWithoutDialing(t *testing.T) { + for _, value := range []string{"", "1"} { + t.Run("sync="+value, func(t *testing.T) { + t.Setenv("CARBONAPI_SYNC_BACKEND_HTTP", value) + var dials atomic.Int64 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.NotFound(w, nil) + })) + server.Config.ConnState = func(_ net.Conn, s http.ConnState) { + if s == http.StateNew { + dials.Add(1) + } + } + server.Start() + defer server.Close() + + z := cfg.DefaultZipperConfig() + z.Backends = []string{server.URL, server.URL + "/"} // duplicates dial once + z.ProtocolBackends = nil + z.BackendsByCluster = nil + z.BackendsByDC = nil + z.PrewarmConnsPerBackend = 3 + ms := newPrometheusMetrics(cfg.DefaultAPIConfig()) + bs, warm, err := initBackends(z, NewZipperPrometheusMetrics(z), &ms, zap.NewNop()) + if err != nil { + t.Fatal(err) + } + prewarmBackends(z, warm, zap.NewNop()) + waitDials := func(want int64) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for dials.Load() != want && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if n := dials.Load(); n != want { + t.Fatalf("%d connections, want %d", n, want) + } + } + waitDials(3) + if warm.Idle() != 3 { + t.Fatalf("idle prewarmed = %d, want 3", warm.Idle()) + } + + // Three concurrent renders take the three prewarmed connections. + errs := make(chan error, 3) + for i := 0; i < 3; i++ { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := bs[0].Render(ctx, types.NewRenderRequest([]string{"a.b"}, 1, 2)) + errs <- err + }() + } + for i := 0; i < 3; i++ { + if err := <-errs; !errors.Is(err, types.ErrMetricsNotFound) { + t.Fatalf("render: %v", err) + } + } + if n := dials.Load(); n != 3 { + t.Fatalf("%d connections after renders on 3 prewarmed, want 3", n) + } + }) + } +} + +func TestBackendPrewarmAddrs(t *testing.T) { + z := cfg.DefaultZipperConfig() + z.Backends = []string{"http://10.0.0.1:8080", "http://10.0.0.1:8080/", "http://store-2", "https://x:1", "::bad"} + z.ProtocolBackends = nil + z.BackendsByCluster = nil + z.BackendsByDC = nil + got := prewarmAddrs(z) + want := []string{"10.0.0.1:8080", "store-2:80"} + if len(got) != len(want) { + t.Fatalf("prewarmAddrs = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("prewarmAddrs = %v, want %v", got, want) + } + } + if d := cfg.DefaultZipperConfig(); d.PrewarmConnsPerBackend <= 0 || d.PrewarmTimeout <= 0 { + t.Fatalf("prewarm is off by default: %d conns, %s", d.PrewarmConnsPerBackend, d.PrewarmTimeout) + } +} diff --git a/pkg/backend/net/warm_dialer.go b/pkg/backend/net/warm_dialer.go new file mode 100644 index 00000000..1a9841c3 --- /dev/null +++ b/pkg/backend/net/warm_dialer.go @@ -0,0 +1,164 @@ +package net + +import ( + "context" + "net" + "sync" + "time" +) + +// DialFunc dials a network connection, like net.Dialer.DialContext. +type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// WarmDialer hands out backend connections that were dialed before the +// process started serving. +// +// A release reload (SIGUSR2, gracehttp re-exec) starts the new process with +// an empty connection pool and moves full traffic onto it at once: every +// render dials, thousands of dials to the stores start within a second, and +// with the 200 ms connect timeout a slow SYN round turns into render 500s. +// The old process keeps serving until the new one calls gracehttp.Serve, so +// dialing ahead of that costs nothing and the new process takes over with a +// warm pool. +type WarmDialer struct { + dial DialFunc + ttl time.Duration + + mu sync.Mutex + warm map[string][]warmConn +} + +type warmConn struct { + net.Conn + dialed time.Time +} + +// NewWarmDialer returns a dialer that serves prewarmed connections first and +// falls back to dial. A prewarmed connection older than ttl is never handed +// out, so a backend's idle timeout cannot have closed it. +func NewWarmDialer(dial DialFunc, ttl time.Duration) *WarmDialer { + return &WarmDialer{dial: dial, ttl: ttl, warm: make(map[string][]warmConn)} +} + +// DialContext returns a prewarmed connection to addr if one is left, and +// dials otherwise. +func (d *WarmDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + if network == "tcp" { + if c := d.take(addr); c != nil { + return c, nil + } + } + return d.dial(ctx, network, addr) +} + +func (d *WarmDialer) take(addr string) net.Conn { + d.mu.Lock() + defer d.mu.Unlock() + conns := d.warm[addr] + for len(conns) > 0 { + c := conns[len(conns)-1] + conns[len(conns)-1] = warmConn{} + conns = conns[:len(conns)-1] + if time.Since(c.dialed) < d.ttl { + d.warm[addr] = conns + return c.Conn + } + _ = c.Close() + } + delete(d.warm, addr) + return nil +} + +// Idle reports how many prewarmed connections are waiting to be handed out. +func (d *WarmDialer) Idle() int { + d.mu.Lock() + defer d.mu.Unlock() + n := 0 + for _, conns := range d.warm { + n += len(conns) + } + return n +} + +// PrewarmResult summarises one Prewarm call. +type PrewarmResult struct { + Dialed, Failed int + Elapsed time.Duration +} + +// Prewarm dials perAddr connections to every address with dial, at most +// parallel at a time in total, and keeps them for DialContext. Dials go round +// by round over the addresses, so no backend sees a burst of more than a few +// concurrent connects. It returns once every dial finished or ctx is done; +// connections still being dialed then are closed. Connections nobody took +// are closed after the dialer's ttl. +func (d *WarmDialer) Prewarm(ctx context.Context, dial DialFunc, addrs []string, perAddr, parallel int) PrewarmResult { + start := time.Now() + var res PrewarmResult + if perAddr <= 0 || len(addrs) == 0 { + return res + } + parallel = max(parallel, 1) + + var ( + mu sync.Mutex + wg sync.WaitGroup + sem = make(chan struct{}, parallel) + ) +dialing: + for round := 0; round < perAddr; round++ { + for _, addr := range addrs { + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break dialing + } + wg.Add(1) + go func(addr string) { + defer wg.Done() + defer func() { <-sem }() + c, err := dial(ctx, "tcp", addr) + mu.Lock() + defer mu.Unlock() + if err != nil { + res.Failed++ + return + } + if ctx.Err() != nil { + _ = c.Close() + res.Failed++ + return + } + res.Dialed++ + d.mu.Lock() + d.warm[addr] = append(d.warm[addr], warmConn{Conn: c, dialed: time.Now()}) + d.mu.Unlock() + }(addr) + } + } + wg.Wait() + time.AfterFunc(d.ttl, d.expire) + res.Elapsed = time.Since(start) + return res +} + +// expire closes prewarmed connections that outlived the ttl. +func (d *WarmDialer) expire() { + d.mu.Lock() + defer d.mu.Unlock() + for addr, conns := range d.warm { + keep := conns[:0] + for _, c := range conns { + if time.Since(c.dialed) < d.ttl { + keep = append(keep, c) + continue + } + _ = c.Close() + } + if len(keep) == 0 { + delete(d.warm, addr) + continue + } + d.warm[addr] = keep + } +} diff --git a/pkg/backend/net/warm_dialer_test.go b/pkg/backend/net/warm_dialer_test.go new file mode 100644 index 00000000..3a65c2dd --- /dev/null +++ b/pkg/backend/net/warm_dialer_test.go @@ -0,0 +1,159 @@ +package net + +import ( + "context" + "errors" + "io" + "net" + "sync/atomic" + "testing" + "time" +) + +// countingListener accepts and holds connections, counting them. +func countingListener(t *testing.T) (string, *atomic.Int64, chan net.Conn) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var n atomic.Int64 + conns := make(chan net.Conn, 128) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + n.Add(1) + conns <- c + } + }() + t.Cleanup(func() { + _ = ln.Close() + close(conns) + for c := range conns { + _ = c.Close() + } + }) + return ln.Addr().String(), &n, conns +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestWarmDialerHandsOutPrewarmedConnsThenDials(t *testing.T) { + addr, accepted, _ := countingListener(t) + base := (&net.Dialer{Timeout: time.Second}).DialContext + d := NewWarmDialer(base, time.Minute) + + res := d.Prewarm(context.Background(), base, []string{addr}, 3, 2) + if res.Dialed != 3 || res.Failed != 0 { + t.Fatalf("prewarm = %+v, want 3 dialed, 0 failed", res) + } + waitFor(t, "3 accepted", func() bool { return accepted.Load() == 3 }) + if d.Idle() != 3 { + t.Fatalf("idle = %d, want 3", d.Idle()) + } + + for i := 0; i < 3; i++ { + c, err := d.DialContext(context.Background(), "tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + } + if got := accepted.Load(); got != 3 { + t.Fatalf("prewarmed conns were not reused: %d accepts after 3 dials", got) + } + c, err := d.DialContext(context.Background(), "tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + waitFor(t, "fallback dial", func() bool { return accepted.Load() == 4 }) + if d.Idle() != 0 { + t.Fatalf("idle = %d, want 0", d.Idle()) + } +} + +func TestWarmDialerExpiresUnusedConns(t *testing.T) { + addr, accepted, conns := countingListener(t) + base := (&net.Dialer{Timeout: time.Second}).DialContext + d := NewWarmDialer(base, 50*time.Millisecond) + + if res := d.Prewarm(context.Background(), base, []string{addr}, 1, 1); res.Dialed != 1 { + t.Fatalf("prewarm = %+v, want 1 dialed", res) + } + server := <-conns + // The expiry closes the client side; the server then reads EOF. + if err := server.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + if _, err := server.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Fatalf("server read = %v, want EOF after expiry", err) + } + if d.Idle() != 0 { + t.Fatalf("idle = %d after expiry, want 0", d.Idle()) + } + // An expired dialer dials afresh. + c, err := d.DialContext(context.Background(), "tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + waitFor(t, "fresh dial", func() bool { return accepted.Load() == 2 }) +} + +func TestWarmDialerSkipsStaleConn(t *testing.T) { + addr, accepted, _ := countingListener(t) + base := (&net.Dialer{Timeout: time.Second}).DialContext + d := NewWarmDialer(base, time.Hour) + if res := d.Prewarm(context.Background(), base, []string{addr}, 1, 1); res.Dialed != 1 { + t.Fatalf("prewarm = %+v", res) + } + d.ttl = 0 // the prewarmed conn is now older than the ttl + c, err := d.DialContext(context.Background(), "tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + waitFor(t, "fresh dial instead of stale conn", func() bool { return accepted.Load() == 2 }) +} + +func TestWarmDialerPrewarmCountsFailuresAndHonoursContext(t *testing.T) { + refused := func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("connection refused") + } + d := NewWarmDialer(refused, time.Minute) + if res := d.Prewarm(context.Background(), refused, []string{"a:1", "b:1"}, 2, 4); res.Dialed != 0 || res.Failed != 4 { + t.Fatalf("prewarm = %+v, want 0 dialed, 4 failed", res) + } + + block := func(ctx context.Context, _, _ string) (net.Conn, error) { + <-ctx.Done() + return nil, ctx.Err() + } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + start := time.Now() + res := d.Prewarm(ctx, block, []string{"a:1", "b:1", "c:1"}, 100, 2) + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("prewarm ignored its context: took %v", elapsed) + } + if res.Dialed != 0 || d.Idle() != 0 { + t.Fatalf("prewarm = %+v, idle %d; want nothing kept", res, d.Idle()) + } + + if res := d.Prewarm(context.Background(), refused, []string{"a:1"}, 0, 4); res != (PrewarmResult{}) { + t.Fatalf("disabled prewarm = %+v, want zero", res) + } +} diff --git a/pkg/cfg/common.go b/pkg/cfg/common.go index d42b3db8..30b225d4 100644 --- a/pkg/cfg/common.go +++ b/pkg/cfg/common.go @@ -57,6 +57,8 @@ func DefaultCommonConfig() Common { KeepAliveInterval: 30 * time.Second, MaxIdleConnsPerHost: 100, IdleConnTimeout: 90 * time.Second, + PrewarmConnsPerBackend: 64, + PrewarmTimeout: 5 * time.Second, GrpcInitialWindowSize: 4 * 1024 * 1024, GrpcInitialConnWindowSize: 4 * 1024 * 1024, @@ -190,7 +192,15 @@ type Common struct { MaxIdleConnsPerHost int `yaml:"maxIdleConnsPerHost"` // IdleConnTimeout closes a pooled backend connection after it has been // idle this long; zero keeps it until the backend closes it. - IdleConnTimeout time.Duration `yaml:"idleConnTimeout"` + IdleConnTimeout time.Duration `yaml:"idleConnTimeout"` + // PrewarmConnsPerBackend is how many connections a starting process + // dials to each HTTP backend before it serves. On a release reload + // (SIGUSR2) the old process keeps serving meanwhile, so the new one takes + // over with a warm pool instead of dialing every store at full load. + // Zero or negative disables it. + PrewarmConnsPerBackend int `yaml:"prewarmConnsPerBackend"` + // PrewarmTimeout bounds the prewarm; the process serves after it either way. + PrewarmTimeout time.Duration `yaml:"prewarmTimeout"` GrpcInitialWindowSize int `yaml:"grpcInitialWindowSize"` GrpcInitialConnWindowSize int `yaml:"grpcInitialConnWindowSize"`