diff --git a/config/carbonzipper.yaml b/config/carbonzipper.yaml index e86362d3..d66c436c 100644 --- a/config/carbonzipper.yaml +++ b/config/carbonzipper.yaml @@ -21,6 +21,17 @@ timeouts: afterStarted: "200s" # Timeout to connect to the server connect: "200ms" + # How long a render keeps retrying a backend that refused, reset or timed + # out the connection before responding (a store restart), while no replica + # group of the same cluster in another DC has answered (data or not-found). + # 0 disables. + connectRetry: "10s" + # How long a render still waits for a replica group once every other + # replica group of the same cluster answered (data or not-found) and at + # least one returned data (a store that accepts connections but answers + # slowly while it warms up). If all the others said not-found, the slow + # group may hold the only copy and is awaited. 0 waits up to afterStarted. + slowReplicaGrace: "1s" # Number of concurrent requests to any given backend - default is no limit. # If set, you likely want >= MaxIdleConnsPerHost diff --git a/pkg/app/carbonapi/app.go b/pkg/app/carbonapi/app.go index 16170607..a677d413 100644 --- a/pkg/app/carbonapi/app.go +++ b/pkg/app/carbonapi/app.go @@ -1,10 +1,12 @@ package carbonapi import ( + "context" "fmt" "log" "net" "net/http" + "net/url" "os" "runtime" "strconv" @@ -324,14 +326,30 @@ func (app *App) deferredAccessLogging(accessLogger *zap.Logger, r *http.Request, } } -func newBackendHTTPTransport(config cfg.Zipper) http.RoundTripper { +// prewarmDialTimeout is the connect budget of a prewarm dial. It is longer +// than the request-path connect timeout because nobody waits on it, and long +// enough to ride out one dropped SYN (1 s initial retransmission). +const prewarmDialTimeout = 2 * time.Second + +// prewarmConnTTL is how long a prewarmed connection may wait for its first +// request before it is closed instead of handed out. +const prewarmConnTTL = 10 * time.Second + +// prewarmParallel caps concurrent prewarm dials across all backends. +const prewarmParallel = 256 + +func newBackendHTTPTransport(config cfg.Zipper, warm *bnet.WarmDialer) http.RoundTripper { + dial := (&net.Dialer{ + Timeout: config.Timeouts.Connect, + KeepAlive: config.KeepAliveInterval, + }).DialContext + if warm != nil { + dial = warm.DialContext + } transport := &http.Transport{ MaxIdleConnsPerHost: config.MaxIdleConnsPerHost, IdleConnTimeout: config.IdleConnTimeout, - DialContext: (&net.Dialer{ - Timeout: config.Timeouts.Connect, - KeepAlive: config.KeepAliveInterval, - }).DialContext, + DialContext: dial, } if os.Getenv("CARBONAPI_SYNC_BACKEND_HTTP") == "1" { return bnet.NewSyncTransport(transport) @@ -339,14 +357,83 @@ func newBackendHTTPTransport(config cfg.Zipper) http.RoundTripper { return transport } +func newWarmDialer(config cfg.Zipper) *bnet.WarmDialer { + return bnet.NewWarmDialer((&net.Dialer{ + Timeout: config.Timeouts.Connect, + KeepAlive: config.KeepAliveInterval, + }).DialContext, prewarmConnTTL) +} + +// prewarmAddrs returns the unique host:port of every HTTP backend. +func prewarmAddrs(config cfg.Zipper) []string { + seen := make(map[string]struct{}) + var addrs []string + for _, b := range config.GetBackends() { + u, err := url.Parse(b.Http) + if err != nil || u.Scheme != "http" || u.Hostname() == "" { + continue + } + addr := u.Host + if u.Port() == "" { + addr = net.JoinHostPort(u.Hostname(), "80") + } + if _, dup := seen[addr]; dup { + continue + } + seen[addr] = struct{}{} + addrs = append(addrs, addr) + } + return addrs +} + +// prewarmBackends dials config.PrewarmConnsPerBackend connections to every +// HTTP backend before the process serves. It runs before gracehttp.Serve, so +// on a reload the old process still serves while this one warms up. +func prewarmBackends(config cfg.Zipper, warm *bnet.WarmDialer, lg *zap.Logger) { + if warm == nil || config.PrewarmConnsPerBackend <= 0 || config.PrewarmTimeout <= 0 { + return + } + addrs := prewarmAddrs(config) + ctx, cancel := context.WithTimeout(context.Background(), config.PrewarmTimeout) + defer cancel() + dial := (&net.Dialer{ + Timeout: max(config.Timeouts.Connect, prewarmDialTimeout), + KeepAlive: config.KeepAliveInterval, + }).DialContext + res := warm.Prewarm(ctx, dial, addrs, config.PrewarmConnsPerBackend, prewarmParallel) + lg.Info("backend prewarm done", + zap.Int("backends", len(addrs)), + zap.Int("conns_per_backend", config.PrewarmConnsPerBackend), + zap.Int("dialed", res.Dialed), + zap.Int("failed", res.Failed), + zap.Duration("elapsed", res.Elapsed), + ) +} + func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *PrometheusMetrics, lg *zap.Logger) ([]backend.Backend, error) { - client := &http.Client{Transport: newBackendHTTPTransport(config)} + bs, _, err := initBackends(config, zms, ms, lg) + return bs, err +} + +func initBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *PrometheusMetrics, lg *zap.Logger) ([]backend.Backend, *bnet.WarmDialer, error) { + warm := newWarmDialer(config) + client := &http.Client{Transport: newBackendHTTPTransport(config, warm)} + backend.SetConnectRetry(backend.ConnectRetry{ + Window: config.Timeouts.ConnectRetry, + Retries: zms.BackendConnectRetries, + Recoveries: zms.BackendConnectRecoveries, + Cancellations: zms.BackendConnectCancels, + }) + backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{ + Grace: config.Timeouts.SlowReplicaGrace, + Cancelled: zms.BackendSlowReplicaCuts, + }) configBackendList := config.GetBackends() backends := make([]backend.Backend, 0, len(configBackendList)) for _, host := range configBackendList { if host.Http == "" { - return nil, fmt.Errorf("backend without http address was provided: %+v", host) + return nil, nil, fmt.Errorf("backend without http address was provided: %+v", host) } dc, cluster, _ := config.InfoOfBackend(host.Http) var b backend.Backend @@ -374,12 +461,12 @@ func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *Prometheu be, err = bnet.New(bConf) } if err != nil { - return nil, errors.Wrapf(err, "could not create backend for host: %s", host) + return nil, nil, errors.Wrapf(err, "could not create backend for host: %s", host) } beDuration, err := ms.BackendDuration.CurryWith(prometheus.Labels{"dc": dc, "cluster": cluster}) if err != nil { - return nil, errors.Wrap(err, "could not curry backend duration metric") + return nil, nil, errors.Wrap(err, "could not curry backend duration metric") } b = backend.NewBackend(be, config.BackendQueueSize, @@ -393,7 +480,7 @@ func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *Prometheu backends = append(backends, b) } - return backends, nil + return backends, warm, nil } // Setup sets up the zipper for future lanuch. @@ -428,10 +515,11 @@ func SetupZipper(configFile string, BuildVersion string, ms *PrometheusMetrics, ) zms := NewZipperPrometheusMetrics(config) - bs, err := InitBackends(config, zms, ms, lg) + bs, warm, err := initBackends(config, zms, ms, lg) if err != nil { lg.Fatal("failed to init backends", zap.Error(err)) } + prewarmBackends(config, warm, lg) return config, bs, expirecache.New(0), tldcache.InitTLDPrefixes(lg, config.TLDCacheExtraPrefixes), zms } diff --git a/pkg/app/carbonapi/backend_conn_test.go b/pkg/app/carbonapi/backend_conn_test.go index 58c22e83..6bd94944 100644 --- a/pkg/app/carbonapi/backend_conn_test.go +++ b/pkg/app/carbonapi/backend_conn_test.go @@ -42,7 +42,7 @@ func TestBackendHTTPTransportSelection(t *testing.T) { config := cfg.DefaultZipperConfig() config.MaxIdleConnsPerHost = 23 config.IdleConnTimeout = 43 * time.Second - transport := newBackendHTTPTransport(config) + transport := newBackendHTTPTransport(config, nil) t.Cleanup((&http.Client{Transport: transport}).CloseIdleConnections) if tc.sync { if _, ok := transport.(*bnet.SyncTransport); !ok { @@ -126,3 +126,91 @@ func testBackendIdleConnTimeoutFromConfig(t *testing.T) { t.Fatalf("%d connections after idleConnTimeout, want 2 (idle connection not closed)", n) } } + +// A reloaded process must take traffic with connections it dialed before +// serving, not dial every store under full load. +func TestBackendPrewarmServesFirstRendersWithoutDialing(t *testing.T) { + for _, value := range []string{"", "1"} { + t.Run("sync="+value, func(t *testing.T) { + t.Setenv("CARBONAPI_SYNC_BACKEND_HTTP", value) + var dials atomic.Int64 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.NotFound(w, nil) + })) + server.Config.ConnState = func(_ net.Conn, s http.ConnState) { + if s == http.StateNew { + dials.Add(1) + } + } + server.Start() + defer server.Close() + + z := cfg.DefaultZipperConfig() + z.Backends = []string{server.URL, server.URL + "/"} // duplicates dial once + z.ProtocolBackends = nil + z.BackendsByCluster = nil + z.BackendsByDC = nil + z.PrewarmConnsPerBackend = 3 + ms := newPrometheusMetrics(cfg.DefaultAPIConfig()) + bs, warm, err := initBackends(z, NewZipperPrometheusMetrics(z), &ms, zap.NewNop()) + if err != nil { + t.Fatal(err) + } + prewarmBackends(z, warm, zap.NewNop()) + waitDials := func(want int64) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for dials.Load() != want && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if n := dials.Load(); n != want { + t.Fatalf("%d connections, want %d", n, want) + } + } + waitDials(3) + if warm.Idle() != 3 { + t.Fatalf("idle prewarmed = %d, want 3", warm.Idle()) + } + + // Three concurrent renders take the three prewarmed connections. + errs := make(chan error, 3) + for i := 0; i < 3; i++ { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := bs[0].Render(ctx, types.NewRenderRequest([]string{"a.b"}, 1, 2)) + errs <- err + }() + } + for i := 0; i < 3; i++ { + if err := <-errs; !errors.Is(err, types.ErrMetricsNotFound) { + t.Fatalf("render: %v", err) + } + } + if n := dials.Load(); n != 3 { + t.Fatalf("%d connections after renders on 3 prewarmed, want 3", n) + } + }) + } +} + +func TestBackendPrewarmAddrs(t *testing.T) { + z := cfg.DefaultZipperConfig() + z.Backends = []string{"http://10.0.0.1:8080", "http://10.0.0.1:8080/", "http://store-2", "https://x:1", "::bad"} + z.ProtocolBackends = nil + z.BackendsByCluster = nil + z.BackendsByDC = nil + got := prewarmAddrs(z) + want := []string{"10.0.0.1:8080", "store-2:80"} + if len(got) != len(want) { + t.Fatalf("prewarmAddrs = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("prewarmAddrs = %v, want %v", got, want) + } + } + if d := cfg.DefaultZipperConfig(); d.PrewarmConnsPerBackend <= 0 || d.PrewarmTimeout <= 0 { + t.Fatalf("prewarm is off by default: %d conns, %s", d.PrewarmConnsPerBackend, d.PrewarmTimeout) + } +} diff --git a/pkg/app/carbonapi/connect_retry_test.go b/pkg/app/carbonapi/connect_retry_test.go new file mode 100644 index 00000000..f4fcc4c2 --- /dev/null +++ b/pkg/app/carbonapi/connect_retry_test.go @@ -0,0 +1,259 @@ +package carbonapi + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/bookingcom/carbonapi/pkg/backend" + "github.com/bookingcom/carbonapi/pkg/types" + "go.uber.org/zap" +) + +// restartingStore fails its first refusals renders, then serves like its fixture. +type restartingStore struct { + *partialRenderBackend + refusals int32 + failure error + calls atomic.Int32 +} + +func (s *restartingStore) Render(ctx context.Context, r types.RenderRequest) ([]types.Metric, error) { + if s.calls.Add(1) <= s.refusals { + return nil, s.failure + } + return s.partialRenderBackend.Render(ctx, r) +} + +func TestRenderRetriesRestartingStore(t *testing.T) { + refused := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", + Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} + afterAccept := &url.Error{Op: "Get", URL: "http://store/render/", Err: io.ErrUnexpectedEOF} + const forever = 1 << 30 + for _, tc := range []struct { + name string + window time.Duration + refusals int32 + failure error + peer bool // another member of the refusing store's replica group answers + otherDC bool // a replica group in another DC holds the same series + otherDown int32 + wantCalls int32 + wantError bool + maxTime time.Duration + }{ + {name: "refused_then_up", refusals: 3, failure: refused, wantCalls: 4}, + {name: "refused_past_window", refusals: forever, failure: refused, wantError: true}, + {name: "refused_peer_answers", refusals: forever, failure: refused, peer: true, wantCalls: 1}, + {name: "failure_after_accept", refusals: forever, failure: afterAccept, wantCalls: 1, wantError: true}, + // The other DCs already answered: no retry, no wait, full data. + {name: "refused_other_group_answers", window: 5 * time.Second, refusals: forever, failure: refused, + otherDC: true, wantCalls: 1, maxTime: 200 * time.Millisecond}, + {name: "all_groups_refused_then_up", refusals: 3, failure: refused, otherDC: true, otherDown: 3}, + {name: "all_groups_refused_past_window", refusals: forever, failure: refused, otherDC: true, otherDown: forever, + wantError: true}, + } { + t.Run(tc.name, func(t *testing.T) { + window := tc.window + if window == 0 { + window = 300 * time.Millisecond + } + backend.SetConnectRetry(backend.ConnectRetry{Window: window, InitialDelay: 10 * time.Millisecond, MaxDelay: 50 * time.Millisecond}) + t.Cleanup(func() { backend.SetConnectRetry(backend.ConnectRetry{}) }) + store := &restartingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc1"}, + refusals: tc.refusals, failure: tc.failure} + stores := []backend.BackendImpl{store, &partialRenderBackend{path: "m.ko", cluster: "cy", dc: "dc1"}} + if tc.peer { + stores = append(stores, &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc1"}) + } + if tc.otherDC { + stores = append(stores, &restartingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc3"}, + refusals: tc.otherDown, failure: refused}) + } + app := partialRenderAppOf(t, stores...) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + elapsed := time.Since(start) + if tc.wantError { + if w.Code != http.StatusInternalServerError { + t.Fatalf("HTTP %d, want 500: %s", w.Code, w.Body.String()) + } + } else { + assertPartialRenderValue(t, w, 100) // 99 from a replica of m.ok + 1 + } + if tc.wantCalls != 0 && store.calls.Load() != tc.wantCalls { + t.Fatalf("store rendered %d times, want %d", store.calls.Load(), tc.wantCalls) + } + if tc.refusals == forever && tc.failure == refused && tc.wantError && (elapsed < window-20*time.Millisecond || store.calls.Load() < 3) { + t.Fatalf("gave up after %v and %d attempts", elapsed, store.calls.Load()) + } + if tc.maxTime != 0 && elapsed > tc.maxTime { + t.Fatalf("request took %v, want under %v", elapsed, tc.maxTime) + } + if elapsed > window+2*time.Second { + t.Fatalf("request took %v", elapsed) + } + }) + } +} + +// The fe-112 canary scenario end to end: one cluster with three replica groups, +// the one-store dc1 group refused, the dc3 and dc2 groups answering +// not-found. The render returns without the retry wait, and the response is +// the one the same stores give with retries disabled. With the series held by +// another cluster it is complete; with the series nowhere, the refusal counts +// as the cluster's not-found, so the render returns the remaining input. +func TestRenderRefusedStoreOtherGroupsNotFound(t *testing.T) { + refused := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", + Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} + for _, tc := range []struct { + name string + dataCluster bool + }{ + {name: "series_in_other_cluster", dataCluster: true}, + {name: "series_nowhere"}, + } { + t.Run(tc.name, func(t *testing.T) { + run := func(window time.Duration) (*restartingStore, int, string, time.Duration) { + backend.SetConnectRetry(backend.ConnectRetry{Window: window, InitialDelay: 10 * time.Millisecond, MaxDelay: 50 * time.Millisecond}) + t.Cleanup(func() { backend.SetConnectRetry(backend.ConnectRetry{}) }) + store := &restartingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "carbon", dc: "dc1"}, + refusals: 1 << 30, failure: refused} + stores := []backend.BackendImpl{store, + &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: "carbon", dc: "dc3"}, + &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: "carbon", dc: "dc2"}, + &partialRenderBackend{path: "m.ko", cluster: "cy", dc: "dc1"}} + if tc.dataCluster { + stores = append(stores, &partialRenderBackend{path: "m.ok", cluster: "cy", dc: "dc2"}) + } + app := partialRenderAppOf(t, stores...) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + return store, w.Code, w.Body.String(), time.Since(start) + } + store, code, body, elapsed := run(5 * time.Second) + if elapsed > 200*time.Millisecond || store.calls.Load() != 1 { + t.Fatalf("took %v and %d store calls, want no retry wait", elapsed, store.calls.Load()) + } + _, baseCode, baseBody, _ := run(0) + if code != baseCode || body != baseBody { + t.Fatalf("HTTP %d %s, without retries HTTP %d %s", code, body, baseCode, baseBody) + } + w := httptest.NewRecorder() + w.Code = code + w.Body.WriteString(body) + if tc.dataCluster { + assertPartialRenderValue(t, w, 100) + } else { + assertPartialRenderValue(t, w, 1) // m.ok not found, m.ko alone + } + }) + } +} + +// namespaceStore answers the glob a.b.* with its series, if it has one, and +// every other target with not-found. +type namespaceStore struct { + *partialRenderBackend + series string + failure error +} + +func (s *namespaceStore) Render(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + if s.failure != nil { + return nil, s.failure + } + if s.series == "" || len(r.Targets) != 1 || r.Targets[0] != "a.b.*" { + return nil, types.ErrMetricsNotFound + } + m := types.Metric{Name: s.series, StartTime: r.From, StopTime: r.Until, StepTime: 60} + for ts := r.From; ts < r.Until; ts += 60 { + m.Values = append(m.Values, 42) + m.IsAbsent = append(m.IsAbsent, false) + } + return []types.Metric{m}, nil +} + +// The fe-112 canary 500s: dashboards pass the literal target "*" as an extra +// argument, as in sumSeries(a.b.*, *). With the one store of the dc1 carbon +// group refused or timing out at dial and the 93 other stores answering +// not-found for "*", the render returns 200 with the a.b.* series instead of +// failing on the unreachable store. If every carbon group is unreachable, the +// failed "*" fetch could hide data, and the render still fails (the required-fetch rule). +func TestRenderStarWithUnreachableSingleStoreGroup(t *testing.T) { + refused := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", + Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} + dialTimeout := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.ErrDeadlineExceeded}} + for _, tc := range []struct { + name string + failure error + allCarbonDown bool + }{ + {name: "refused", failure: refused}, + {name: "dial_timeout", failure: dialTimeout}, + {name: "every_carbon_group_unreachable", failure: refused, allCarbonDown: true}, + } { + t.Run(tc.name, func(t *testing.T) { + // connectRetry 10s as in production: no retry wait either way. + backend.SetConnectRetry(backend.ConnectRetry{Window: 10 * time.Second, InitialDelay: 10 * time.Millisecond, MaxDelay: 50 * time.Millisecond}) + t.Cleanup(func() { backend.SetConnectRetry(backend.ConnectRetry{}) }) + if tc.allCarbonDown { + backend.SetConnectRetry(backend.ConnectRetry{}) + } + store := func(name, cluster, dc string) *namespaceStore { + return &namespaceStore{partialRenderBackend: &partialRenderBackend{path: name, cluster: cluster, dc: dc}} + } + stores := []backend.BackendImpl{&namespaceStore{partialRenderBackend: &partialRenderBackend{path: "store-1", cluster: "carbon", dc: "dc1"}, failure: tc.failure}} + for _, dc := range []string{"dc3", "dc2"} { + for i := range 3 { + s := store(fmt.Sprintf("carbon-%s-%d", dc, i), "carbon", dc) + if tc.allCarbonDown { + s.failure = tc.failure + } + stores = append(stores, s) + } + } + for c := range 29 { + for _, dc := range []string{"dc1", "dc3", "dc2"} { + s := store(fmt.Sprintf("other-%d-%s", c, dc), fmt.Sprintf("other-%d", c), dc) + if c == 0 { + s.series = "a.b.c" // a.b.* lives in another cluster + } + stores = append(stores, s) + } + } + if len(stores) != 94 { + t.Fatalf("%d stores", len(stores)) + } + app := partialRenderAppOf(t, stores...) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(a.b.*, *)") + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("render took %v", elapsed) + } + if tc.allCarbonDown { + if w.Code != http.StatusInternalServerError { + t.Fatalf("HTTP %d, want 500 while every carbon copy is unreachable: %s", w.Code, w.Body.String()) + } + return + } + assertPartialRenderValue(t, w, 42) + _, err := Render(app.TopLevelDomainCache, nil, false, app.Backends, app.ZipperConfig.RenderReplicaMismatchConfig, + context.Background(), "*", 1794392400, 1794393000, app.ZipperMetrics, zap.NewNop()) + var notFound types.ErrNotFound + if !errors.As(err, ¬Found) { + t.Fatalf("fetch of * = %v, want not-found", err) + } + }) + } +} diff --git a/pkg/app/carbonapi/metrics.go b/pkg/app/carbonapi/metrics.go index d49a01b2..4242893d 100644 --- a/pkg/app/carbonapi/metrics.go +++ b/pkg/app/carbonapi/metrics.go @@ -53,6 +53,11 @@ type ZipperPrometheusMetrics struct { BackendResponses *prometheus.CounterVec + BackendConnectRetries prometheus.Counter + BackendConnectRecoveries prometheus.Counter + BackendConnectCancels prometheus.Counter + BackendSlowReplicaCuts prometheus.Counter + RenderOutDurationExp *prometheus.HistogramVec FindOutDuration *prometheus.HistogramVec @@ -342,6 +347,10 @@ func registerPrometheusMetrics(ms *PrometheusMetrics, zms *ZipperPrometheusMetri prometheus.MustRegister(zms.RenderFixedMismatches) prometheus.MustRegister(zms.RenderMismatchedResponses) prometheus.MustRegister(zms.BackendResponses) + prometheus.MustRegister(zms.BackendConnectRetries) + prometheus.MustRegister(zms.BackendConnectRecoveries) + prometheus.MustRegister(zms.BackendConnectCancels) + prometheus.MustRegister(zms.BackendSlowReplicaCuts) prometheus.MustRegister(zms.RenderOutDurationExp) prometheus.MustRegister(zms.FindOutDuration) prometheus.MustRegister(zms.BackendEnqueuedRequests) @@ -462,5 +471,29 @@ func NewZipperPrometheusMetrics(config cfg.Zipper) *ZipperPrometheusMetrics { }, []string{"status", "request"}, ), + BackendConnectRetries: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "backend_connect_retries_total", + Help: "Render fetches repeated after a backend refused or dropped the connection before responding", + }, + ), + BackendConnectRecoveries: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "backend_connect_retry_recoveries_total", + Help: "Repeated render fetches that the backend answered with data or not-found", + }, + ), + BackendConnectCancels: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "backend_connect_retries_cancelled_total", + Help: "Render fetches of an unreachable backend given up without waiting for the retry window because another replica group of the same cluster answered (data or not-found)", + }, + ), + BackendSlowReplicaCuts: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "backend_slow_replica_cancelled_total", + Help: "Render fetches of a backend no longer awaited because every other replica group of the same cluster had answered, at least one with data, and this backend stayed silent for timeouts.slowReplicaGrace", + }, + ), } } diff --git a/pkg/app/carbonapi/partial_render_test.go b/pkg/app/carbonapi/partial_render_test.go index 35b882a4..2d93afcd 100644 --- a/pkg/app/carbonapi/partial_render_test.go +++ b/pkg/app/carbonapi/partial_render_test.go @@ -29,6 +29,9 @@ type partialRenderBackend struct { path string err error empty bool + // Leave cluster empty for the strict, ungrouped-backend regression tests. + cluster string + dc string } func (b *partialRenderBackend) Contains(paths []string) bool { @@ -39,9 +42,11 @@ func (b *partialRenderBackend) Contains(paths []string) bool { } return false } -func (b *partialRenderBackend) Logger() *zap.Logger { return zap.NewNop() } -func (b *partialRenderBackend) GetServerAddress() string { return b.path } -func (b *partialRenderBackend) BackendInfo() (string, string, string) { return b.path, "test", "test" } +func (b *partialRenderBackend) Logger() *zap.Logger { return zap.NewNop() } +func (b *partialRenderBackend) GetServerAddress() string { return b.path } +func (b *partialRenderBackend) BackendInfo() (string, string, string) { + return b.path, b.cluster, b.dc +} func (b *partialRenderBackend) Render(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { if b.err != nil || b.empty { return nil, b.err @@ -59,6 +64,15 @@ func (b *partialRenderBackend) Render(_ context.Context, r types.RenderRequest) } func partialRenderApp(t *testing.T, stores ...*partialRenderBackend) *App { + t.Helper() + impls := make([]backend.BackendImpl, len(stores)) + for i, store := range stores { + impls[i] = store + } + return partialRenderAppOf(t, impls...) +} + +func partialRenderAppOf(t *testing.T, stores ...backend.BackendImpl) *App { t.Helper() functions.New(nil, zap.NewNop()) c := cfg.DefaultAPIConfig() diff --git a/pkg/app/carbonapi/replica_group_render_test.go b/pkg/app/carbonapi/replica_group_render_test.go new file mode 100644 index 00000000..732c1017 --- /dev/null +++ b/pkg/app/carbonapi/replica_group_render_test.go @@ -0,0 +1,97 @@ +package carbonapi + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "syscall" + "testing" + + "github.com/bookingcom/carbonapi/pkg/carbonapipb" + "github.com/bookingcom/carbonapi/pkg/types" + "go.uber.org/zap" +) + +func TestRenderReplicaGroupFailure(t *testing.T) { + refused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED} + dialTimeout := &net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded} + ioTimeout := &net.OpError{Op: "read", Net: "tcp", Err: syscall.ETIMEDOUT} + for _, tc := range []struct { + name string + failure error + cluster string + peerCluster string + peerDC string + peerError error + peerEmpty bool + wantError bool + wantData bool + }{ + {name: "Vanguard_one_refused_93_not_found", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: types.ErrMetricsNotFound}, + {name: "dial_timeout_and_not_found", failure: dialTimeout, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: types.ErrMetricsNotFound}, + {name: "io_timeout_and_not_found", failure: ioTimeout, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: types.ErrMetricsNotFound}, + {name: "wrapped_not_found", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: fmt.Errorf("store: %w", types.ErrMetricsNotFound)}, + {name: "empty_success", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerEmpty: true}, + {name: "all_group_members_refused", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", peerError: refused, wantError: true}, + {name: "data_replica_wins", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc1", wantData: true}, + {name: "other_group_cannot_recover", failure: refused, cluster: "cy", peerCluster: "infra", peerDC: "dc1", peerError: types.ErrMetricsNotFound, wantError: true}, + // A bare dial error is not an http.Client connect failure (no *url.Error), + // so the cluster not-found rule does not apply; real refusals are in + // TestRenderStarWithUnreachableSingleStoreGroup. + {name: "other_dc_cannot_recover", failure: refused, cluster: "cy", peerCluster: "cy", peerDC: "dc4", peerError: types.ErrMetricsNotFound, wantError: true}, + {name: "ungrouped_unchanged", failure: refused, peerError: types.ErrMetricsNotFound, wantError: true}, + } { + t.Run(tc.name, func(t *testing.T) { + stores := []*partialRenderBackend{{path: "m.ok", err: tc.failure, cluster: tc.cluster, dc: "dc1"}} + for range 2 { + stores = append(stores, &partialRenderBackend{path: "m.ok", err: tc.peerError, empty: tc.peerEmpty, cluster: tc.peerCluster, dc: tc.peerDC}) + } + // The canary had 94 replies: one refusal and 93 not-found, + // including the two peers in the refused store's replica group. + for i := range 91 { + stores = append(stores, &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: fmt.Sprintf("other-%d", i), dc: "dc1"}) + } + // Even in the same group, success for another target must not hide this + // failed fetch. Path routing excludes this peer from the m.ok request. + stores = append(stores, &partialRenderBackend{path: "m.ko", cluster: tc.cluster, dc: "dc1"}) + app := partialRenderApp(t, stores...) + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + if tc.wantError { + if w.Code != http.StatusInternalServerError { + t.Fatalf("HTTP %d, want 500: %s", w.Code, w.Body.String()) + } + } else { + want := 1.0 + if tc.wantData { + want = 100 + } + assertPartialRenderValue(t, w, want) + } + + // Exercise ordinary and grouped fetches separately; replies for another + // target, group or DC must not establish success for this request. + metrics, err := Render(app.TopLevelDomainCache, nil, false, app.Backends, app.ZipperConfig.RenderReplicaMismatchConfig, + context.Background(), "m.ok", 1794392400, 1794393000, app.ZipperMetrics, zap.NewNop()) + checkReplicaGroupFetch(t, err, len(metrics), tc.wantError, tc.wantData) + replies := sendGroupedRenderRequests(app, context.Background(), []string{"m.ok", "m.ko"}, 1794392400, 1794393000, &carbonapipb.AccessLogDetails{}, zap.NewNop()) + checkReplicaGroupFetch(t, replies[0].error, len(replies[0].data), tc.wantError, tc.wantData) + if replies[1].error != nil || len(replies[1].data) != 1 { + t.Fatalf("independent input changed: %+v", replies[1]) + } + }) + } +} + +func checkReplicaGroupFetch(t *testing.T, err error, nMetrics int, wantError, wantData bool) { + t.Helper() + var notFound types.ErrNotFound + fatal := err != nil && !errors.As(err, ¬Found) + if fatal != wantError { + t.Fatalf("fetch error = %v, want fatal %v", err, wantError) + } + if (nMetrics != 0) != wantData { + t.Fatalf("got %d metrics, want data %v", nMetrics, wantData) + } +} diff --git a/pkg/app/carbonapi/slow_replica_test.go b/pkg/app/carbonapi/slow_replica_test.go new file mode 100644 index 00000000..e271fef1 --- /dev/null +++ b/pkg/app/carbonapi/slow_replica_test.go @@ -0,0 +1,69 @@ +package carbonapi + +import ( + "context" + "testing" + "time" + + "github.com/bookingcom/carbonapi/pkg/backend" + "github.com/bookingcom/carbonapi/pkg/types" +) + +// stallingStore accepts renders and answers only after stall, like a store +// warming up after a restart. +type stallingStore struct { + *partialRenderBackend + stall time.Duration + release chan struct{} +} + +func (s *stallingStore) Render(ctx context.Context, r types.RenderRequest) ([]types.Metric, error) { + select { + case <-s.release: + case <-time.After(s.stall): + } + return s.partialRenderBackend.Render(ctx, r) +} + +// One store of a three-DC cluster stalls for 10 s: the render answers 200 with +// the other DCs' data after the grace instead of waiting for the store. +func TestRenderDoesNotWaitForSlowReplica(t *testing.T) { + const grace = time.Second + backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{Grace: grace}) + t.Cleanup(func() { backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{}) }) + slow := &stallingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc2"}, + stall: 10 * time.Second, release: make(chan struct{})} + t.Cleanup(func() { close(slow.release) }) + app := partialRenderAppOf(t, slow, + &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc1"}, + &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc3"}, + &partialRenderBackend{path: "m.ko", cluster: "cy", dc: "dc1"}) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + if elapsed := time.Since(start); elapsed > 1500*time.Millisecond { + t.Fatalf("render took %v, want about the %v grace", elapsed, grace) + } + assertPartialRenderValue(t, w, 100) // 99 from a replica of m.ok + 1 +} + +// One store stalls and the other DCs of its cluster answer not-found: the +// slow store may hold the only copy, so the render waits for it and returns +// its data rather than cutting it after the grace. +func TestRenderWaitsForSlowPossibleOnlyHolder(t *testing.T) { + const stall = 300 * time.Millisecond + backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{Grace: 20 * time.Millisecond}) + t.Cleanup(func() { backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{}) }) + slow := &stallingStore{partialRenderBackend: &partialRenderBackend{path: "m.ok", cluster: "cx", dc: "dc2"}, + stall: stall, release: make(chan struct{})} + t.Cleanup(func() { close(slow.release) }) + app := partialRenderAppOf(t, slow, + &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: "cx", dc: "dc1"}, + &partialRenderBackend{path: "m.ok", err: types.ErrMetricsNotFound, cluster: "cx", dc: "dc3"}, + &partialRenderBackend{path: "m.ko", cluster: "cy", dc: "dc1"}) + start := time.Now() + w := partialRenderRequest(app, "sumSeries(m.ok,m.ko)") + if elapsed := time.Since(start); elapsed < stall-20*time.Millisecond { + t.Fatalf("render returned after %v without the possible only holder", elapsed) + } + assertPartialRenderValue(t, w, 100) // 99 from the slow store + 1 +} diff --git a/pkg/app/carbonapi/upstream.go b/pkg/app/carbonapi/upstream.go index cfce6cfc..29f48f24 100644 --- a/pkg/app/carbonapi/upstream.go +++ b/pkg/app/carbonapi/upstream.go @@ -107,8 +107,8 @@ func Info(cache *expirecache.Cache, TLDPrefixes []tldcache.TopLevelDomainPrefix, return infos, err } -// An empty successful replica reply cannot recover a failed series fetch. -// A replica that supplies data still wins over errors from other replicas. +// Backend fan-in has already recovered errors within answered replica groups. +// Remaining errors need data to recover; an unrelated empty reply is insufficient. func renderErrorsFanIn(metrics []types.Metric, errs []error, nBackends int) error { if len(metrics) == 0 { nBackends = len(errs) diff --git a/pkg/backend/connect_retry.go b/pkg/backend/connect_retry.go new file mode 100644 index 00000000..2e8c3945 --- /dev/null +++ b/pkg/backend/connect_retry.go @@ -0,0 +1,314 @@ +package backend + +import ( + "context" + "errors" + "io" + "net" + "net/url" + "sync/atomic" + "syscall" + "time" + + "github.com/bookingcom/carbonapi/pkg/types" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +// ConnectRetry keeps render fetches alive across a short backend outage, such +// as a store restart. Only fetches that failed before the backend sent any +// response are repeated; a failure after the backend answered stays final. +type ConnectRetry struct { + // Window is the total backoff spent on one fetch. Zero disables retries. + Window time.Duration + // InitialDelay doubles after every attempt up to MaxDelay. + InitialDelay, MaxDelay time.Duration + + Retries prometheus.Counter // repeated fetches + Recoveries prometheus.Counter // repeated fetches that the backend answered + // Cancellations counts unreachable fetches given up because another + // replica group of the same cluster answered for the target (data or + // not-found). + Cancellations prometheus.Counter +} + +var connectRetry atomic.Pointer[ConnectRetry] + +// SetConnectRetry replaces the process-wide render retry policy. +func SetConnectRetry(c ConnectRetry) { + if c.InitialDelay <= 0 { + c.InitialDelay = 250 * time.Millisecond + } + if c.MaxDelay < c.InitialDelay { + c.MaxDelay = max(2*time.Second, c.InitialDelay) + } + connectRetry.Store(&c) +} + +// unreachableError marks a fetch that never reached a backend response and +// remembers the backend to repeat it against. +type unreachableError struct { + error + backend Backend +} + +func (e unreachableError) Unwrap() error { return e.error } + +func (b *Backend) renderError(err error) error { + if connectFailure(err) { + return unreachableError{err, *b} + } + return err +} + +// connectFailure reports whether err is a refused, reset or timed-out +// connection seen before any response. http.Client.Do wraps exactly those +// failures in *url.Error; body reads and decoding happen after the backend +// answered and never produce one. +func connectFailure(err error) bool { + var urlErr *url.Error + if !errors.As(err, &urlErr) { + return false + } + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "dial" { + return true + } + return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, io.EOF) +} + +type renderFetch struct { + request types.RenderRequest + groups []renderGroup + messages [][]types.Metric +} + +// retryUnreachable repeats unreachable fetches that no replica group has +// answered, with one backoff schedule shared by all fetches of a request. +// A fetch is repeated only while no other replica group of its cluster has +// answered for the target at all. Replica groups of one cluster hold the same +// names, so a definitive answer from any of them, data or not-found, already +// says what the unreachable store would return: with data the merged result is +// complete without it, and with not-found the names live in another cluster +// (renders fan out to every cluster) or nowhere. Waiting for the store would +// only delay the response. Retries are for the case where every copy of the +// cluster is unreachable, such as a one-group cluster restarting. It returns +// early when the request deadline cannot fit another attempt. +func retryUnreachable(ctx context.Context, fetches []renderFetch, lg *zap.Logger) { + policy := connectRetry.Load() + if policy == nil || policy.Window <= 0 { + return + } + cancelCovered(fetches, policy, lg) + if !anyUnreachable(fetches) { + return + } + start := time.Now() + delay := policy.InitialDelay + for attempt := 1; anyUnreachable(fetches); attempt++ { + wait := min(delay, policy.Window-time.Since(start)) + if wait <= 0 { + return + } + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) <= wait { + return + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + delay = min(2*delay, policy.MaxDelay) + + var flights []retryFlight + for fi := range fetches { + f := &fetches[fi] + for gi := range f.groups { + g := &f.groups[gi] + if !g.unrecovered() { + continue + } + var sent []unreachableError + kept := g.errs[:0] + for _, err := range g.errs { + u, ok := err.(unreachableError) + if !ok { + kept = append(kept, err) + continue + } + lg.Debug("retrying render after backend connect failure", + zap.String("backend", u.backend.GetServerAddress()), + zap.Strings("targets", f.request.Targets), + zap.Int("attempt", attempt), + zap.Duration("elapsed", time.Since(start)), + zap.Error(u.error)) + u.backend.SendRender(ctx, f.request, g.results, g.errors) + sent = append(sent, u) + } + clear(g.errs[len(kept):]) + g.errs = kept + if len(sent) != 0 { + flights = append(flights, retryFlight{fetch: fi, group: gi, sent: sent}) + if policy.Retries != nil { + policy.Retries.Add(float64(len(sent))) + } + } + } + } + awaitRetries(fetches, flights) + for _, fl := range flights { + if fl.answered == 0 { + continue + } + if policy.Recoveries != nil { + policy.Recoveries.Add(float64(fl.answered)) + } + lg.Debug("render recovered after backend connect retry", + zap.Strings("targets", fetches[fl.fetch].request.Targets), + zap.Int("attempt", attempt), + zap.Duration("elapsed", time.Since(start))) + } + cancelCovered(fetches, policy, lg) + } +} + +// retryFlight is one replica group's repeated fetches within an attempt. +type retryFlight struct { + fetch, group int + sent []unreachableError + pending int // replies still awaited + answered int +} + +type retryReply struct { + flight int + metrics []types.Metric + err error +} + +// awaitRetries collects the replies of one attempt. As soon as a replica group +// answers (data or not-found), the same fetch's repeats to other groups of that +// cluster are no longer awaited: they keep their unreachable error for +// cancelCovered, and their late replies are dropped. +func awaitRetries(fetches []renderFetch, flights []retryFlight) { + outstanding := 0 + for i := range flights { + flights[i].pending = len(flights[i].sent) + outstanding += flights[i].pending + } + // Buffered for every reply, so abandoned forwarders never block. + replies := make(chan retryReply, outstanding) + for i := range flights { + g := &fetches[flights[i].fetch].groups[flights[i].group] + go func(flight, n int, results <-chan []types.Metric, errs <-chan error) { + for range n { + metrics, err := nextReply(results, errs) + replies <- retryReply{flight, metrics, err} + } + }(i, flights[i].pending, g.results, g.errors) + } + for outstanding > 0 { + r := <-replies + fl := &flights[r.flight] + if fl.pending == 0 { + continue // abandoned + } + fl.pending-- + outstanding-- + f := &fetches[fl.fetch] + g := &f.groups[fl.group] + var answer bool + if f.messages, answer = g.take(r.metrics, r.err, f.messages); !answer { + continue + } + fl.answered++ + if g.cluster == "" { + continue + } + for j := range flights { + o := &flights[j] + if j == r.flight || o.fetch != fl.fetch || o.pending == 0 || f.groups[o.group].cluster != g.cluster { + continue + } + og := &f.groups[o.group] + for _, u := range o.sent[len(o.sent)-o.pending:] { + og.errs = append(og.errs, u) + } + outstanding -= o.pending + o.pending = 0 + } + } +} + +// cancelCovered gives up the unreachable fetches of every replica group whose +// target another group of the same cluster has answered for, with data or +// not-found. Their errors become final plain connect errors and the fan-in +// decides the render exactly as without retries (renderGroupErrors): they +// count as the cluster's not-found when every other group of the cluster +// answered not-found, and otherwise stay errors for the required-fetch and replica-group rules. Ungrouped +// backends are never covered: they never recover each other's errors. +func cancelCovered(fetches []renderFetch, policy *ConnectRetry, lg *zap.Logger) { + for fi := range fetches { + f := &fetches[fi] + for gi := range f.groups { + g := &f.groups[gi] + if !g.unrecovered() || !f.covered(g.cluster) { + continue + } + n := 0 + for i, err := range g.errs { + if u, ok := err.(unreachableError); ok { + g.errs[i] = u.error + n++ + } + } + if n == 0 { + continue + } + if policy.Cancellations != nil { + policy.Cancellations.Add(float64(n)) + } + lg.Debug("render connect retry cancelled: another replica group of the cluster answered", + zap.Strings("targets", f.request.Targets), + zap.String("cluster", g.cluster), + zap.Int("fetches", n)) + } + } +} + +// covered reports whether a replica group of cluster gave a definitive answer +// for this fetch: data or the backend's own not-found. A not-found counts as +// much as data. The canary of the data-only rule (30s refusal of a one-store +// group) still waited the whole window for every glob held in another +// cluster, because the refused cluster's other groups could only answer +// not-found: 41 HTTP 500 and 67 499 in 30s on the refused frontend. +func (f *renderFetch) covered(cluster string) bool { + if cluster == "" { + return false + } + for gi := range f.groups { + if f.groups[gi].cluster == cluster && f.groups[gi].answered { + return true + } + } + return false +} + +func anyUnreachable(fetches []renderFetch) bool { + for fi := range fetches { + for gi := range fetches[fi].groups { + g := &fetches[fi].groups[gi] + if !g.unrecovered() { + continue + } + for _, err := range g.errs { + if _, ok := err.(unreachableError); ok { + return true + } + } + } + } + return false +} diff --git a/pkg/backend/connect_retry_test.go b/pkg/backend/connect_retry_test.go new file mode 100644 index 00000000..118ab811 --- /dev/null +++ b/pkg/backend/connect_retry_test.go @@ -0,0 +1,557 @@ +package backend + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "sync/atomic" + "syscall" + "testing" + "time" + + bnet "github.com/bookingcom/carbonapi/pkg/backend/net" + "github.com/bookingcom/carbonapi/pkg/cfg" + "github.com/bookingcom/carbonapi/pkg/types" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "go.uber.org/zap" +) + +var refusedDial = &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", + Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}} + +type retryTestBackend struct { + coalesceTestBackend + cluster, dc string +} + +func (b retryTestBackend) BackendInfo() (string, string, string) { return b.cluster, b.cluster, b.dc } + +// retryStore fails its first refusals calls with failure, then answers. +func retryStore(cluster string, refusals int32, failure error, answer error) (Backend, *atomic.Int32) { + return retryStoreIn("dc1", cluster, refusals, failure, answer) +} + +// retryStoreIn is retryStore in the replica group of dc. +func retryStoreIn(dc, cluster string, refusals int32, failure error, answer error) (Backend, *atomic.Int32) { + return retryStoreFunc(dc, cluster, refusals, failure, func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + if answer != nil { + return nil, answer + } + return coalesceMetrics(r.Targets), nil + }) +} + +func retryStoreFunc(dc, cluster string, refusals int32, failure error, + serve func(context.Context, types.RenderRequest) ([]types.Metric, error)) (Backend, *atomic.Int32) { + calls := new(atomic.Int32) + b := coalesceBackend(func(ctx context.Context, r types.RenderRequest) ([]types.Metric, error) { + if calls.Add(1) <= refusals { + return nil, failure + } + return serve(ctx, r) + }, 4) + b.BackendImpl = retryTestBackend{coalesceTestBackend: b.BackendImpl.(coalesceTestBackend), cluster: cluster, dc: dc} + b.Proc() + return b, calls +} + +func counterValue(c prometheus.Counter) float64 { + m := &dto.Metric{} + if err := c.Write(m); err != nil { + panic(err) + } + return m.GetCounter().GetValue() +} + +func fastConnectRetry(t *testing.T, window time.Duration) (retries, recoveries prometheus.Counter) { + t.Helper() + retries, recoveries, _ = fastConnectRetryCancels(t, window) + return retries, recoveries +} + +func fastConnectRetryCancels(t *testing.T, window time.Duration) (retries, recoveries, cancels prometheus.Counter) { + t.Helper() + retries = prometheus.NewCounter(prometheus.CounterOpts{Name: "test_retries"}) + recoveries = prometheus.NewCounter(prometheus.CounterOpts{Name: "test_recoveries"}) + cancels = prometheus.NewCounter(prometheus.CounterOpts{Name: "test_cancels"}) + SetConnectRetry(ConnectRetry{Window: window, InitialDelay: 10 * time.Millisecond, MaxDelay: 40 * time.Millisecond, + Retries: retries, Recoveries: recoveries, Cancellations: cancels}) + t.Cleanup(func() { connectRetry.Store(nil) }) + return retries, recoveries, cancels +} + +// fetchBoth runs the ordinary and the grouped (stable batch) fan-in. +func fetchBoth(t *testing.T, ctx context.Context, backends []Backend, path string, check func(name string, metrics []types.Metric, errs []error)) { + t.Helper() + request := types.NewRenderRequest([]string{path}, 1, 4) + metrics, _, errs := Renders(ctx, backends, request, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + check("ordinary", metrics, errs) + replies := RendersGrouped(ctx, [][]Backend{backends, backends}, []types.RenderRequest{request, types.NewRenderRequest([]string{path + ".2"}, 1, 4)}, + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + for _, reply := range replies { + check("grouped", reply.Metrics, reply.Errors) + } +} + +func TestConnectRetryRefusedThenUp(t *testing.T) { + for _, grouped := range []bool{false, true} { + retries, recoveries := fastConnectRetry(t, 2*time.Second) + // The store refuses its first three connections, then serves data. + b, _ := retryStore("cx", 3, refusedDial, nil) + start := time.Now() + request := types.NewRenderRequest([]string{"m"}, 1, 4) + replies := []GroupedRenderReply{{}} + if grouped { + replies = RendersGrouped(context.Background(), [][]Backend{{b}, {b}}, + []types.RenderRequest{request, types.NewRenderRequest([]string{"n"}, 1, 4)}, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + } else { + replies[0].Metrics, _, replies[0].Errors = Renders(context.Background(), []Backend{b}, request, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + } + for _, reply := range replies { + if len(reply.Errors) != 0 || len(reply.Metrics) != 1 || reply.Metrics[0].Values[1] != 42 { + t.Fatalf("grouped %v: %+v, want full data", grouped, reply) + } + } + if counterValue(retries) == 0 || counterValue(recoveries) == 0 || time.Since(start) > time.Second { + t.Fatalf("grouped %v: retries %v recoveries %v after %v", grouped, counterValue(retries), counterValue(recoveries), time.Since(start)) + } + } +} + +func TestConnectRetryWindowExhausted(t *testing.T) { + const window = 200 * time.Millisecond + retries, recoveries := fastConnectRetry(t, window) + b, _ := retryStore("cx", 1<<30, refusedDial, nil) + start := time.Now() + metrics, _, errs := Renders(context.Background(), []Backend{b}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + elapsed := time.Since(start) + if len(metrics) != 0 || len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("metrics %+v errs %v, want the refusal", metrics, errs) + } + var u unreachableError + if errors.As(errs[0], &u) { + t.Fatal("internal retry marker leaked") + } + if elapsed < window-20*time.Millisecond || elapsed > window+time.Second { + t.Fatalf("gave up after %v, want about %v", elapsed, window) + } + if counterValue(retries) == 0 || counterValue(recoveries) != 0 { + t.Fatalf("retries %v recoveries %v", counterValue(retries), counterValue(recoveries)) + } +} + +func TestConnectRetryRespectsDeadline(t *testing.T) { + fastConnectRetry(t, 10*time.Second) + b, _ := retryStore("cx", 1<<30, refusedDial, nil) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + _, _, errs := Renders(ctx, []Backend{b}, types.NewRenderRequest([]string{"m"}, 1, 4), cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("errs %v, want the refusal", errs) + } + if ctx.Err() != nil { + t.Fatal("retries ran past the request deadline") + } +} + +func TestConnectRetrySkipsAnsweredGroup(t *testing.T) { + for _, peer := range []error{nil, types.ErrMetricsNotFound} { + retries, _ := fastConnectRetry(t, 2*time.Second) + refused, calls := retryStore("cy", 1<<30, refusedDial, nil) + answering, _ := retryStore("cy", 0, nil, peer) + start := time.Now() + fetchBoth(t, context.Background(), []Backend{refused, answering}, "m", func(name string, metrics []types.Metric, errs []error) { + wantErrs := 1 + if peer != nil { + wantErrs = 2 + } + if len(errs) != wantErrs || (peer == nil) != (len(metrics) == 1) { + t.Fatalf("%s peer %v: metrics %+v errs %v, want group recovery", name, peer, metrics, errs) + } + for _, err := range errs { + if !errors.Is(err, types.ErrMetricsNotFound) { + t.Fatalf("%s peer %v: unrecovered %v", name, peer, errs) + } + } + }) + if calls.Load() == 0 || counterValue(retries) != 0 || time.Since(start) > time.Second { + t.Fatalf("peer %v: answered group was retried: retries %v after %v", peer, counterValue(retries), time.Since(start)) + } + } +} + +// A store whose whole replica group is unreachable is not retried once another +// DC's group of the same cluster returned data: the render returns at the +// healthy groups' speed with their full data, and the refusal stays a plain +// error that the required-fetch rule recovers from the other group's data. +func TestConnectRetryCancelledByOtherGroup(t *testing.T) { + retries, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + refused, _ := retryStoreIn("dc1", "cy", 1<<30, refusedDial, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fra, _ := retryStoreIn("dc2", "cy", 0, nil, nil) + start := time.Now() + fetchBoth(t, context.Background(), []Backend{refused, lhr, fra}, "m", func(name string, metrics []types.Metric, errs []error) { + if len(metrics) != 1 || metrics[0].Values[1] != 42 { + t.Fatalf("%s: metrics %+v, want the other groups' data", name, metrics) + } + if len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("%s: errs %v, want the refusal", name, errs) + } + var u unreachableError + if errors.As(errs[0], &u) { + t.Fatalf("%s: internal retry marker leaked", name) + } + }) + if elapsed := time.Since(start); elapsed > 200*time.Millisecond { + t.Fatalf("waited %v for a store other groups had already covered", elapsed) + } + // One ordinary and two grouped fetches. + if counterValue(retries) != 0 || counterValue(cancels) != 3 { + t.Fatalf("retries %v cancels %v, want 0/3", counterValue(retries), counterValue(cancels)) + } +} + +// Only an answer from a replica group of the same cluster makes a retry +// redundant. +func TestConnectRetryNotCancelledWithoutReplicaAnswer(t *testing.T) { + const window = 150 * time.Millisecond + for _, tc := range []struct { + name string + dc, cluster string + answer error + ungroupedPeer bool + }{ + {name: "other_cluster_data", dc: "dc3", cluster: "carbon"}, + {name: "other_cluster_not_found", dc: "dc3", cluster: "carbon", answer: types.ErrMetricsNotFound}, + {name: "other_group_server_error", dc: "dc3", cluster: "cy", answer: bnet.ErrHTTPCode(http.StatusServiceUnavailable)}, + {name: "ungrouped", ungroupedPeer: true}, + } { + t.Run(tc.name, func(t *testing.T) { + retries, _, cancels := fastConnectRetryCancels(t, window) + cluster := "cy" + if tc.ungroupedPeer { + cluster = "" + } + refused, _ := retryStoreIn("dc1", cluster, 1<<30, refusedDial, nil) + peer, _ := retryStoreIn(tc.dc, tc.cluster, 0, nil, tc.answer) + start := time.Now() + Renders(context.Background(), []Backend{refused, peer}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if elapsed := time.Since(start); elapsed < window-20*time.Millisecond || counterValue(retries) == 0 || counterValue(cancels) != 0 { + t.Fatalf("gave up after %v: retries %v cancels %v, want the full window", elapsed, counterValue(retries), counterValue(cancels)) + } + }) + } +} + +// The fe-112 canary: the refused store is the only member of one of its +// cluster's replica groups, the other groups of that cluster answer not-found, +// and the series lives in another cluster. Replicas of one cluster hold the +// same names, so their not-found already tells what the refused store would +// say: the fetch is given up at once instead of after the whole retry window, +// and the refusal counts as the cluster's not-found (renderGroupErrors). With a +// second group of the cluster unreachable too, not every other group answered, +// so both refusals stay plain connect errors. +func TestConnectRetryCancelledByOtherGroupsNotFound(t *testing.T) { + for _, tc := range []struct { + name string + dataCluster bool // a group of another cluster holds the series + lhrRefused bool // a second group of the refused cluster is unreachable too + wantCancels float64 + }{ + {name: "series_in_other_cluster", dataCluster: true, wantCancels: 3}, + {name: "series_nowhere", wantCancels: 3}, + {name: "two_groups_refused_one_not_found", dataCluster: true, lhrRefused: true, wantCancels: 6}, + } { + t.Run(tc.name, func(t *testing.T) { + retries, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + refused, _ := retryStoreIn("dc1", "carbon", 1<<30, refusedDial, nil) + lhrRefusals := int32(0) + if tc.lhrRefused { + lhrRefusals = 1 << 30 + } + lhr, _ := retryStoreIn("dc3", "carbon", lhrRefusals, refusedDial, types.ErrMetricsNotFound) + fra, _ := retryStoreIn("dc2", "carbon", 0, nil, types.ErrMetricsNotFound) + backends := []Backend{refused, lhr, fra} + if tc.dataCluster { + cy, _ := retryStoreIn("dc2", "cy", 0, nil, nil) + backends = append(backends, cy) + } + start := time.Now() + fetchBoth(t, context.Background(), backends, "m", func(name string, metrics []types.Metric, errs []error) { + if (len(metrics) == 1) != tc.dataCluster { + t.Fatalf("%s: metrics %+v, want data %v", name, metrics, tc.dataCluster) + } + refusals, notFound := 0, 0 + for _, err := range errs { + var u unreachableError + switch { + case errors.As(err, &u): + t.Fatalf("%s: internal retry marker leaked: %v", name, errs) + case errors.Is(err, syscall.ECONNREFUSED): + refusals++ + case errors.Is(err, types.ErrMetricsNotFound): + notFound++ + } + } + wantRefusals := 0 + if tc.lhrRefused { + wantRefusals = 2 + } + if refusals != wantRefusals || notFound != 3-wantRefusals { + t.Fatalf("%s: errs %v, want %d refusals as plain errors and the rest not-found", name, errs, wantRefusals) + } + }) + if elapsed := time.Since(start); elapsed > 200*time.Millisecond { + t.Fatalf("waited %v although the refused cluster's other groups answered not-found", elapsed) + } + if counterValue(retries) != 0 || counterValue(cancels) != tc.wantCancels { + t.Fatalf("retries %v cancels %v, want 0/%v", counterValue(retries), counterValue(cancels), tc.wantCancels) + } + }) + } +} + +// Every group of the cluster refuses, then one comes back and answers +// not-found: the others' retries stop there. +func TestConnectRetryAllGroupsRefusedThenNotFound(t *testing.T) { + retries, recoveries, cancels := fastConnectRetryCancels(t, 2*time.Second) + ams, _ := retryStoreIn("dc1", "carbon", 1<<30, refusedDial, nil) + lhr, _ := retryStoreIn("dc3", "carbon", 1<<30, refusedDial, nil) + fra, _ := retryStoreIn("dc2", "carbon", 3, refusedDial, types.ErrMetricsNotFound) + start := time.Now() + metrics, _, errs := Renders(context.Background(), []Backend{ams, lhr, fra}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("waited %v after a group answered not-found", elapsed) + } + if len(metrics) != 0 || len(errs) != 3 { + t.Fatalf("metrics %+v errs %v", metrics, errs) + } + if counterValue(retries) < 9 || counterValue(recoveries) != 1 || counterValue(cancels) != 2 { + t.Fatalf("retries %v recoveries %v cancels %v", counterValue(retries), counterValue(recoveries), counterValue(cancels)) + } +} + +// When every group was unreachable and one recovers, the others' repeats are +// not awaited, even if a just-restarted store accepts but answers slowly. +func TestConnectRetryStopsWaitingOnceAGroupRecovers(t *testing.T) { + _, recoveries, cancels := fastConnectRetryCancels(t, 5*time.Second) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + slow, _ := retryStoreFunc("dc1", "cy", 1, refusedDial, func(context.Context, types.RenderRequest) ([]types.Metric, error) { + <-release + return nil, refusedDial + }) + lhr, _ := retryStoreIn("dc3", "cy", 1, refusedDial, nil) + start := time.Now() + metrics, _, errs := Renders(context.Background(), []Backend{slow, lhr}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if elapsed := time.Since(start); elapsed > 300*time.Millisecond { + t.Fatalf("waited %v on a slow repeat after another group recovered", elapsed) + } + if len(metrics) != 1 || len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("metrics %+v errs %v, want lhr data and the ams refusal", metrics, errs) + } + var u unreachableError + if errors.As(errs[0], &u) { + t.Fatal("internal retry marker leaked") + } + if counterValue(recoveries) != 1 || counterValue(cancels) != 1 { + t.Fatalf("recoveries %v cancels %v, want 1/1", counterValue(recoveries), counterValue(cancels)) + } +} + +// All replica groups refuse briefly: retries continue until one is back. +func TestConnectRetryAllGroupsRefusedThenUp(t *testing.T) { + retries, recoveries, cancels := fastConnectRetryCancels(t, 2*time.Second) + ams, _ := retryStoreIn("dc1", "cy", 1<<30, refusedDial, nil) + lhr, _ := retryStoreIn("dc3", "cy", 3, refusedDial, nil) + metrics, _, errs := Renders(context.Background(), []Backend{ams, lhr}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if len(metrics) != 1 || metrics[0].Values[1] != 42 || len(errs) != 1 || !errors.Is(errs[0], syscall.ECONNREFUSED) { + t.Fatalf("metrics %+v errs %v, want lhr data after its restart", metrics, errs) + } + if counterValue(retries) < 6 || counterValue(recoveries) != 1 || counterValue(cancels) != 1 { + t.Fatalf("retries %v recoveries %v cancels %v", counterValue(retries), counterValue(recoveries), counterValue(cancels)) + } +} + +func TestConnectRetryKeepsFailuresAfterAccept(t *testing.T) { + for _, failure := range []error{ + bnet.ErrHTTPCode(http.StatusServiceUnavailable), + io.ErrUnexpectedEOF, + &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNRESET}, // body read, no *url.Error + &url.Error{Op: "Get", URL: "http://store/render/", Err: context.DeadlineExceeded}, + } { + retries, _ := fastConnectRetry(t, 2*time.Second) + b, calls := retryStore("cx", 1, failure, nil) + _, _, errs := Renders(context.Background(), []Backend{b}, types.NewRenderRequest([]string{"m"}, 1, 4), + cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + if len(errs) != 1 || errs[0] != failure || calls.Load() != 1 || counterValue(retries) != 0 { + t.Fatalf("%v: errs %v calls %d retries %v, want one final failure", failure, errs, calls.Load(), counterValue(retries)) + } + } +} + +// Real transports: a closed port and a peer that drops the connection before +// any response are connect failures; failures after a response started are not. +func TestConnectFailureClassification(t *testing.T) { + closed, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + refusedAddr := closed.Addr().String() + closed.Close() + + serve := func(handle func(net.Conn)) string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) + go func() { + for { + c, err := l.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + buf := make([]byte, 4096) + _, _ = c.Read(buf) + handle(c) + }() + } + }() + return l.Addr().String() + } + dropped := serve(func(c net.Conn) { + if tc, ok := c.(*net.TCPConn); ok { + _ = tc.SetLinger(0) // RST instead of FIN + } + }) + truncated := serve(func(c net.Conn) { + _, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: application/x-protobuf\r\nContent-Length: 1000\r\n\r\npartial") + }) + failed := serve(func(c net.Conn) { + _, _ = io.WriteString(c, "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n") + }) + + for _, transport := range []http.RoundTripper{ + &http.Transport{DisableCompression: true}, + bnet.NewSyncTransport(&http.Transport{DisableCompression: true}), + } { + for _, tc := range []struct { + addr string + want bool + }{{refusedAddr, true}, {dropped, true}, {truncated, false}, {failed, false}} { + nb, err := bnet.New(bnet.Config{Address: tc.addr, Cluster: "c", Client: &http.Client{Transport: transport}, Timeout: 2 * time.Second}) + if err != nil { + t.Fatal(err) + } + _, err = nb.Render(context.Background(), types.NewRenderRequest([]string{"m"}, 1, 4)) + if err == nil || connectFailure(err) != tc.want { + t.Errorf("%T %s: %v classified %v, want %v", transport, tc.addr, err, err != nil && connectFailure(err), tc.want) + } + } + } +} + +// The fe-112 canary fetch of the literal target "*": 94 stores, 93 answer +// not-found and the one store of the dc1 carbon group is refused or times out +// at dial. The carbon cluster's other groups answered not-found, so the failed +// store counts as not-found too and the fetch is the cluster's not-found, with +// or without connect retries. Only when no other group of its cluster gave a +// definitive not-found does the connect error stay and fail the fetch (the required-fetch rule). +func TestRenderUnreachableGroupClusterNotFound(t *testing.T) { + dialTimeout := &url.Error{Op: "Get", URL: "http://store/render/", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.ErrDeadlineExceeded}} + serverError := bnet.ErrHTTPCode(http.StatusServiceUnavailable) + for _, tc := range []struct { + name string + failure error + carbon map[string]error // answer of the other carbon groups; nil: not-found + data bool // a carbon group returns data + wantFails int // connect failures still returned as errors + }{ + {name: "refused", failure: refusedDial}, + {name: "dial_timeout", failure: dialTimeout}, + {name: "every_copy_unreachable", failure: refusedDial, carbon: map[string]error{"dc3": refusedDial, "dc2": refusedDial}, wantFails: 7}, + {name: "another_group_unreachable", failure: refusedDial, carbon: map[string]error{"dc3": refusedDial}, wantFails: 4}, + {name: "another_group_server_error", failure: refusedDial, carbon: map[string]error{"dc3": serverError}, wantFails: 1}, + {name: "another_group_has_data", failure: refusedDial, data: true, wantFails: 1}, + } { + for _, window := range []time.Duration{0, 400 * time.Millisecond} { + t.Run(fmt.Sprintf("%s/retry_%v", tc.name, window), func(t *testing.T) { + fastConnectRetryCancels(t, window) + if window == 0 { + connectRetry.Store(nil) + } + // carbon: the one-store dc1 group plus three stores each in dc3 and dc2. + failed, _ := retryStoreIn("dc1", "carbon", 1<<30, tc.failure, nil) + backends := []Backend{failed} + for _, dc := range []string{"dc3", "dc2"} { + for i := range 3 { + var answer error = types.ErrMetricsNotFound + if tc.data && dc == "dc2" && i == 0 { + answer = nil + } + refusals, failure := int32(0), error(nil) + if err := tc.carbon[dc]; errors.Is(err, syscall.ECONNREFUSED) { + refusals, failure = 1<<30, err + } else if err != nil { + answer = err + } + b, _ := retryStoreIn(dc, "carbon", refusals, failure, answer) + backends = append(backends, b) + } + } + // 87 more stores in 29 other clusters, one per DC, all not-found. + for c := range 29 { + for _, dc := range []string{"dc1", "dc3", "dc2"} { + b, _ := retryStoreIn(dc, fmt.Sprintf("other-%d", c), 0, nil, types.ErrMetricsNotFound) + backends = append(backends, b) + } + } + if len(backends) != 94 { + t.Fatalf("%d backends", len(backends)) + } + start := time.Now() + fetchBoth(t, context.Background(), backends, "*", func(name string, metrics []types.Metric, errs []error) { + if (len(metrics) != 0) != tc.data || len(errs) != 94-len(metrics) { + t.Fatalf("%s: %d metrics, %d errors", name, len(metrics), len(errs)) + } + fails, others := 0, 0 + for _, err := range errs { + var u unreachableError + var notFound types.ErrNotFound + switch { + case errors.As(err, &u): + t.Fatalf("%s: internal retry marker leaked: %v", name, err) + case connectFailure(err): + fails++ + case !errors.As(err, ¬Found): + others++ + } + } + if fails != tc.wantFails { + t.Fatalf("%s: %d connect failures returned, want %d: %v", name, fails, tc.wantFails, errs) + } + if (others != 0) != (tc.carbon["dc3"] == serverError) { + t.Fatalf("%s: %d other errors: %v", name, others, errs) + } + }) + if tc.wantFails == 0 && time.Since(start) > 200*time.Millisecond { + t.Fatalf("waited %v although every other carbon group answered not-found", time.Since(start)) + } + }) + } + } +} diff --git a/pkg/backend/net/warm_dialer.go b/pkg/backend/net/warm_dialer.go new file mode 100644 index 00000000..1a9841c3 --- /dev/null +++ b/pkg/backend/net/warm_dialer.go @@ -0,0 +1,164 @@ +package net + +import ( + "context" + "net" + "sync" + "time" +) + +// DialFunc dials a network connection, like net.Dialer.DialContext. +type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// WarmDialer hands out backend connections that were dialed before the +// process started serving. +// +// A release reload (SIGUSR2, gracehttp re-exec) starts the new process with +// an empty connection pool and moves full traffic onto it at once: every +// render dials, thousands of dials to the stores start within a second, and +// with the 200 ms connect timeout a slow SYN round turns into render 500s. +// The old process keeps serving until the new one calls gracehttp.Serve, so +// dialing ahead of that costs nothing and the new process takes over with a +// warm pool. +type WarmDialer struct { + dial DialFunc + ttl time.Duration + + mu sync.Mutex + warm map[string][]warmConn +} + +type warmConn struct { + net.Conn + dialed time.Time +} + +// NewWarmDialer returns a dialer that serves prewarmed connections first and +// falls back to dial. A prewarmed connection older than ttl is never handed +// out, so a backend's idle timeout cannot have closed it. +func NewWarmDialer(dial DialFunc, ttl time.Duration) *WarmDialer { + return &WarmDialer{dial: dial, ttl: ttl, warm: make(map[string][]warmConn)} +} + +// DialContext returns a prewarmed connection to addr if one is left, and +// dials otherwise. +func (d *WarmDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + if network == "tcp" { + if c := d.take(addr); c != nil { + return c, nil + } + } + return d.dial(ctx, network, addr) +} + +func (d *WarmDialer) take(addr string) net.Conn { + d.mu.Lock() + defer d.mu.Unlock() + conns := d.warm[addr] + for len(conns) > 0 { + c := conns[len(conns)-1] + conns[len(conns)-1] = warmConn{} + conns = conns[:len(conns)-1] + if time.Since(c.dialed) < d.ttl { + d.warm[addr] = conns + return c.Conn + } + _ = c.Close() + } + delete(d.warm, addr) + return nil +} + +// Idle reports how many prewarmed connections are waiting to be handed out. +func (d *WarmDialer) Idle() int { + d.mu.Lock() + defer d.mu.Unlock() + n := 0 + for _, conns := range d.warm { + n += len(conns) + } + return n +} + +// PrewarmResult summarises one Prewarm call. +type PrewarmResult struct { + Dialed, Failed int + Elapsed time.Duration +} + +// Prewarm dials perAddr connections to every address with dial, at most +// parallel at a time in total, and keeps them for DialContext. Dials go round +// by round over the addresses, so no backend sees a burst of more than a few +// concurrent connects. It returns once every dial finished or ctx is done; +// connections still being dialed then are closed. Connections nobody took +// are closed after the dialer's ttl. +func (d *WarmDialer) Prewarm(ctx context.Context, dial DialFunc, addrs []string, perAddr, parallel int) PrewarmResult { + start := time.Now() + var res PrewarmResult + if perAddr <= 0 || len(addrs) == 0 { + return res + } + parallel = max(parallel, 1) + + var ( + mu sync.Mutex + wg sync.WaitGroup + sem = make(chan struct{}, parallel) + ) +dialing: + for round := 0; round < perAddr; round++ { + for _, addr := range addrs { + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break dialing + } + wg.Add(1) + go func(addr string) { + defer wg.Done() + defer func() { <-sem }() + c, err := dial(ctx, "tcp", addr) + mu.Lock() + defer mu.Unlock() + if err != nil { + res.Failed++ + return + } + if ctx.Err() != nil { + _ = c.Close() + res.Failed++ + return + } + res.Dialed++ + d.mu.Lock() + d.warm[addr] = append(d.warm[addr], warmConn{Conn: c, dialed: time.Now()}) + d.mu.Unlock() + }(addr) + } + } + wg.Wait() + time.AfterFunc(d.ttl, d.expire) + res.Elapsed = time.Since(start) + return res +} + +// expire closes prewarmed connections that outlived the ttl. +func (d *WarmDialer) expire() { + d.mu.Lock() + defer d.mu.Unlock() + for addr, conns := range d.warm { + keep := conns[:0] + for _, c := range conns { + if time.Since(c.dialed) < d.ttl { + keep = append(keep, c) + continue + } + _ = c.Close() + } + if len(keep) == 0 { + delete(d.warm, addr) + continue + } + d.warm[addr] = keep + } +} diff --git a/pkg/backend/net/warm_dialer_test.go b/pkg/backend/net/warm_dialer_test.go new file mode 100644 index 00000000..3a65c2dd --- /dev/null +++ b/pkg/backend/net/warm_dialer_test.go @@ -0,0 +1,159 @@ +package net + +import ( + "context" + "errors" + "io" + "net" + "sync/atomic" + "testing" + "time" +) + +// countingListener accepts and holds connections, counting them. +func countingListener(t *testing.T) (string, *atomic.Int64, chan net.Conn) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var n atomic.Int64 + conns := make(chan net.Conn, 128) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + n.Add(1) + conns <- c + } + }() + t.Cleanup(func() { + _ = ln.Close() + close(conns) + for c := range conns { + _ = c.Close() + } + }) + return ln.Addr().String(), &n, conns +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestWarmDialerHandsOutPrewarmedConnsThenDials(t *testing.T) { + addr, accepted, _ := countingListener(t) + base := (&net.Dialer{Timeout: time.Second}).DialContext + d := NewWarmDialer(base, time.Minute) + + res := d.Prewarm(context.Background(), base, []string{addr}, 3, 2) + if res.Dialed != 3 || res.Failed != 0 { + t.Fatalf("prewarm = %+v, want 3 dialed, 0 failed", res) + } + waitFor(t, "3 accepted", func() bool { return accepted.Load() == 3 }) + if d.Idle() != 3 { + t.Fatalf("idle = %d, want 3", d.Idle()) + } + + for i := 0; i < 3; i++ { + c, err := d.DialContext(context.Background(), "tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + } + if got := accepted.Load(); got != 3 { + t.Fatalf("prewarmed conns were not reused: %d accepts after 3 dials", got) + } + c, err := d.DialContext(context.Background(), "tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + waitFor(t, "fallback dial", func() bool { return accepted.Load() == 4 }) + if d.Idle() != 0 { + t.Fatalf("idle = %d, want 0", d.Idle()) + } +} + +func TestWarmDialerExpiresUnusedConns(t *testing.T) { + addr, accepted, conns := countingListener(t) + base := (&net.Dialer{Timeout: time.Second}).DialContext + d := NewWarmDialer(base, 50*time.Millisecond) + + if res := d.Prewarm(context.Background(), base, []string{addr}, 1, 1); res.Dialed != 1 { + t.Fatalf("prewarm = %+v, want 1 dialed", res) + } + server := <-conns + // The expiry closes the client side; the server then reads EOF. + if err := server.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + if _, err := server.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Fatalf("server read = %v, want EOF after expiry", err) + } + if d.Idle() != 0 { + t.Fatalf("idle = %d after expiry, want 0", d.Idle()) + } + // An expired dialer dials afresh. + c, err := d.DialContext(context.Background(), "tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + waitFor(t, "fresh dial", func() bool { return accepted.Load() == 2 }) +} + +func TestWarmDialerSkipsStaleConn(t *testing.T) { + addr, accepted, _ := countingListener(t) + base := (&net.Dialer{Timeout: time.Second}).DialContext + d := NewWarmDialer(base, time.Hour) + if res := d.Prewarm(context.Background(), base, []string{addr}, 1, 1); res.Dialed != 1 { + t.Fatalf("prewarm = %+v", res) + } + d.ttl = 0 // the prewarmed conn is now older than the ttl + c, err := d.DialContext(context.Background(), "tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + waitFor(t, "fresh dial instead of stale conn", func() bool { return accepted.Load() == 2 }) +} + +func TestWarmDialerPrewarmCountsFailuresAndHonoursContext(t *testing.T) { + refused := func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("connection refused") + } + d := NewWarmDialer(refused, time.Minute) + if res := d.Prewarm(context.Background(), refused, []string{"a:1", "b:1"}, 2, 4); res.Dialed != 0 || res.Failed != 4 { + t.Fatalf("prewarm = %+v, want 0 dialed, 4 failed", res) + } + + block := func(ctx context.Context, _, _ string) (net.Conn, error) { + <-ctx.Done() + return nil, ctx.Err() + } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + start := time.Now() + res := d.Prewarm(ctx, block, []string{"a:1", "b:1", "c:1"}, 100, 2) + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("prewarm ignored its context: took %v", elapsed) + } + if res.Dialed != 0 || d.Idle() != 0 { + t.Fatalf("prewarm = %+v, idle %d; want nothing kept", res, d.Idle()) + } + + if res := d.Prewarm(context.Background(), refused, []string{"a:1"}, 0, 4); res != (PrewarmResult{}) { + t.Fatalf("disabled prewarm = %+v, want zero", res) + } +} diff --git a/pkg/backend/render_batch.go b/pkg/backend/render_batch.go index a49d32ea..a41d392c 100644 --- a/pkg/backend/render_batch.go +++ b/pkg/backend/render_batch.go @@ -86,7 +86,7 @@ func (b *Backend) renderOne(r *renderReq) { res, err := b.renderMetrics(r.Ctx, r.RenderRequest) t.ObserveDuration() if err != nil { - r.Errors <- err + r.Errors <- b.renderError(err) return } b.addSourceMetaToMetrics(res) @@ -116,6 +116,7 @@ func (b *Backend) renderBatch(batch []*renderReq) { } // Each original render keeps its own replica fan-in. In particular, a // slow or failed replica must not trigger another whole-query retry. + err = b.renderError(err) for _, r := range batch { r.Errors <- err } diff --git a/pkg/backend/render_fanin.go b/pkg/backend/render_fanin.go new file mode 100644 index 00000000..dd862556 --- /dev/null +++ b/pkg/backend/render_fanin.go @@ -0,0 +1,159 @@ +package backend + +import ( + "errors" + + "github.com/bookingcom/carbonapi/pkg/types" +) + +// Keep replica replies together until errors have been classified. Only the +// backends actually selected for this request can establish that a group answered. +type renderGroup struct { + n int + cluster string + results chan []types.Metric + errors chan error + answered bool + data bool // a member returned at least one series + errs []error +} + +func newRenderGroups(backends []Backend) ([]renderGroup, []int) { + groups := make([]renderGroup, 0) + membership := make([]int, len(backends)) + byGroup := make(map[[2]string]int) + for i, b := range backends { + _, cluster, dc := b.BackendInfo() + if cluster == "" { + // Ungrouped backends share channels, but never recover each other's errors. + dc = "" + } + key := [2]string{dc, cluster} + g, ok := byGroup[key] + if !ok { + g = len(groups) + byGroup[key] = g + groups = append(groups, renderGroup{cluster: cluster}) + } + groups[g].n++ + membership[i] = g + } + for i := range groups { + groups[i].results = make(chan []types.Metric, groups[i].n) + groups[i].errors = make(chan error, groups[i].n) + } + return groups, membership +} + +// next waits for one member's reply. +func (g *renderGroup) next() ([]types.Metric, error) { + return nextReply(g.results, g.errors) +} + +func nextReply(results <-chan []types.Metric, errs <-chan error) ([]types.Metric, error) { + select { + case metrics := <-results: + return metrics, nil + case err := <-errs: + return nil, err + } +} + +// take records one member's reply and reports whether it was an answer: data +// or the backend's own not-found. +func (g *renderGroup) take(metrics []types.Metric, err error, messages [][]types.Metric) ([][]types.Metric, bool) { + if err != nil { + g.errs = append(g.errs, err) + var notFound types.ErrNotFound + if !errors.As(err, ¬Found) { + return messages, false + } + } else { + messages = append(messages, metrics) + g.data = g.data || len(metrics) != 0 + } + g.answered = true + return messages, true +} + +// receive collects n replies and returns how many of them were answers. +func (g *renderGroup) receive(n int, messages [][]types.Metric) ([][]types.Metric, int) { + answers := 0 + for range n { + metrics, err := g.next() + var answer bool + if messages, answer = g.take(metrics, err, messages); answer { + answers++ + } + } + return messages, answers +} + +// unrecovered reports whether this group's errors still fail the fetch. +func (g *renderGroup) unrecovered() bool { return g.cluster == "" || !g.answered } + +func collectRenderGroups(f *renderFetch) { + for i := range f.groups { + f.messages, _ = f.groups[i].receive(f.groups[i].n, f.messages) + } +} + +// renderGroupErrors returns the errors that still decide the fetch: an +// answered group's errors become not-found, and so do the connect failures of +// a group whose cluster's other replica groups all answered not-found +// (clusterNotFound). Every other error is returned as the backend reported it. +func renderGroupErrors(groups []renderGroup) []error { + var errs []error + for gi := range groups { + g := &groups[gi] + notFound := !g.unrecovered() + unreachableNotFound := !notFound && clusterNotFound(groups, gi) + for _, err := range g.errs { + switch { + case notFound, unreachableNotFound && connectFailure(err): + err = types.ErrMetricsNotFound + default: + if u, ok := err.(unreachableError); ok { + err = u.error + } + } + errs = append(errs, err) + } + } + return errs +} + +// clusterNotFound reports whether the cluster of group gi has at least one +// other replica group and every one of them gave a definitive not-found: some +// member answered, and none returned data. Replica groups of one cluster hold +// the same names, so that is what gi's refused or timed-out stores would have +// answered too; their connect failures then count as the cluster's not-found +// instead of failing the fetch. The fe-112 canary needed this: the literal +// target "*" (as in sumSeries(x, *)) got not-found from 93 stores and a refusal +// from the one store of the dc1 carbon group, and the required-fetch rule (not-found only +// when every error is a not-found) turned the whole render into HTTP 500. +// +// The connect failure stays an error, so the required-fetch rule still fails a fetch no store has +// data for, when gi is its cluster's only group, when any other group is +// unreachable too or failed (for example HTTP 5xx), or when another group +// returned data (the render then has that data anyway). Server errors and +// failures after the backend accepted the request are never converted, and +// ungrouped backends never recover each other. +func clusterNotFound(groups []renderGroup, gi int) bool { + cluster := groups[gi].cluster + if cluster == "" { + return false + } + others := 0 + for oi := range groups { + o := &groups[oi] + if oi == gi || o.cluster != cluster { + continue + } + if !o.answered || o.data { + return false + } + others++ + } + return others != 0 +} diff --git a/pkg/backend/render_fanin_test.go b/pkg/backend/render_fanin_test.go new file mode 100644 index 00000000..5bea1106 --- /dev/null +++ b/pkg/backend/render_fanin_test.go @@ -0,0 +1,91 @@ +package backend + +import ( + "context" + "errors" + "syscall" + "testing" + "time" + + "github.com/bookingcom/carbonapi/pkg/cfg" + "github.com/bookingcom/carbonapi/pkg/types" + "go.uber.org/zap" +) + +func TestRenderGroupAnswersStayPerTarget(t *testing.T) { + for _, grouped := range []bool{false, true} { + name := "ordinary" + if grouped { + name = "coalesced" + } + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var backends []Backend + for replica := range 3 { + b := coalesceBackend(func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + if replica == 0 { + return nil, syscall.ECONNREFUSED + } + for _, path := range r.Targets { + if path == "failed" { + return nil, syscall.ECONNREFUSED + } + } + var metrics []types.Metric + for _, path := range r.Targets { + if path == "data" { + metrics = append(metrics, coalesceMetrics([]string{path})...) + } + } + if len(metrics) == 0 { + return nil, types.ErrMetricsNotFound + } + return metrics, nil + }, 4) + b.Proc() + backends = append(backends, b) + } + paths := []string{"missing", "failed", "data"} + requests := make([]types.RenderRequest, len(paths)) + choices := make([][]Backend, len(paths)) + for i, path := range paths { + requests[i] = types.NewRenderRequest([]string{path}, 1, 4) + choices[i] = backends + } + var replies []GroupedRenderReply + if grouped { + replies = RendersGrouped(ctx, choices, requests, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + } else { + for _, request := range requests { + metrics, stats, errs := Renders(ctx, backends, request, cfg.RenderReplicaMismatchConfig{}, zap.NewNop()) + replies = append(replies, GroupedRenderReply{Metrics: metrics, Stats: stats, Errors: errs}) + } + } + for i, reply := range replies { + wantErrors := 3 + if paths[i] == "data" { + wantErrors = 1 + if len(reply.Metrics) != 1 || reply.Metrics[0].Name != "data" || reply.Metrics[0].Values[1] != 42 { + t.Fatalf("data lost: %+v", reply) + } + } else if len(reply.Metrics) != 0 { + t.Fatalf("%s acquired another target's data: %+v", paths[i], reply) + } + if len(reply.Errors) != wantErrors { + t.Fatalf("%s errors = %v, want %d", paths[i], reply.Errors, wantErrors) + } + for _, err := range reply.Errors { + var notFound types.ErrNotFound + if paths[i] == "failed" { + if !errors.Is(err, syscall.ECONNREFUSED) { + t.Fatalf("entirely failed group recovered: %v", reply.Errors) + } + } else if !errors.As(err, ¬Found) { + t.Fatalf("%s retained replica failure: %v", paths[i], reply.Errors) + } + } + } + }) + } +} diff --git a/pkg/backend/rpc.go b/pkg/backend/rpc.go index eb23e463..95275652 100644 --- a/pkg/backend/rpc.go +++ b/pkg/backend/rpc.go @@ -21,25 +21,17 @@ func Renders(ctx context.Context, backends []Backend, request types.RenderReques return nil, types.MetricRenderStats{}, nil } - msgCh := make(chan []types.Metric, len(backends)) - errCh := make(chan error, len(backends)) - for _, backend := range backends { - backend.SendRender(ctx, request, msgCh, errCh) - } - - msgs := make([][]types.Metric, 0, len(backends)) - errs := make([]error, 0, len(backends)) - for i := 0; i < len(backends); i++ { - select { - case msg := <-msgCh: - msgs = append(msgs, msg) - case err := <-errCh: - errs = append(errs, err) - } + groups, membership := newRenderGroups(backends) + for i, backend := range backends { + g := groups[membership[i]] + backend.SendRender(ctx, request, g.results, g.errors) } - metrics, stats := types.MergeMetrics(msgs, replicaMismatchConfig, lg) - return metrics, stats, errs + fetches := []renderFetch{{request: request, groups: groups}} + collectRenderFetches(fetches, lg) + retryUnreachable(ctx, fetches, lg) + metrics, stats := types.MergeMetrics(fetches[0].messages, replicaMismatchConfig, lg) + return metrics, stats, renderGroupErrors(groups) } // Infos makes Info calls to multiple backends. diff --git a/pkg/backend/slow_replica.go b/pkg/backend/slow_replica.go new file mode 100644 index 00000000..1b8231b1 --- /dev/null +++ b/pkg/backend/slow_replica.go @@ -0,0 +1,236 @@ +package backend + +import ( + "errors" + "sync/atomic" + "time" + + "github.com/bookingcom/carbonapi/pkg/types" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +// SlowReplicaGrace bounds how long a render waits for a replica group that is +// still silent after every other replica group of the same cluster gave a +// definitive answer for the target (data or not-found) and at least one of +// them returned data, as a store warming up after a restart does: it accepts +// connections but answers only at the backend timeout. +type SlowReplicaGrace struct { + // Grace is the extra wait granted to the slow group. Zero waits for every + // backend as before. + Grace time.Duration + // Cancelled counts backend fetches no longer awaited after the grace. + Cancelled prometheus.Counter +} + +var slowReplicaGrace atomic.Pointer[SlowReplicaGrace] + +// SetSlowReplicaGrace replaces the process-wide slow replica policy. +func SetSlowReplicaGrace(s SlowReplicaGrace) { slowReplicaGrace.Store(&s) } + +// errSlowReplica is the error of a fetch abandoned after the grace. Like any +// other failure, it stays an error only when no member of its own group +// answered; the render already has the other groups' data. +var errSlowReplica = errors.New("render not awaited: every other replica group answered, one with data, and this backend did not answer within the slow replica grace") + +// cuttable reports whether some cluster of this fetch has several replica +// groups, so that one of them can ever be left behind. +func (f *renderFetch) cuttable() bool { + seen := make(map[string]struct{}, len(f.groups)) + for i := range f.groups { + c := f.groups[i].cluster + if c == "" { + continue + } + if _, ok := seen[c]; ok { + return true + } + seen[c] = struct{}{} + } + return false +} + +// coveredByOthers reports whether the slow group g may be cut after the grace. +// That needs both: +// +// - every other replica group of g's cluster has received all its replies +// and gave a definitive answer: data or the backend's own not-found. A +// group that is still pending, or only refused or failed, says nothing +// about the target, so g is awaited. +// - at least one of those groups returned data. If every other group said +// not-found, g may be the only holder of the series (replicas can diverge, +// for example for names created while a DC was down), so g is awaited in +// full. This is where the slow grace differs from the connect retry +// (connect_retry.go), which also gives up on a not-found cover: a refused +// store cannot answer at all until it is back, while a slow one is +// reachable and will deliver its copy. +func (f *renderFetch) coveredByOthers(gi int, pending []int) bool { + g := &f.groups[gi] + data := false + for oi := range f.groups { + o := &f.groups[oi] + if oi == gi || o.cluster != g.cluster { + continue + } + if pending[oi] != 0 || !o.answered { + return false + } + data = data || o.data + } + return data +} + +type fanInReply struct { + fetch, group int + metrics []types.Metric + err error +} + +// collectRenderFetches waits for every backend reply of every fetch, except +// that a replica group still silent policy.Grace after all other groups of its +// cluster answered, at least one with data (coveredByOthers), is no longer +// awaited. Its outstanding fetches get +// errSlowReplica; their late replies are dropped. The in-flight requests are +// not cancelled: coalesced batches share one context across targets, and +// another target may still need the same backend. +func collectRenderFetches(fetches []renderFetch, lg *zap.Logger) { + policy := slowReplicaGrace.Load() + if policy == nil || policy.Grace <= 0 { + for i := range fetches { + collectRenderGroups(&fetches[i]) + } + return + } + var watched []int + for i := range fetches { + if fetches[i].cuttable() { + watched = append(watched, i) + } + } + if len(watched) != 0 { + collectWithGrace(fetches, watched, policy, lg) + } + // Fetches that cannot be cut keep the ordinary fan-in. Their replies are + // buffered, so collecting them afterwards adds no wait. + for i := range fetches { + if !fetches[i].cuttable() { + collectRenderGroups(&fetches[i]) + } + } +} + +func collectWithGrace(fetches []renderFetch, watched []int, policy *SlowReplicaGrace, lg *zap.Logger) { + pending := make(map[int][]int, len(watched)) // fetch -> group -> replies awaited + cutAt := make(map[int][]time.Time, len(watched)) // fetch -> group -> grace deadline + byGroup := make(map[int][][][]types.Metric, len(watched)) + outstanding := 0 + for _, fi := range watched { + f := &fetches[fi] + pending[fi] = make([]int, len(f.groups)) + cutAt[fi] = make([]time.Time, len(f.groups)) + byGroup[fi] = make([][][]types.Metric, len(f.groups)) + for gi := range f.groups { + pending[fi][gi] = f.groups[gi].n + outstanding += f.groups[gi].n + } + } + // Buffered for every reply, so forwarders of abandoned groups never block. + replies := make(chan fanInReply, outstanding) + for _, fi := range watched { + for gi := range fetches[fi].groups { + g := &fetches[fi].groups[gi] + go func(fi, gi, n int, results <-chan []types.Metric, errs <-chan error) { + for range n { + metrics, err := nextReply(results, errs) + replies <- fanInReply{fi, gi, metrics, err} + } + }(fi, gi, g.n, g.results, g.errors) + } + } + + var timer *time.Timer + var next time.Time + arm := func() { + next = time.Time{} + for _, fi := range watched { + for gi, at := range cutAt[fi] { + if !at.IsZero() && pending[fi][gi] != 0 && (next.IsZero() || at.Before(next)) { + next = at + } + } + } + if timer != nil { + timer.Stop() + timer = nil + } + if !next.IsZero() { + timer = time.NewTimer(time.Until(next)) + } + } + defer func() { + if timer != nil { + timer.Stop() + } + }() + + for outstanding > 0 { + var expired <-chan time.Time + if timer != nil { + expired = timer.C + } + select { + case r := <-replies: + if pending[r.fetch][r.group] == 0 { + continue // abandoned group + } + pending[r.fetch][r.group]-- + outstanding-- + f := &fetches[r.fetch] + byGroup[r.fetch][r.group], _ = f.groups[r.group].take(r.metrics, r.err, byGroup[r.fetch][r.group]) + // Deadlines only appear; a group's coverage never goes away. + changed := false + for gi := range f.groups { + if pending[r.fetch][gi] != 0 && cutAt[r.fetch][gi].IsZero() && f.groups[gi].cluster != "" && f.coveredByOthers(gi, pending[r.fetch]) { + cutAt[r.fetch][gi] = time.Now().Add(policy.Grace) + changed = true + } + } + if changed { + arm() + } + case <-expired: + now := time.Now() + for _, fi := range watched { + f := &fetches[fi] + for gi := range f.groups { + n := pending[fi][gi] + if n == 0 || cutAt[fi][gi].IsZero() || cutAt[fi][gi].After(now) { + continue + } + g := &f.groups[gi] + for range n { + g.errs = append(g.errs, errSlowReplica) + } + pending[fi][gi] = 0 + outstanding -= n + if policy.Cancelled != nil { + policy.Cancelled.Add(float64(n)) + } + lg.Debug("render stopped waiting for a slow replica group: every other group answered, one with data", + zap.Strings("targets", f.request.Targets), + zap.String("cluster", g.cluster), + zap.Int("fetches", n), + zap.Duration("grace", policy.Grace)) + } + } + timer = nil + arm() + } + } + // Keep the ordinary fan-in's message order: by group, then by arrival. + for _, fi := range watched { + for _, msgs := range byGroup[fi] { + fetches[fi].messages = append(fetches[fi].messages, msgs...) + } + } +} diff --git a/pkg/backend/slow_replica_test.go b/pkg/backend/slow_replica_test.go new file mode 100644 index 00000000..58b4059c --- /dev/null +++ b/pkg/backend/slow_replica_test.go @@ -0,0 +1,376 @@ +package backend + +import ( + "context" + "errors" + "syscall" + "testing" + "time" + + "github.com/bookingcom/carbonapi/pkg/cfg" + "github.com/bookingcom/carbonapi/pkg/types" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +func slowGrace(t *testing.T, grace time.Duration) prometheus.Counter { + t.Helper() + cuts := prometheus.NewCounter(prometheus.CounterOpts{Name: "test_slow_replica_cuts"}) + SetSlowReplicaGrace(SlowReplicaGrace{Grace: grace, Cancelled: cuts}) + t.Cleanup(func() { slowReplicaGrace.Store(nil) }) + return cuts +} + +// stallingStore accepts the render, as a warming-up store does, and answers +// with data after stall (or when the test ends). +func stallingStore(t *testing.T, dc, cluster string, stall time.Duration) Backend { + t.Helper() + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + b, _ := retryStoreFunc(dc, cluster, 0, nil, func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + select { + case <-release: + case <-time.After(stall): + } + return coalesceMetrics(r.Targets), nil + }) + return b +} + +// renderTimed runs the ordinary and the grouped fan-in and reports each one's +// replies and duration. +func renderTimed(t *testing.T, backends []Backend, mismatch cfg.RenderReplicaMismatchConfig, + check func(name string, elapsed time.Duration, replies []GroupedRenderReply)) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + request := types.NewRenderRequest([]string{"m"}, 1, 4) + start := time.Now() + reply := GroupedRenderReply{} + reply.Metrics, reply.Stats, reply.Errors = Renders(ctx, backends, request, mismatch, zap.NewNop()) + check("ordinary", time.Since(start), []GroupedRenderReply{reply}) + start = time.Now() + replies := RendersGrouped(ctx, [][]Backend{backends, backends}, + []types.RenderRequest{request, types.NewRenderRequest([]string{"m.2"}, 1, 4)}, mismatch, zap.NewNop()) + check("grouped", time.Since(start), replies) +} + +func assertFullData(t *testing.T, name string, reply GroupedRenderReply) { + t.Helper() + if len(reply.Metrics) != 1 || reply.Metrics[0].Values[1] != 42 || reply.Metrics[0].IsAbsent[1] { + t.Fatalf("%s: metrics %+v, want full data", name, reply.Metrics) + } +} + +// Two DCs answer at once and the third accepts and stalls for 10 s, as a +// restarting store does: the render returns the merged data after the grace, +// and the abandoned fetch is a plain error that no longer fails the render. +func TestSlowReplicaCutAfterGrace(t *testing.T) { + const grace = time.Second + for _, mode := range []cfg.ReplicaMatchMode{cfg.ReplicaMatchModeNormal, cfg.ReplicaMatchModeMajority} { + t.Run(string(mode), func(t *testing.T) { + cuts := slowGrace(t, grace) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fra := stallingStore(t, "dc2", "cy", 10*time.Second) + backends := []Backend{fra, ams, lhr} + renderTimed(t, backends, cfg.RenderReplicaMismatchConfig{RenderReplicaMatchMode: mode}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < grace-50*time.Millisecond || elapsed > 1500*time.Millisecond { + t.Fatalf("%s: took %v, want the %v grace", name, elapsed, grace) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 1 || !errors.Is(reply.Errors[0], errSlowReplica) { + t.Fatalf("%s: errs %v, want the abandoned fetch", name, reply.Errors) + } + // renderErrorsFanIn: fewer errors than backends never fails a render with data. + if len(reply.Errors) >= len(backends) { + t.Fatalf("%s: %d errors for %d backends would fail the render", name, len(reply.Errors), len(backends)) + } + } + }) + // One ordinary and two grouped fetches. + if got := counterValue(cuts); got != 3 { + t.Fatalf("cancelled %v slow fetches, want 3", got) + } + }) + } +} + +// A slow store whose group-mates answered not-found leaves its group answered: +// the abandoned fetch is recovered as not-found, like the group's other errors. +func TestSlowReplicaCutInAnsweredGroup(t *testing.T) { + cuts := slowGrace(t, 100*time.Millisecond) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fraPeer, _ := retryStoreIn("dc2", "cy", 0, nil, types.ErrMetricsNotFound) + fra := stallingStore(t, "dc2", "cy", 10*time.Second) + renderTimed(t, []Backend{fraPeer, fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed > 600*time.Millisecond { + t.Fatalf("%s: took %v", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + var notFound types.ErrNotFound + for _, err := range reply.Errors { + if !errors.As(err, ¬Found) { + t.Fatalf("%s: errs %v, want only not-found", name, reply.Errors) + } + } + } + }) + if counterValue(cuts) != 3 { + t.Fatalf("cancelled %v slow fetches, want 3", counterValue(cuts)) + } +} + +// A not-found counts as a definitive answer once another group has data: the +// slow group is cut after the grace with one group holding the data and the +// other answering not-found. +func TestSlowReplicaCutWithNotFoundGroup(t *testing.T) { + const grace = 100 * time.Millisecond + cuts := slowGrace(t, grace) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, types.ErrMetricsNotFound) + fra := stallingStore(t, "dc2", "cy", 10*time.Second) + renderTimed(t, []Backend{fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < grace-20*time.Millisecond || elapsed > grace+500*time.Millisecond { + t.Fatalf("%s: took %v, want the %v grace", name, elapsed, grace) + } + for _, reply := range replies { + assertFullData(t, name, reply) + var slow bool + for _, err := range reply.Errors { + slow = slow || errors.Is(err, errSlowReplica) + } + if len(reply.Errors) != 2 || !slow { + t.Fatalf("%s: errs %v, want the not-found and the abandoned fetch", name, reply.Errors) + } + } + }) + if counterValue(cuts) != 3 { + t.Fatalf("cancelled %v slow fetches, want 3", counterValue(cuts)) + } +} + +// A slow group is awaited in full unless every other group of its cluster gave +// a definitive answer and one of them returned data. If all the others said +// not-found, it may hold the only copy of the series. +func TestSlowReplicaAwaitedWithoutFullCoverage(t *testing.T) { + const stall = 300 * time.Millisecond + for _, tc := range []struct { + name string + others func() []Backend + }{ + {name: "only_copy", others: func() []Backend { + ams, _ := retryStoreIn("dc1", "cy", 0, nil, types.ErrMetricsNotFound) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, types.ErrMetricsNotFound) + return []Backend{ams, lhr} + }}, + {name: "other_group_server_error", others: func() []Backend { + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, errors.New("upstream HTTP 503")) + return []Backend{ams, lhr} + }}, + {name: "other_cluster_data", others: func() []Backend { + ams, _ := retryStoreIn("dc1", "carbon", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "carbon", 0, nil, nil) + return []Backend{ams, lhr} + }}, + {name: "ungrouped", others: func() []Backend { + a, _ := retryStoreIn("dc1", "", 0, nil, nil) + b, _ := retryStoreIn("dc3", "", 0, nil, nil) + return []Backend{a, b} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + cuts := slowGrace(t, 20*time.Millisecond) + cluster := "cy" + if tc.name == "ungrouped" { + cluster = "" + } + fra := stallingStore(t, "dc2", cluster, stall) + renderTimed(t, append([]Backend{fra}, tc.others()...), cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < stall-20*time.Millisecond { + t.Fatalf("%s: returned after %v without the slow group", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + for _, err := range reply.Errors { + if errors.Is(err, errSlowReplica) { + t.Fatalf("%s: slow group cut: %v", name, reply.Errors) + } + } + } + }) + if counterValue(cuts) != 0 { + t.Fatalf("cancelled %v slow fetches, want 0", counterValue(cuts)) + } + }) + } +} + +// Healthy groups are unaffected: all fast is unchanged, and a group that is +// slower but answers within the grace still heals the merged series. +func TestSlowReplicaWithinGraceStillMerged(t *testing.T) { + cuts := slowGrace(t, time.Second) + holey := func(dc string, delay time.Duration, value float64, absent bool) Backend { + b, _ := retryStoreFunc(dc, "cy", 0, nil, func(_ context.Context, r types.RenderRequest) ([]types.Metric, error) { + time.Sleep(delay) + ms := coalesceMetrics(r.Targets) + for i := range ms { + ms[i].Values[2], ms[i].IsAbsent[2] = value, absent + } + return ms, nil + }) + return b + } + for _, tc := range []struct { + name string + delay time.Duration + max time.Duration + }{ + {name: "all_fast", max: 200 * time.Millisecond}, + {name: "slower_within_grace", delay: 150 * time.Millisecond, max: 600 * time.Millisecond}, + } { + t.Run(tc.name, func(t *testing.T) { + backends := []Backend{holey("dc1", 0, 0, true), holey("dc3", 0, 0, true), holey("dc2", tc.delay, 7, false)} + renderTimed(t, backends, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < tc.delay || elapsed > tc.max { + t.Fatalf("%s: took %v", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 0 || reply.Metrics[0].IsAbsent[2] || reply.Metrics[0].Values[2] != 7 { + t.Fatalf("%s: %+v, want fra's point healed in and no errors", name, reply) + } + } + }) + }) + } + if counterValue(cuts) != 0 { + t.Fatalf("cancelled %v fetches, want 0", counterValue(cuts)) + } +} + +// Grace 0, or no policy, waits for the slow store as before. +func TestSlowReplicaGraceZeroWaits(t *testing.T) { + const stall = 300 * time.Millisecond + for _, name := range []string{"zero", "unset"} { + t.Run(name, func(t *testing.T) { + var cuts prometheus.Counter + if name == "zero" { + cuts = slowGrace(t, 0) + } + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fra := stallingStore(t, "dc2", "cy", stall) + renderTimed(t, []Backend{fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < stall-20*time.Millisecond { + t.Fatalf("%s: returned after %v without the slow store", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 0 { + t.Fatalf("%s: errs %v", name, reply.Errors) + } + } + }) + if cuts != nil && counterValue(cuts) != 0 { + t.Fatalf("cancelled %v fetches with grace 0", counterValue(cuts)) + } + }) + } +} + +// The refused-store path (connect-retry cancellation) and the slow-store grace compose: a refusal is +// cancelled at once, a stall after the grace, and neither counts the other. +func TestSlowReplicaWithRefusedStore(t *testing.T) { + const grace = 200 * time.Millisecond + t.Run("refused_only", func(t *testing.T) { + _, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + cuts := slowGrace(t, grace) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + fra, _ := retryStoreIn("dc2", "cy", 1<<30, refusedDial, nil) + renderTimed(t, []Backend{fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed > 150*time.Millisecond { + t.Fatalf("%s: took %v for a refused store", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + } + }) + if counterValue(cancels) != 3 || counterValue(cuts) != 0 { + t.Fatalf("connect cancels %v slow cuts %v, want 3/0", counterValue(cancels), counterValue(cuts)) + } + }) + t.Run("refused_and_stalled_in_one_group", func(t *testing.T) { + retries, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + cuts := slowGrace(t, grace) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 0, nil, nil) + refused, _ := retryStoreIn("dc2", "cy", 1<<30, refusedDial, nil) + stalled := stallingStore(t, "dc2", "cy", 10*time.Second) + renderTimed(t, []Backend{refused, stalled, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < grace-20*time.Millisecond || elapsed > grace+500*time.Millisecond { + t.Fatalf("%s: took %v, want the %v grace", name, elapsed, grace) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 2 { + t.Fatalf("%s: errs %v, want the refusal and the abandoned fetch", name, reply.Errors) + } + var refusal, slow bool + for _, err := range reply.Errors { + var u unreachableError + if errors.As(err, &u) { + t.Fatalf("%s: internal retry marker leaked", name) + } + refusal = refusal || errors.Is(err, syscall.ECONNREFUSED) + slow = slow || errors.Is(err, errSlowReplica) + } + if !refusal || !slow { + t.Fatalf("%s: errs %v", name, reply.Errors) + } + } + }) + if counterValue(retries) != 0 || counterValue(cancels) != 3 || counterValue(cuts) != 3 { + t.Fatalf("retries %v connect cancels %v slow cuts %v, want 0/3/3", counterValue(retries), counterValue(cancels), counterValue(cuts)) + } + }) + // A refusing group gave no definitive answer, so the stalled group is not + // cut: two bad groups at once wait for the slow one, and then the refusal + // is cancelled. + t.Run("refused_group_and_stalled_group", func(t *testing.T) { + const stall = 300 * time.Millisecond + _, _, cancels := fastConnectRetryCancels(t, 5*time.Second) + cuts := slowGrace(t, 20*time.Millisecond) + ams, _ := retryStoreIn("dc1", "cy", 0, nil, nil) + lhr, _ := retryStoreIn("dc3", "cy", 1<<30, refusedDial, nil) + fra := stallingStore(t, "dc2", "cy", stall) + renderTimed(t, []Backend{fra, ams, lhr}, cfg.RenderReplicaMismatchConfig{}, + func(name string, elapsed time.Duration, replies []GroupedRenderReply) { + if elapsed < stall-20*time.Millisecond || elapsed > stall+500*time.Millisecond { + t.Fatalf("%s: took %v", name, elapsed) + } + for _, reply := range replies { + assertFullData(t, name, reply) + if len(reply.Errors) != 1 || !errors.Is(reply.Errors[0], syscall.ECONNREFUSED) { + t.Fatalf("%s: errs %v, want the refusal only", name, reply.Errors) + } + } + }) + if counterValue(cancels) != 3 || counterValue(cuts) != 0 { + t.Fatalf("connect cancels %v slow cuts %v, want 3/0", counterValue(cancels), counterValue(cuts)) + } + }) +} diff --git a/pkg/backend/stable_render.go b/pkg/backend/stable_render.go index a4c9bedd..005821c3 100644 --- a/pkg/backend/stable_render.go +++ b/pkg/backend/stable_render.go @@ -33,38 +33,37 @@ func RendersGrouped(ctx context.Context, choices [][]Backend, requests []types.R calls []renderDispatch } groups := make(map[*requestQueue[*renderReq]]*group) - results := make([]chan []types.Metric, len(requests)) - errors := make([]chan error, len(requests)) + fetches := make([]renderFetch, len(requests)) for i, request := range requests { - results[i] = make(chan []types.Metric, len(choices[i])) - errors[i] = make(chan error, len(choices[i])) - for _, b := range choices[i] { + var membership []int + fetches[i].request = request + fetches[i].groups, membership = newRenderGroups(choices[i]) + for j, b := range choices[i] { g := groups[b.fastRenderQ] if g == nil { g = &group{backend: b} groups[b.fastRenderQ] = g } - g.calls = append(g.calls, renderDispatch{request, results[i], errors[i]}) + replicas := fetches[i].groups[membership[j]] + g.calls = append(g.calls, renderDispatch{request, replicas.results, replicas.errors}) } } for _, g := range groups { g.backend.sendRenderGroup(ctx, g.calls) } + // One fan-in covers every target, so the slow replica grace of each runs + // from when its other groups answered, not after the previous target. + collectRenderFetches(fetches, lg) + // One backoff schedule covers every target, so a restarting store delays + // the request once rather than once per target. + retryUnreachable(ctx, fetches, lg) replies := make([]GroupedRenderReply, len(requests)) - for i := range requests { + for i := range fetches { if len(choices[i]) == 0 { continue } - messages := make([][]types.Metric, 0, len(choices[i])) - for range choices[i] { - select { - case metrics := <-results[i]: - messages = append(messages, metrics) - case err := <-errors[i]: - replies[i].Errors = append(replies[i].Errors, err) - } - } - replies[i].Metrics, replies[i].Stats = types.MergeMetrics(messages, mismatch, lg) + replies[i].Errors = renderGroupErrors(fetches[i].groups) + replies[i].Metrics, replies[i].Stats = types.MergeMetrics(fetches[i].messages, mismatch, lg) } return replies } diff --git a/pkg/cfg/api_test.go b/pkg/cfg/api_test.go index 06b61eb6..5164e2d5 100644 --- a/pkg/cfg/api_test.go +++ b/pkg/cfg/api_test.go @@ -66,9 +66,11 @@ loggerConfig: MaxProcs: 16, Timeouts: Timeouts{ - Global: 10 * time.Minute, - AfterStarted: 10 * time.Minute, - Connect: 200 * time.Millisecond, + Global: 10 * time.Minute, + AfterStarted: 10 * time.Minute, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 1024, KeepAliveInterval: 30 * time.Second, @@ -167,9 +169,11 @@ loggerConfig: MaxProcs: 16, Timeouts: Timeouts{ - Global: 10 * time.Minute, - AfterStarted: 10 * time.Minute, - Connect: 200 * time.Millisecond, + Global: 10 * time.Minute, + AfterStarted: 10 * time.Minute, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 1024, KeepAliveInterval: 30 * time.Second, diff --git a/pkg/cfg/common.go b/pkg/cfg/common.go index 4a637195..30b225d4 100644 --- a/pkg/cfg/common.go +++ b/pkg/cfg/common.go @@ -47,14 +47,18 @@ func DefaultCommonConfig() Common { MaxProcs: 1, Timeouts: Timeouts{ - Global: 10000 * time.Millisecond, - AfterStarted: 2 * time.Second, - Connect: 200 * time.Millisecond, + Global: 10000 * time.Millisecond, + AfterStarted: 2 * time.Second, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 20, KeepAliveInterval: 30 * time.Second, MaxIdleConnsPerHost: 100, IdleConnTimeout: 90 * time.Second, + PrewarmConnsPerBackend: 64, + PrewarmTimeout: 5 * time.Second, GrpcInitialWindowSize: 4 * 1024 * 1024, GrpcInitialConnWindowSize: 4 * 1024 * 1024, @@ -188,7 +192,15 @@ type Common struct { MaxIdleConnsPerHost int `yaml:"maxIdleConnsPerHost"` // IdleConnTimeout closes a pooled backend connection after it has been // idle this long; zero keeps it until the backend closes it. - IdleConnTimeout time.Duration `yaml:"idleConnTimeout"` + IdleConnTimeout time.Duration `yaml:"idleConnTimeout"` + // PrewarmConnsPerBackend is how many connections a starting process + // dials to each HTTP backend before it serves. On a release reload + // (SIGUSR2) the old process keeps serving meanwhile, so the new one takes + // over with a warm pool instead of dialing every store at full load. + // Zero or negative disables it. + PrewarmConnsPerBackend int `yaml:"prewarmConnsPerBackend"` + // PrewarmTimeout bounds the prewarm; the process serves after it either way. + PrewarmTimeout time.Duration `yaml:"prewarmTimeout"` GrpcInitialWindowSize int `yaml:"grpcInitialWindowSize"` GrpcInitialConnWindowSize int `yaml:"grpcInitialConnWindowSize"` @@ -373,6 +385,21 @@ type Timeouts struct { Global time.Duration `yaml:"global"` AfterStarted time.Duration `yaml:"afterStarted"` Connect time.Duration `yaml:"connect"` + // ConnectRetry is how long a render keeps retrying a backend that refused, + // reset or timed out the connection before sending any response, as during + // a store restart. The retry stops as soon as a replica group of the same + // cluster in another DC has answered for the target, with data or + // not-found. Zero disables the retry. + ConnectRetry time.Duration `yaml:"connectRetry"` + // SlowReplicaGrace is how long a render still waits for a replica group + // once every other replica group of the same cluster answered for the + // target (data or not-found) and at least one of them returned data, as + // when a store warming up after a restart accepts connections but answers + // slowly. The slow group's fetches are then dropped and the other groups' + // merged data returned. If every other group said not-found, the slow + // group may hold the only copy and is awaited. Zero waits for every + // backend up to afterStarted. + SlowReplicaGrace time.Duration `yaml:"slowReplicaGrace"` } type ProtocolBackend struct { diff --git a/pkg/cfg/common_test.go b/pkg/cfg/common_test.go index 9b17a1f0..70690d43 100644 --- a/pkg/cfg/common_test.go +++ b/pkg/cfg/common_test.go @@ -83,9 +83,11 @@ monitoring: }, MaxProcs: 32, Timeouts: Timeouts{ - Global: 20 * time.Second, - AfterStarted: 15 * time.Second, - Connect: 200 * time.Millisecond, + Global: 20 * time.Second, + AfterStarted: 15 * time.Second, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, @@ -262,9 +264,11 @@ monitoring: }, MaxProcs: 32, Timeouts: Timeouts{ - Global: 20 * time.Second, - AfterStarted: 15 * time.Second, - Connect: 200 * time.Millisecond, + Global: 20 * time.Second, + AfterStarted: 15 * time.Second, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, @@ -508,9 +512,11 @@ monitoring: }, MaxProcs: 32, Timeouts: Timeouts{ - Global: 20 * time.Second, - AfterStarted: 15 * time.Second, - Connect: 200 * time.Millisecond, + Global: 20 * time.Second, + AfterStarted: 15 * time.Second, + Connect: 200 * time.Millisecond, + ConnectRetry: 10 * time.Second, + SlowReplicaGrace: time.Second, }, ConcurrencyLimitPerServer: 2048, KeepAliveInterval: 30 * time.Second, diff --git a/pkg/types/merge_replicas_test.go b/pkg/types/merge_replicas_test.go new file mode 100644 index 00000000..9dec9928 --- /dev/null +++ b/pkg/types/merge_replicas_test.go @@ -0,0 +1,524 @@ +package types + +import ( + "math" + "math/rand" + "reflect" + "sort" + "testing" + + "github.com/bookingcom/carbonapi/pkg/cfg" +) + +// legacyMergeMetrics is mergeMetrics as of bk-master 89d6f213, before the +// freshest-shape base and per-reply skipping. The differential tests below use +// it as the oracle. +func legacyMergeMetrics(metrics []Metric, replicaMismatchConfig cfg.RenderReplicaMismatchConfig) (metric Metric, stats MetricRenderStats) { + if len(metrics) == 0 { + return Metric{}, MetricRenderStats{} + } + + if len(metrics) == 1 { + m := metrics[0] + m.materializePoints() + return m, MetricRenderStats{} + } + + var equalityFunc floatEqualityFunc + if replicaMismatchConfig.RenderReplicaMismatchApproximateCheck { + equalityFunc = AreFloatsApproximatelyEqual + } + + replicaMatchMode := replicaMismatchConfig.RenderReplicaMatchMode + + var mismatches, fixedMismatches int + + sort.Sort(byStepTime(metrics)) + healed := 0 + + clusterMap := make(map[string]bool) + for _, m := range metrics { + for _, cl := range m.SourceClusters { + clusterMap[cl] = true + } + } + clusters := make([]string, 0, len(clusterMap)) + for cl := range clusterMap { + clusters = append(clusters, cl) + } + sort.Strings(clusters) + // metrics[0] has the highest resolution of metrics + metric = metrics[0] + metric.materializePoints() + var cursors []packedPointCursor + for _, replica := range metrics[1:] { + if replica.packed != nil { + cursors = make([]packedPointCursor, len(metrics)) + for i := range cursors { + cursors[i].block = -1 + } + break + } + } + metric.SourceClusters = clusters + // Schema compatibility is constant across all points. Preserve the old + // first-incompatible-replica boundary without rechecking it per point. + replicaLimit := 1 + for replicaLimit < len(metrics) && metrics[replicaLimit].StepTime == metric.StepTime && metrics[replicaLimit].pointCount() == len(metric.Values) { + replicaLimit++ + } + valuesForPoint := make([]float64, 0, len(metrics)) + isMismatchFindConfig := replicaMatchMode != cfg.ReplicaMatchModeNormal + for i := range metric.Values { + if cursors != nil && i&63 == 0 { + for j := 1; j < replicaLimit; j++ { + if p := metrics[j].packed; p != nil { + cursors[j].advanceToBlock(p, i>>6) + } + } + } + pointExists := !metric.IsAbsent[i] + shouldLookForMismatch := isMismatchFindConfig + mismatchObserved := false + valuesForPoint = valuesForPoint[:0] + if pointExists { + valuesForPoint = append(valuesForPoint, metric.Values[i]) + } + for j := 1; j < replicaLimit; j++ { + if pointExists && !shouldLookForMismatch { + break + } + m := metrics[j] + + var value float64 + if m.packed != nil { + var absent bool + value, absent = cursors[j].pointAt(m.packed, i) + if absent { + continue + } + } else { + if m.IsAbsent[i] { + continue + } + value = m.Values[i] + } + + valuesForPoint = append(valuesForPoint, value) + + if !pointExists { + metric.IsAbsent[i] = false + metric.Values[i] = value + healed++ + pointExists = true + } + + if !mismatchObserved { + mismatchObserved = (equalityFunc == nil && metric.Values[i] != value) || + (equalityFunc != nil && !equalityFunc(metric.Values[i], value)) + if mismatchObserved && replicaMatchMode == cfg.ReplicaMatchModeCheck { + // mismatch exists, enough for check mode + shouldLookForMismatch = false + } + } + } + if !mismatchObserved { + continue + } + + mismatches++ + if replicaMatchMode == cfg.ReplicaMatchModeMajority { + majorityValue, isMajority, err := getPointMajorityValue(valuesForPoint, equalityFunc) + if err == nil && isMajority { + metric.Values[i] = majorityValue + fixedMismatches++ + } + } + } + + return metric, MetricRenderStats{ + DataPointCount: len(metric.Values), + MismatchCount: mismatches, + FixedMismatchCount: fixedMismatches, + } +} + +var mergeTestConfigs = func() []cfg.RenderReplicaMismatchConfig { + var configs []cfg.RenderReplicaMismatchConfig + for _, mode := range []cfg.ReplicaMatchMode{cfg.ReplicaMatchModeNormal, cfg.ReplicaMatchModeCheck, cfg.ReplicaMatchModeMajority} { + for _, approx := range []bool{false, true} { + configs = append(configs, cfg.RenderReplicaMismatchConfig{RenderReplicaMatchMode: mode, RenderReplicaMismatchApproximateCheck: approx}) + } + } + return configs +}() + +// mergeTestInput returns an independent copy of replies, packing the flagged +// ones, because mergeMetrics sorts its input and heals the base in place. +func mergeTestInput(ms []Metric, packed []bool) []Metric { + out := make([]Metric, len(ms)) + for i, m := range ms { + m.Values = append([]float64(nil), m.Values...) + m.IsAbsent = append([]bool(nil), m.IsAbsent...) + m.SourceClusters = append([]string(nil), m.SourceClusters...) + if packed != nil && packed[i] { + m.PackPoints() + } + out[i] = m + } + return out +} + +func assertSameMerge(t *testing.T, got Metric, gotStats MetricRenderStats, want Metric, wantStats MetricRenderStats) { + t.Helper() + if gotStats != wantStats { + t.Fatalf("stats differ: got %+v, want %+v", gotStats, wantStats) + } + if got.packed != nil || want.packed != nil || len(got.Values) != len(want.Values) { + t.Fatalf("unmaterialized or different length output: got %d points, want %d", len(got.Values), len(want.Values)) + } + for i := range got.Values { + if math.Float64bits(got.Values[i]) != math.Float64bits(want.Values[i]) { + t.Fatalf("value bits differ at %d: got %x, want %x", i, math.Float64bits(got.Values[i]), math.Float64bits(want.Values[i])) + } + } + got.Values, want.Values = nil, nil + if !reflect.DeepEqual(got, want) { + t.Fatalf("metadata or absence differs:\ngot %+v\nwant %+v", got, want) + } +} + +// mergeTestReply builds a 60 s reply starting at start; NaN marks an absent point. +func mergeTestReply(cluster string, start int32, values ...float64) Metric { + m := Metric{ + Name: "metric", + StartTime: start, + StopTime: start + 60*int32(len(values)), + StepTime: 60, + Values: make([]float64, len(values)), + IsAbsent: make([]bool, len(values)), + SourceClusters: []string{cluster}, + } + for i, v := range values { + if math.IsNaN(v) { + m.IsAbsent[i] = true + } else { + m.Values[i] = v + } + } + return m +} + +func assertMergedPoints(t *testing.T, got Metric, want Metric) { + t.Helper() + if got.StartTime != want.StartTime || got.StopTime != want.StopTime || !reflect.DeepEqual(got.IsAbsent, want.IsAbsent) || !reflect.DeepEqual(got.Values, want.Values) { + t.Fatalf("got %d-%d %v %v, want %d-%d %v %v", got.StartTime, got.StopTime, got.Values, got.IsAbsent, want.StartTime, want.StopTime, want.Values, want.IsAbsent) + } +} + +var nan = math.NaN() + +func TestMergeBaseTieBreaks(t *testing.T) { + for _, tc := range []struct { + name string + replies []Metric + want int + }{ + {"one shape keeps the first reply", []Metric{mergeTestReply("A", 600, 1, nan), mergeTestReply("B", 600, 1, 2)}, 0}, + {"latest present point decides the shape", []Metric{mergeTestReply("A", 600, 1, 2, 3), mergeTestReply("B", 600, 1, 2, 3, 4)}, 1}, + {"then most present points", []Metric{mergeTestReply("A", 600, nan, nan, 3, 4, 5), mergeTestReply("B", 660, 2, 3, 4, 5)}, 1}, + {"then reply order", []Metric{mergeTestReply("A", 600, nan, 2, 3, 4, 5), mergeTestReply("B", 660, 2, 3, 4, 5)}, 0}, + {"base is the first reply of the freshest shape", []Metric{mergeTestReply("A", 600, 1, 2, 3, nan, nan), mergeTestReply("B", 600, 1, 2, 3), mergeTestReply("C", 600, 1, 2, 3, 4, 5)}, 0}, + {"all absent keeps the first reply", []Metric{mergeTestReply("A", 600, nan, nan), mergeTestReply("B", 600, nan)}, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := mergeBase(tc.replies); got != tc.want { + t.Fatalf("mergeBase = %d, want %d", got, tc.want) + } + packed := mergeTestInput(tc.replies, nil) + for i := range packed { + packed[i].packed = &packedPoints{n: len(packed[i].Values), values: make([]uint64, (len(packed[i].Values)+63)/64), absent: make([]uint64, (len(packed[i].Values)+63)/64)} + for j, absent := range packed[i].IsAbsent { + if absent { + packed[i].packed.absent[j/64] |= uint64(1) << (uint(j) & 63) + } + } + packed[i].Values, packed[i].IsAbsent = nil, nil + } + if got := mergeBase(packed); got != tc.want { + t.Fatalf("packed mergeBase = %d, want %d", got, tc.want) + } + }) + } +} + +// A store that answers first but still lacks its newest points must not +// leave nulls that another reply can fill. +func TestMergeMetricsStaleFirstReplyHealedFromFresherReply(t *testing.T) { + for _, tc := range []struct { + name string + replies []Metric + want Metric + legacyBroken bool + }{ + { + name: "trailing absent points", + replies: []Metric{mergeTestReply("A", 600, 1, 2, 3, nan, nan), mergeTestReply("B", 600, 1, 2, 3, 4, 5)}, + want: mergeTestReply("", 600, 1, 2, 3, 4, 5), + }, + { + name: "shorter series", + replies: []Metric{mergeTestReply("A", 600, 1, 2, 3), mergeTestReply("B", 600, 1, 2, 3, 4, 5)}, + want: mergeTestReply("", 600, 1, 2, 3, 4, 5), + legacyBroken: true, + }, + { + name: "shorter, stale and complete replies", + replies: []Metric{ + mergeTestReply("A", 600, 1, 2, 3), + mergeTestReply("B", 600, 1, 2, 3, nan, nan), + mergeTestReply("C", 600, 1, nan, 3, 4, 5), + }, + want: mergeTestReply("", 600, 1, 2, 3, 4, 5), + legacyBroken: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + for _, c := range mergeTestConfigs { + got, stats := mergeMetrics(mergeTestInput(tc.replies, nil), c) + assertMergedPoints(t, got, tc.want) + if stats.MismatchCount != 0 { + t.Fatalf("%s: %d mismatches, want 0", c.String(), stats.MismatchCount) + } + legacy, _ := legacyMergeMetrics(mergeTestInput(tc.replies, nil), c) + if broken := !reflect.DeepEqual(legacy.Values, tc.want.Values) || !reflect.DeepEqual(legacy.IsAbsent, tc.want.IsAbsent); broken != tc.legacyBroken { + t.Fatalf("%s: legacy merge broken=%v, want %v (%v %v)", c.String(), broken, tc.legacyBroken, legacy.Values, legacy.IsAbsent) + } + } + }) + } +} + +// A reply with another point count in the middle of the list is skipped on +// its own; the replies after it still fill the base's gaps. +func TestMergeMetricsSkipsOnlyMismatchedReply(t *testing.T) { + replies := []Metric{ + mergeTestReply("A", 600, 1, nan, 3, 4, 5), + mergeTestReply("B", 600, 1, 2, 3, 4), + mergeTestReply("C", 600, 1, 2, 3, 4, nan), + } + want := mergeTestReply("", 600, 1, 2, 3, 4, 5) + for _, c := range mergeTestConfigs { + got, stats := mergeMetrics(mergeTestInput(replies, nil), c) + assertMergedPoints(t, got, want) + if stats.MismatchCount != 0 || stats.DataPointCount != 5 { + t.Fatalf("%s: stats %+v", c.String(), stats) + } + if legacy, _ := legacyMergeMetrics(mergeTestInput(replies, nil), c); !legacy.IsAbsent[1] { + t.Fatalf("%s: legacy merge filled the gap; the test no longer covers the old limit", c.String()) + } + } +} + +func TestMergeMetricsMajorityUnchangedOnDisagreement(t *testing.T) { + majority := cfg.RenderReplicaMismatchConfig{RenderReplicaMatchMode: cfg.ReplicaMatchModeMajority} + + // One shape, first reply stale: the base stays the first reply, so a + // strict majority wins and an unresolved conflict keeps the base value, + // exactly as before. + replies := []Metric{ + mergeTestReply("A", 600, 1, 2, 7, nan), + mergeTestReply("B", 600, 1, 5, 8, 10), + mergeTestReply("C", 600, 1, 5, 9, 11), + } + got, stats := mergeMetrics(mergeTestInput(replies, nil), majority) + assertMergedPoints(t, got, mergeTestReply("", 600, 1, 5, 7, 10)) + if stats != (MetricRenderStats{DataPointCount: 4, MismatchCount: 3, FixedMismatchCount: 1}) { + t.Fatalf("stats %+v", stats) + } + for _, c := range mergeTestConfigs { + got, gs := mergeMetrics(mergeTestInput(replies, nil), c) + want, ws := legacyMergeMetrics(mergeTestInput(replies, nil), c) + assertSameMerge(t, got, gs, want, ws) + } + + // A shorter reply in the middle no longer removes the replies after it + // from the vote. + replies = []Metric{ + mergeTestReply("A", 600, 1, 2, 3, nan), + mergeTestReply("B", 600, 1, 5, 3), + mergeTestReply("C", 600, 1, 5, 3, 4), + mergeTestReply("D", 600, 1, 5, 9, 4), + } + got, stats = mergeMetrics(mergeTestInput(replies, nil), majority) + assertMergedPoints(t, got, mergeTestReply("", 600, 1, 5, 3, 4)) + if stats != (MetricRenderStats{DataPointCount: 4, MismatchCount: 2, FixedMismatchCount: 2}) { + t.Fatalf("stats %+v", stats) + } +} + +var mergeTestValueBits = []uint64{ + 0, 1 << 63, // +0 and -0 + 0x7ff0000000000000, 0xfff0000000000000, // +Inf and -Inf + 0x7ff8000000000000, 0x7ff8000000000123, 0xfff8000000000001, // NaN payloads + 0x3fd3333333333333, 0x3fd3333333333334, // 0.3 and 0.1+0.2 + 1, +} + +func mergeTestValue(rng *rand.Rand) float64 { + if rng.Intn(3) == 0 { + return math.Float64frombits(mergeTestValueBits[rng.Intn(len(mergeTestValueBits))]) + } + // A small value range so replies often agree and majorities form. + return float64(rng.Intn(4)) +} + +func mergeTestAbsence(rng *rand.Rand, n int) []bool { + absent := make([]bool, n) + density := []float64{0, 0.05, 0.5, 0.95, 1}[rng.Intn(5)] + for i := range absent { + absent[i] = rng.Float64() < density + } + if rng.Intn(3) == 0 { // stale tail, as from a store still replaying + for i := n - rng.Intn(n+1); i < n; i++ { + absent[i] = true + } + } + if rng.Intn(4) == 0 { // leading gap + for i, end := 0, rng.Intn(n+1); i < end; i++ { + absent[i] = true + } + } + return absent +} + +var mergeTestSizes = []int{1, 2, 5, 64, 129, 300} + +// Replies of one shape whose present points agree must merge bit-identically +// to the old algorithm, including NaN payloads, absent-slot payloads and the +// mismatch counts, in every match mode. +func TestMergeMetricsAgreeingRepliesMatchLegacy(t *testing.T) { + rng := rand.New(rand.NewSource(20261113)) + for iter := 0; iter < 300; iter++ { + n := mergeTestSizes[iter%len(mergeTestSizes)] + values, absentValues := make([]float64, n), make([]float64, n) + for i := range values { + values[i] = mergeTestValue(rng) + // Decoders write 0 or NaN into absent slots. + absentValues[i] = math.Float64frombits([]uint64{0, 0x7ff8000000000001}[rng.Intn(2)]) + } + var replies []Metric + var packed []bool + for r := 0; r < 2+rng.Intn(4); r++ { + m := Metric{Name: "a", StartTime: 600, StopTime: 600 + 60*int32(n), StepTime: 60, Values: make([]float64, n), IsAbsent: mergeTestAbsence(rng, n), SourceClusters: []string{string(rune('A' + r))}} + for i := range m.Values { + m.Values[i] = values[i] + if m.IsAbsent[i] { + m.Values[i] = absentValues[i] + } + } + replies = append(replies, m) + packed = append(packed, rng.Intn(2) == 0) + } + if rng.Intn(3) == 0 { // a coarser archive, ignored by both + coarse := Metric{Name: "a", StartTime: 600, StopTime: 600 + 120*int32(n/2+1), StepTime: 120, Values: make([]float64, n/2+1), IsAbsent: mergeTestAbsence(rng, n/2+1), SourceClusters: []string{"Z"}} + for i := range coarse.Values { + coarse.Values[i] = mergeTestValue(rng) + } + at := rng.Intn(len(replies) + 1) + replies = append(replies[:at], append([]Metric{coarse}, replies[at:]...)...) + packed = append(packed[:at], append([]bool{false}, packed[at:]...)...) + } + for _, c := range mergeTestConfigs { + want, ws := legacyMergeMetrics(mergeTestInput(replies, nil), c) + got, gs := mergeMetrics(mergeTestInput(replies, packed), c) + assertSameMerge(t, got, gs, want, ws) + } + } +} + +// mergeTestFreshest is a direct statement of the base rule: among the +// finest-step replies, the latest present point, then the most present +// points, then reply order. +func mergeTestFreshest(sorted []Metric) int { + key := func(m Metric) (int64, int) { + last, present := int64(math.MinInt64), 0 + for i, absent := range m.IsAbsent { + if !absent { + last = int64(m.StartTime) + int64(i)*int64(m.StepTime) + present++ + } + } + return last, present + } + best := 0 + for j := 1; j < len(sorted) && sorted[j].StepTime == sorted[0].StepTime; j++ { + lj, pj := key(sorted[j]) + lb, pb := key(sorted[best]) + if lj > lb || (lj == lb && pj > pb) { + best = j + } + } + return best +} + +// For any mix of shapes and values, including disagreeing values in majority +// mode, the merge equals the old algorithm run on the replies of the freshest +// shape (in reply order) followed by the others, which it then ignores. +func TestMergeMetricsMixedShapesMatchLegacyOnFreshestShape(t *testing.T) { + rng := rand.New(rand.NewSource(20261114)) + for iter := 0; iter < 300; iter++ { + n := mergeTestSizes[iter%len(mergeTestSizes)] + shared := make([]float64, n+4) + for i := range shared { + shared[i] = mergeTestValue(rng) + } + var replies []Metric + var packed []bool + for r := 0; r < 2+rng.Intn(5); r++ { + count, start, step := n, int32(600), int32(60) + switch rng.Intn(5) { + case 0: // shorter: missing its newest or oldest points + count = max(1, n-1-rng.Intn(3)) + start += 60 * int32(rng.Intn(2)*(n-count)) + case 1: // longer + count = n + 1 + rng.Intn(3) + start -= 60 * int32(rng.Intn(2)*(count-n)) + case 2: + if rng.Intn(3) == 0 { + step, count = 120, n/2+1 + } + } + m := Metric{Name: "a", StartTime: start, StopTime: start + step*int32(count), StepTime: step, Values: make([]float64, count), IsAbsent: mergeTestAbsence(rng, count), SourceClusters: []string{string(rune('A' + r))}} + agree := rng.Intn(2) == 0 + for i := range m.Values { + if agree { + m.Values[i] = shared[i] + } else { + m.Values[i] = mergeTestValue(rng) + } + } + replies = append(replies, m) + packed = append(packed, rng.Intn(2) == 0) + } + + sorted := mergeTestInput(replies, nil) + sort.Stable(byStepTime(sorted)) + freshest := sorted[mergeTestFreshest(sorted)] + var ordered, rest []Metric + for _, m := range sorted { + if m.StepTime == freshest.StepTime && len(m.Values) == len(freshest.Values) { + ordered = append(ordered, m) + } else { + rest = append(rest, m) + } + } + ordered = append(ordered, rest...) + + for _, c := range mergeTestConfigs { + want, ws := legacyMergeMetrics(mergeTestInput(ordered, nil), c) + got, gs := mergeMetrics(mergeTestInput(replies, packed), c) + assertSameMerge(t, got, gs, want, ws) + } + } +} diff --git a/pkg/types/packed_points.go b/pkg/types/packed_points.go index 68c03d07..85c1de14 100644 --- a/pkg/types/packed_points.go +++ b/pkg/types/packed_points.go @@ -89,6 +89,52 @@ func (m Metric) pointCount() int { return len(m.Values) } +// lastPresentTime returns the timestamp of the newest present point, or +// math.MinInt64 when every point is absent. +func (m Metric) lastPresentTime() int64 { + last := -1 + if p := m.packed; p != nil { + for block := len(p.absent) - 1; block >= 0 && last < 0; block-- { + present := ^p.absent[block] + if valid := p.n - block*64; valid < 64 { + present &= uint64(1)<= 0; i-- { + if !m.IsAbsent[i] { + last = i + break + } + } + } + if last < 0 { + return math.MinInt64 + } + return int64(m.StartTime) + int64(last)*int64(m.StepTime) +} + +// presentCount returns the number of present points. +func (m Metric) presentCount() int { + if p := m.packed; p != nil { + absent := 0 + for _, block := range p.absent { + absent += bits.OnesCount64(block) + } + return p.n - absent + } + present := 0 + for _, absent := range m.IsAbsent { + if !absent { + present++ + } + } + return present +} + func (c *packedPointCursor) advanceToBlock(p *packedPoints, block int) { for c.block < block { c.offset = c.next diff --git a/pkg/types/types.go b/pkg/types/types.go index 2ff38da6..f6da66f5 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -289,6 +289,63 @@ func getPointMajorityValue(values []float64, equalityFunc floatEqualityFunc) (ma return m, majorityCount > valuesCount/2, nil } +// mergeBase returns the index of the reply that mergeMetrics starts from. +// metrics must be sorted by step, finest first. +// +// When all finest-step replies have the same point count, the base is the +// first reply, as it always was: replies of one shape heal each other's +// absent points in reply order whichever of them is the base. +// +// When their point counts differ, the freshest finest-step reply decides the +// shape: the one with the latest present point, then the most present points, +// then the earliest in reply order. The base is the first reply with that +// shape. A reply that lacks its newest points, for example from a store still +// replaying after a restart, therefore cannot cut the merged series short. +func mergeBase(metrics []Metric) int { + step, count := metrics[0].StepTime, metrics[0].pointCount() + sameShape := true + for _, m := range metrics[1:] { + if m.StepTime != step { + break + } + if m.pointCount() != count { + sameShape = false + break + } + } + if sameShape { + return 0 + } + + freshest := 0 + freshestLast, freshestPresent := metrics[0].lastPresentTime(), -1 + for j := 1; j < len(metrics) && metrics[j].StepTime == step; j++ { + last := metrics[j].lastPresentTime() + if last < freshestLast { + continue + } + present := -1 + if last == freshestLast { + if freshestPresent < 0 { + freshestPresent = metrics[freshest].presentCount() + } + present = metrics[j].presentCount() + if present <= freshestPresent { + continue + } + } + freshest, freshestLast, freshestPresent = j, last, present + } + + count = metrics[freshest].pointCount() + for j := 0; j < freshest; j++ { + if metrics[j].pointCount() == count { + return j + } + } + return freshest +} + func mergeMetrics(metrics []Metric, replicaMismatchConfig cfg.RenderReplicaMismatchConfig) (metric Metric, stats MetricRenderStats) { if len(metrics) == 0 { return Metric{}, MetricRenderStats{} @@ -323,31 +380,36 @@ func mergeMetrics(metrics []Metric, replicaMismatchConfig cfg.RenderReplicaMisma clusters = append(clusters, cl) } sort.Strings(clusters) - // metrics[0] has the highest resolution of metrics - metric = metrics[0] + base := mergeBase(metrics) + metric = metrics[base] metric.materializePoints() + // Every reply with the base's step and point count heals the base, in reply + // order; the base is the first reply of its shape, so they all follow it. + // A reply of another shape is skipped on its own: it no longer hides the + // replies after it. Schema compatibility is constant across all points, so + // it is checked once here and not per point. + var peerBuf [8]int + peers := peerBuf[:0] + packedPeer := false + for j := base + 1; j < len(metrics); j++ { + if metrics[j].StepTime == metric.StepTime && metrics[j].pointCount() == len(metric.Values) { + peers = append(peers, j) + packedPeer = packedPeer || metrics[j].packed != nil + } + } var cursors []packedPointCursor - for _, replica := range metrics[1:] { - if replica.packed != nil { - cursors = make([]packedPointCursor, len(metrics)) - for i := range cursors { - cursors[i].block = -1 - } - break + if packedPeer { + cursors = make([]packedPointCursor, len(metrics)) + for i := range cursors { + cursors[i].block = -1 } } metric.SourceClusters = clusters - // Schema compatibility is constant across all points. Preserve the old - // first-incompatible-replica boundary without rechecking it per point. - replicaLimit := 1 - for replicaLimit < len(metrics) && metrics[replicaLimit].StepTime == metric.StepTime && metrics[replicaLimit].pointCount() == len(metric.Values) { - replicaLimit++ - } valuesForPoint := make([]float64, 0, len(metrics)) isMismatchFindConfig := replicaMatchMode != cfg.ReplicaMatchModeNormal for i := range metric.Values { if cursors != nil && i&63 == 0 { - for j := 1; j < replicaLimit; j++ { + for _, j := range peers { if p := metrics[j].packed; p != nil { cursors[j].advanceToBlock(p, i>>6) } @@ -360,7 +422,7 @@ func mergeMetrics(metrics []Metric, replicaMismatchConfig cfg.RenderReplicaMisma if pointExists { valuesForPoint = append(valuesForPoint, metric.Values[i]) } - for j := 1; j < replicaLimit; j++ { + for _, j := range peers { if pointExists && !shouldLookForMismatch { break }