Skip to content
11 changes: 11 additions & 0 deletions config/carbonzipper.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,17 @@ timeouts:
afterStarted: "200s"
# Timeout to connect to the server
connect: "200ms"
# How long a render keeps retrying a backend that refused, reset or timed
# out the connection before responding (a store restart), while no replica
# group of the same cluster in another DC has answered (data or not-found).
# 0 disables.
connectRetry: "10s"
# How long a render still waits for a replica group once every other
# replica group of the same cluster answered (data or not-found) and at
# least one returned data (a store that accepts connections but answers
# slowly while it warms up). If all the others said not-found, the slow
# group may hold the only copy and is awaited. 0 waits up to afterStarted.
slowReplicaGrace: "1s"

# Number of concurrent requests to any given backend - default is no limit.
# If set, you likely want >= MaxIdleConnsPerHost
Expand Down
110 changes: 99 additions & 11 deletions pkg/app/carbonapi/app.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package carbonapi

import (
"context"
"fmt"
"log"
"net"
"net/http"
"net/url"
"os"
"runtime"
"strconv"
Expand Down Expand Up @@ -324,29 +326,114 @@ func (app *App) deferredAccessLogging(accessLogger *zap.Logger, r *http.Request,
}
}

func newBackendHTTPTransport(config cfg.Zipper) http.RoundTripper {
// prewarmDialTimeout is the connect budget of a prewarm dial. It is longer
// than the request-path connect timeout because nobody waits on it, and long
// enough to ride out one dropped SYN (1 s initial retransmission).
const prewarmDialTimeout = 2 * time.Second

// prewarmConnTTL is how long a prewarmed connection may wait for its first
// request before it is closed instead of handed out.
const prewarmConnTTL = 10 * time.Second

// prewarmParallel caps concurrent prewarm dials across all backends.
const prewarmParallel = 256

func newBackendHTTPTransport(config cfg.Zipper, warm *bnet.WarmDialer) http.RoundTripper {
dial := (&net.Dialer{
Timeout: config.Timeouts.Connect,
KeepAlive: config.KeepAliveInterval,
}).DialContext
if warm != nil {
dial = warm.DialContext
}
transport := &http.Transport{
MaxIdleConnsPerHost: config.MaxIdleConnsPerHost,
IdleConnTimeout: config.IdleConnTimeout,
DialContext: (&net.Dialer{
Timeout: config.Timeouts.Connect,
KeepAlive: config.KeepAliveInterval,
}).DialContext,
DialContext: dial,
}
if os.Getenv("CARBONAPI_SYNC_BACKEND_HTTP") == "1" {
return bnet.NewSyncTransport(transport)
}
return transport
}

func newWarmDialer(config cfg.Zipper) *bnet.WarmDialer {
return bnet.NewWarmDialer((&net.Dialer{
Timeout: config.Timeouts.Connect,
KeepAlive: config.KeepAliveInterval,
}).DialContext, prewarmConnTTL)
}

// prewarmAddrs returns the unique host:port of every HTTP backend.
func prewarmAddrs(config cfg.Zipper) []string {
seen := make(map[string]struct{})
var addrs []string
for _, b := range config.GetBackends() {
u, err := url.Parse(b.Http)
if err != nil || u.Scheme != "http" || u.Hostname() == "" {
continue
}
addr := u.Host
if u.Port() == "" {
addr = net.JoinHostPort(u.Hostname(), "80")
}
if _, dup := seen[addr]; dup {
continue
}
seen[addr] = struct{}{}
addrs = append(addrs, addr)
}
return addrs
}

// prewarmBackends dials config.PrewarmConnsPerBackend connections to every
// HTTP backend before the process serves. It runs before gracehttp.Serve, so
// on a reload the old process still serves while this one warms up.
func prewarmBackends(config cfg.Zipper, warm *bnet.WarmDialer, lg *zap.Logger) {
if warm == nil || config.PrewarmConnsPerBackend <= 0 || config.PrewarmTimeout <= 0 {
return
}
addrs := prewarmAddrs(config)
ctx, cancel := context.WithTimeout(context.Background(), config.PrewarmTimeout)
defer cancel()
dial := (&net.Dialer{
Timeout: max(config.Timeouts.Connect, prewarmDialTimeout),
KeepAlive: config.KeepAliveInterval,
}).DialContext
res := warm.Prewarm(ctx, dial, addrs, config.PrewarmConnsPerBackend, prewarmParallel)
lg.Info("backend prewarm done",
zap.Int("backends", len(addrs)),
zap.Int("conns_per_backend", config.PrewarmConnsPerBackend),
zap.Int("dialed", res.Dialed),
zap.Int("failed", res.Failed),
zap.Duration("elapsed", res.Elapsed),
)
}

func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *PrometheusMetrics, lg *zap.Logger) ([]backend.Backend, error) {
client := &http.Client{Transport: newBackendHTTPTransport(config)}
bs, _, err := initBackends(config, zms, ms, lg)
return bs, err
}

func initBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *PrometheusMetrics, lg *zap.Logger) ([]backend.Backend, *bnet.WarmDialer, error) {
warm := newWarmDialer(config)
client := &http.Client{Transport: newBackendHTTPTransport(config, warm)}
backend.SetConnectRetry(backend.ConnectRetry{
Window: config.Timeouts.ConnectRetry,
Retries: zms.BackendConnectRetries,
Recoveries: zms.BackendConnectRecoveries,
Cancellations: zms.BackendConnectCancels,
})
backend.SetSlowReplicaGrace(backend.SlowReplicaGrace{
Grace: config.Timeouts.SlowReplicaGrace,
Cancelled: zms.BackendSlowReplicaCuts,
})

configBackendList := config.GetBackends()
backends := make([]backend.Backend, 0, len(configBackendList))
for _, host := range configBackendList {
if host.Http == "" {
return nil, fmt.Errorf("backend without http address was provided: %+v", host)
return nil, nil, fmt.Errorf("backend without http address was provided: %+v", host)
}
dc, cluster, _ := config.InfoOfBackend(host.Http)
var b backend.Backend
Expand Down Expand Up @@ -374,12 +461,12 @@ func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *Prometheu
be, err = bnet.New(bConf)
}
if err != nil {
return nil, errors.Wrapf(err, "could not create backend for host: %s", host)
return nil, nil, errors.Wrapf(err, "could not create backend for host: %s", host)
}

beDuration, err := ms.BackendDuration.CurryWith(prometheus.Labels{"dc": dc, "cluster": cluster})
if err != nil {
return nil, errors.Wrap(err, "could not curry backend duration metric")
return nil, nil, errors.Wrap(err, "could not curry backend duration metric")
}
b = backend.NewBackend(be,
config.BackendQueueSize,
Expand All @@ -393,7 +480,7 @@ func InitBackends(config cfg.Zipper, zms *ZipperPrometheusMetrics, ms *Prometheu
backends = append(backends, b)
}

return backends, nil
return backends, warm, nil
}

// Setup sets up the zipper for future lanuch.
Expand Down Expand Up @@ -428,10 +515,11 @@ func SetupZipper(configFile string, BuildVersion string, ms *PrometheusMetrics,
)

zms := NewZipperPrometheusMetrics(config)
bs, err := InitBackends(config, zms, ms, lg)
bs, warm, err := initBackends(config, zms, ms, lg)
if err != nil {
lg.Fatal("failed to init backends", zap.Error(err))
}
prewarmBackends(config, warm, lg)

return config, bs, expirecache.New(0), tldcache.InitTLDPrefixes(lg, config.TLDCacheExtraPrefixes), zms
}
90 changes: 89 additions & 1 deletion pkg/app/carbonapi/backend_conn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func TestBackendHTTPTransportSelection(t *testing.T) {
config := cfg.DefaultZipperConfig()
config.MaxIdleConnsPerHost = 23
config.IdleConnTimeout = 43 * time.Second
transport := newBackendHTTPTransport(config)
transport := newBackendHTTPTransport(config, nil)
t.Cleanup((&http.Client{Transport: transport}).CloseIdleConnections)
if tc.sync {
if _, ok := transport.(*bnet.SyncTransport); !ok {
Expand Down Expand Up @@ -126,3 +126,91 @@ func testBackendIdleConnTimeoutFromConfig(t *testing.T) {
t.Fatalf("%d connections after idleConnTimeout, want 2 (idle connection not closed)", n)
}
}

// A reloaded process must take traffic with connections it dialed before
// serving, not dial every store under full load.
func TestBackendPrewarmServesFirstRendersWithoutDialing(t *testing.T) {
for _, value := range []string{"", "1"} {
t.Run("sync="+value, func(t *testing.T) {
t.Setenv("CARBONAPI_SYNC_BACKEND_HTTP", value)
var dials atomic.Int64
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.NotFound(w, nil)
}))
server.Config.ConnState = func(_ net.Conn, s http.ConnState) {
if s == http.StateNew {
dials.Add(1)
}
}
server.Start()
defer server.Close()

z := cfg.DefaultZipperConfig()
z.Backends = []string{server.URL, server.URL + "/"} // duplicates dial once
z.ProtocolBackends = nil
z.BackendsByCluster = nil
z.BackendsByDC = nil
z.PrewarmConnsPerBackend = 3
ms := newPrometheusMetrics(cfg.DefaultAPIConfig())
bs, warm, err := initBackends(z, NewZipperPrometheusMetrics(z), &ms, zap.NewNop())
if err != nil {
t.Fatal(err)
}
prewarmBackends(z, warm, zap.NewNop())
waitDials := func(want int64) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for dials.Load() != want && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if n := dials.Load(); n != want {
t.Fatalf("%d connections, want %d", n, want)
}
}
waitDials(3)
if warm.Idle() != 3 {
t.Fatalf("idle prewarmed = %d, want 3", warm.Idle())
}

// Three concurrent renders take the three prewarmed connections.
errs := make(chan error, 3)
for i := 0; i < 3; i++ {
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := bs[0].Render(ctx, types.NewRenderRequest([]string{"a.b"}, 1, 2))
errs <- err
}()
}
for i := 0; i < 3; i++ {
if err := <-errs; !errors.Is(err, types.ErrMetricsNotFound) {
t.Fatalf("render: %v", err)
}
}
if n := dials.Load(); n != 3 {
t.Fatalf("%d connections after renders on 3 prewarmed, want 3", n)
}
})
}
}

func TestBackendPrewarmAddrs(t *testing.T) {
z := cfg.DefaultZipperConfig()
z.Backends = []string{"http://10.0.0.1:8080", "http://10.0.0.1:8080/", "http://store-2", "https://x:1", "::bad"}
z.ProtocolBackends = nil
z.BackendsByCluster = nil
z.BackendsByDC = nil
got := prewarmAddrs(z)
want := []string{"10.0.0.1:8080", "store-2:80"}
if len(got) != len(want) {
t.Fatalf("prewarmAddrs = %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("prewarmAddrs = %v, want %v", got, want)
}
}
if d := cfg.DefaultZipperConfig(); d.PrewarmConnsPerBackend <= 0 || d.PrewarmTimeout <= 0 {
t.Fatalf("prewarm is off by default: %d conns, %s", d.PrewarmConnsPerBackend, d.PrewarmTimeout)
}
}
Loading
Loading