Share a single etcd3 client logger across all clients
Currently the API server creates one etcd client per CRD. If clients aren't provided a logger they'll each create their own. These loggers can account for ~20% of API server memory consumption on a cluster with hundreds of CRDs. Signed-off-by: Nic Cope <nicc@rk0n.org>
This commit is contained in:
		| @@ -27,10 +27,12 @@ import ( | |||||||
| 	"time" | 	"time" | ||||||
|  |  | ||||||
| 	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" | 	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" | ||||||
|  | 	"go.etcd.io/etcd/client/pkg/v3/logutil" | ||||||
| 	"go.etcd.io/etcd/client/pkg/v3/transport" | 	"go.etcd.io/etcd/client/pkg/v3/transport" | ||||||
| 	clientv3 "go.etcd.io/etcd/client/v3" | 	clientv3 "go.etcd.io/etcd/client/v3" | ||||||
| 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" | 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" | ||||||
| 	"go.uber.org/zap" | 	"go.uber.org/zap" | ||||||
|  | 	"go.uber.org/zap/zapcore" | ||||||
| 	"google.golang.org/grpc" | 	"google.golang.org/grpc" | ||||||
|  |  | ||||||
| 	"k8s.io/apimachinery/pkg/runtime" | 	"k8s.io/apimachinery/pkg/runtime" | ||||||
| @@ -64,6 +66,14 @@ const ( | |||||||
| 	dbMetricsMonitorJitter = 0.5 | 	dbMetricsMonitorJitter = 0.5 | ||||||
| ) | ) | ||||||
|  |  | ||||||
|  | // TODO(negz): Stop using a package scoped logger. At the time of writing we're | ||||||
|  | // creating an etcd client for each CRD. We need to pass each etcd client a | ||||||
|  | // logger or each client will create its own, which comes with a significant | ||||||
|  | // memory cost (around 20% of the API server's memory when hundreds of CRDs are | ||||||
|  | // present). The correct fix here is to not create a client per CRD. See | ||||||
|  | // https://github.com/kubernetes/kubernetes/issues/111476 for more. | ||||||
|  | var etcd3ClientLogger *zap.Logger | ||||||
|  |  | ||||||
| func init() { | func init() { | ||||||
| 	// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of | 	// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of | ||||||
| 	// using the global prometheus registry and using our own wrapped global registry, | 	// using the global prometheus registry and using our own wrapped global registry, | ||||||
| @@ -71,6 +81,12 @@ func init() { | |||||||
| 	// For reference: https://github.com/kubernetes/kubernetes/pull/81387 | 	// For reference: https://github.com/kubernetes/kubernetes/pull/81387 | ||||||
| 	legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics) | 	legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics) | ||||||
| 	dbMetricsMonitors = make(map[string]struct{}) | 	dbMetricsMonitors = make(map[string]struct{}) | ||||||
|  |  | ||||||
|  | 	l, err := logutil.CreateDefaultZapLogger(zapcore.InfoLevel) | ||||||
|  | 	if err != nil { | ||||||
|  | 		l = zap.NewNop() | ||||||
|  | 	} | ||||||
|  | 	etcd3ClientLogger = l | ||||||
| } | } | ||||||
|  |  | ||||||
| func newETCD3HealthCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() error, error) { | func newETCD3HealthCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() error, error) { | ||||||
| @@ -217,6 +233,7 @@ var newETCD3Client = func(c storagebackend.TransportConfig) (*clientv3.Client, e | |||||||
| 		} | 		} | ||||||
| 		dialOptions = append(dialOptions, grpc.WithContextDialer(dialer)) | 		dialOptions = append(dialOptions, grpc.WithContextDialer(dialer)) | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	cfg := clientv3.Config{ | 	cfg := clientv3.Config{ | ||||||
| 		DialTimeout:          dialTimeout, | 		DialTimeout:          dialTimeout, | ||||||
| 		DialKeepAliveTime:    keepaliveTime, | 		DialKeepAliveTime:    keepaliveTime, | ||||||
| @@ -224,10 +241,7 @@ var newETCD3Client = func(c storagebackend.TransportConfig) (*clientv3.Client, e | |||||||
| 		DialOptions:          dialOptions, | 		DialOptions:          dialOptions, | ||||||
| 		Endpoints:            c.ServerList, | 		Endpoints:            c.ServerList, | ||||||
| 		TLS:                  tlsConfig, | 		TLS:                  tlsConfig, | ||||||
|  | 		Logger:               etcd3ClientLogger, | ||||||
| 		// This logger uses a significant amount of memory when many CRDs (i.e. |  | ||||||
| 		// 1,000+) are added to the API server, so we disable it. |  | ||||||
| 		Logger: zap.NewNop(), |  | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	return clientv3.New(cfg) | 	return clientv3.New(cfg) | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user
	 Nic Cope
					Nic Cope