[otel-tracing] Initial opentelemetry support

Add basic intiialization of opentelemetry including minimum support to
be able to read open telemetry config from config.toml and initialize
exporter. Tracer is initialized and ready to be be used for creating
spans, sub spans etc. With no opentelemetry configuration enabled in
config file, this patch is a no-op.

Basic config stub to be added to use opentelemetry is to add following
in config.toml. We use otlp exporter with default port 4317.

[otel]
  exporter_name = "otlp"
  exporter_endpoint = "0.0.0.1:4317"

otel-collector binary needs to run listening at the same port.

Signed-off-by: Alakesh Haloi <alakeshh@amazon.com>

services/server/config/config.go

@@ -67,6 +67,8 @@ type Config struct {
 	Timeouts map[string]string `toml:"timeouts"`
 	// Imports are additional file path list to config files that can overwrite main config file fields
 	Imports []string `toml:"imports"`
+	// OpenTelemetry configuration
+	OpenTelemetry OpenTelemetryConfig `toml:"otel"`
 
 	StreamProcessors map[string]StreamProcessor `toml:"stream_processors"`
 }
@@ -165,6 +167,14 @@ type ProxyPlugin struct {
 	Address string `toml:"address"`
 }
 
+// OpenTelemetryConfig provides open telemetry configuration
+type OpenTelemetryConfig struct {
+	ServiceName        string  `toml:"service_name"`
+	ExporterName       string  `toml:"exporter_name"`
+	ExporterEndpoint   string  `toml:"exporter_endpoint"`
+	TraceSamplingRatio float64 `toml:"trace_sampling_ratio"`
+}
+
 // BoltConfig defines the configuration values for the bolt plugin, which is
 // loaded here, rather than back registered in the metadata package.
 type BoltConfig struct {
@@ -203,6 +213,24 @@ func (bc *BoltConfig) Validate() error {
 	}
 }
 
+const (
+	// ExporterTypeOTLP represents the open telemetry exporter OTLP
+	ExporterTypeOTLP = "otlp"
+)
+
+// Validate OpenTelemetry config
+func (cfg *OpenTelemetryConfig) Validate() error {
+	switch cfg.ExporterName {
+	case ExporterTypeOTLP:
+		if cfg.ServiceName == "" {
+			return errors.Wrapf(errdefs.ErrInvalidArgument, "missing service name in config %+v", cfg)
+		}
+		return nil
+	default:
+		return errors.Wrapf(errdefs.ErrInvalidArgument, "unsupported exporter: %+v", cfg)
+	}
+}
+
 // Decode unmarshals a plugin specific configuration by plugin id
 func (c *Config) Decode(p *plugin.Registration) (interface{}, error) {
 	id := p.URI()

services/server/server.go

@@ -51,9 +51,11 @@ import (
 	"github.com/containerd/containerd/sys"
 	"github.com/containerd/ttrpc"
 	metrics "github.com/docker/go-metrics"
+	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/pkg/errors"
 	bolt "go.etcd.io/bbolt"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/backoff"
 	"google.golang.org/grpc/credentials"
@@ -98,8 +100,14 @@ func New(ctx context.Context, config *srvconfig.Config) (*Server, error) {
 	}
 
 	serverOpts := []grpc.ServerOption{
-		grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
-		grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
+		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
+			otelgrpc.StreamServerInterceptor(),
+			grpc.StreamServerInterceptor(grpc_prometheus.StreamServerInterceptor),
+		)),
+		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
+			otelgrpc.UnaryServerInterceptor(),
+			grpc.UnaryServerInterceptor(grpc_prometheus.UnaryServerInterceptor),
+		)),
 	}
 	if config.GRPC.MaxRecvMsgSize > 0 {
 		serverOpts = append(serverOpts, grpc.MaxRecvMsgSize(config.GRPC.MaxRecvMsgSize))