Add integration test for tracing on image pull

Create an in-memory exporter and global tracer provider
Pull image with client which should create spans
Validate spans in the exporter

Signed-off-by: Tony Fang <nhfang@amazon.com>

integration/client/client_test.go

@@ -26,6 +26,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/identity"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
+	exec "golang.org/x/sys/execabs"
+
 	. "github.com/containerd/containerd"
 	"github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/errdefs"
@@ -36,10 +43,6 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/pkg/testutil"
 	"github.com/containerd/containerd/platforms"
-	"github.com/opencontainers/go-digest"
-	"github.com/opencontainers/image-spec/identity"
-	"github.com/sirupsen/logrus"
-	exec "golang.org/x/sys/execabs"
 )
 
 var (
@@ -457,6 +460,34 @@ func TestImagePullWithConcurrencyLimit(t *testing.T) {
 	}
 }
 
+func TestImagePullWithTracing(t *testing.T) {
+	client, err := newClient(t, address)
+	require.NoError(t, err)
+	defer client.Close()
+
+	ctx := namespaces.WithNamespace(context.Background(), "tracing")
+
+	//create in memory exporter and global tracer provider for test
+	exp, tp := newInMemoryExporterTracer()
+	//set the tracer provider global available
+	otel.SetTracerProvider(tp)
+	// Shutdown properly so nothing leaks.
+	defer func() { _ = tp.Shutdown(ctx) }()
+
+	//do an image pull which is instrumented, we should expect spans in the exporter
+	_, err = client.Pull(ctx, testImage, WithPlatformMatcher(platforms.Default()))
+	require.NoError(t, err)
+
+	err = tp.ForceFlush(ctx)
+	require.NoError(t, err)
+
+	//The span name was defined in client.pull when instrumented it
+	spanNameExpected := "pull.Pull"
+	spans := exp.GetSpans()
+	validateRootSpan(t, spanNameExpected, spans)
+
+}
+
 func TestClientReconnect(t *testing.T) {
 	t.Parallel()
 

integration/client/go.mod

@@ -16,6 +16,8 @@ require (
 	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.1
+	go.opentelemetry.io/otel v1.12.0
+	go.opentelemetry.io/otel/sdk v1.12.0
 	golang.org/x/sys v0.4.0
 )
 
@@ -52,7 +54,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/otel v1.12.0 // indirect
 	go.opentelemetry.io/otel/trace v1.12.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
 	golang.org/x/net v0.5.0 // indirect

integration/client/go.sum

@@ -1205,6 +1205,7 @@ go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2b
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
 go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU=
+go.opentelemetry.io/otel/sdk v1.12.0 h1:8npliVYV7qc0t1FKdpU08eMnOjgPFMnriPhn0HH4q3o=
 go.opentelemetry.io/otel/sdk v1.12.0/go.mod h1:WYcvtgquYvgODEvxOry5owO2y9MyciW7JqMz6cpXShE=
 go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
 go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=

integration/client/tracing.go

@@ -0,0 +1,61 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+//This file defines funcs to adding integration test around opentelemetry tracing by
+// First, create tracer provider and in memory exporter to store generated spans from code.
+// Then run the instrumented code where we expect spans.
+// Then we check if the spans in exporter match expectation, like span name, status code and etc.
+
+package client
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/otel/codes"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+// newInMemoryExporterTracer creates in memory exporter and tracer provider to be
+// used as tracing test
+func newInMemoryExporterTracer() (*tracetest.InMemoryExporter, *sdktrace.TracerProvider) {
+	//create in memory exporter
+	exp := tracetest.NewInMemoryExporter()
+
+	//create tracer provider
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exp),
+	)
+
+	return exp, tp
+}
+
+// validateRootSpan takes span slice as input, check if there are rootspans match the expected
+// name and the status code is not error
+func validateRootSpan(t *testing.T, spanNameExpected string, spans []tracetest.SpanStub) {
+	for _, span := range spans {
+		//We only look for root span
+		//A span is root span if its parent SpanContext is invalid
+		if !span.Parent.IsValid() {
+			if span.Name == spanNameExpected {
+				assert.NotEqual(t, span.Status.Code, codes.Error)
+				return
+			}
+		}
+	}
+	t.Fatalf("Expected span %s not found", spanNameExpected)
+}