Merge pull request #7616 from swagatbora90/trace-cri-runtime

Add tracing spans to CRI runtime service apis

client/client.go

@@ -59,6 +59,7 @@ import (
 	"github.com/containerd/containerd/v2/pkg/dialer"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/containerd/v2/plugins"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/platforms"
@@ -284,6 +285,8 @@ func (c *Client) Containers(ctx context.Context, filters ...string) ([]Container
 // NewContainer will create a new container with the provided id.
 // The id must be unique within the namespace.
 func (c *Client) NewContainer(ctx context.Context, id string, opts ...NewContainerOpts) (Container, error) {
+	ctx, span := tracing.StartSpan(ctx, "client.NewContainer")
+	defer span.End()
 	ctx, done, err := c.WithLease(ctx)
 	if err != nil {
 		return nil, err
@@ -301,6 +304,13 @@ func (c *Client) NewContainer(ctx context.Context, id string, opts ...NewContain
 			return nil, err
 		}
 	}
+
+	span.SetAttributes(
+		tracing.Attribute("container.id", container.ID),
+		tracing.Attribute("container.image.ref", container.Image),
+		tracing.Attribute("container.runtime.name", container.Runtime.Name),
+		tracing.Attribute("container.snapshotter.name", container.Snapshotter),
+	)
 	r, err := c.ContainerService().Create(ctx, container)
 	if err != nil {
 		return nil, err
@@ -310,10 +320,21 @@ func (c *Client) NewContainer(ctx context.Context, id string, opts ...NewContain
 
 // LoadContainer loads an existing container from metadata
 func (c *Client) LoadContainer(ctx context.Context, id string) (Container, error) {
+	ctx, span := tracing.StartSpan(ctx, "client.LoadContainer")
+	defer span.End()
 	r, err := c.ContainerService().Get(ctx, id)
 	if err != nil {
 		return nil, err
 	}
+
+	span.SetAttributes(
+		tracing.Attribute("container.id", r.ID),
+		tracing.Attribute("container.image.ref", r.Image),
+		tracing.Attribute("container.runtime.name", r.Runtime.Name),
+		tracing.Attribute("container.snapshotter.name", r.Snapshotter),
+		tracing.Attribute("container.createdAt", r.CreatedAt.Format(time.RFC3339)),
+		tracing.Attribute("container.updatedAt", r.UpdatedAt.Format(time.RFC3339)),
+	)
 	return containerFromRecord(c, r), nil
 }
 

client/container.go

@@ -32,6 +32,7 @@ import (
 	"github.com/containerd/containerd/v2/core/images"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/oci"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/fifo"
 	"github.com/containerd/typeurl/v2"
@@ -140,6 +141,10 @@ func (c *container) Labels(ctx context.Context) (map[string]string, error) {
 }
 
 func (c *container) SetLabels(ctx context.Context, labels map[string]string) (map[string]string, error) {
+	ctx, span := tracing.StartSpan(ctx, "container.SetLabels",
+		tracing.WithAttribute("container.id", c.id),
+	)
+	defer span.End()
 	container := containers.Container{
 		ID:     c.id,
 		Labels: labels,
@@ -175,6 +180,10 @@ func (c *container) Spec(ctx context.Context) (*oci.Spec, error) {
 // Delete deletes an existing container
 // an error is returned if the container has running tasks
 func (c *container) Delete(ctx context.Context, opts ...DeleteOpts) error {
+	ctx, span := tracing.StartSpan(ctx, "container.Delete",
+		tracing.WithAttribute("container.id", c.id),
+	)
+	defer span.End()
 	if _, err := c.loadTask(ctx, nil); err == nil {
 		return fmt.Errorf("cannot delete running task %v: %w", c.id, errdefs.ErrFailedPrecondition)
 	}
@@ -211,6 +220,8 @@ func (c *container) Image(ctx context.Context) (Image, error) {
 }
 
 func (c *container) NewTask(ctx context.Context, ioCreate cio.Creator, opts ...NewTaskOpts) (_ Task, err error) {
+	ctx, span := tracing.StartSpan(ctx, "container.NewTask")
+	defer span.End()
 	i, err := ioCreate(c.id)
 	if err != nil {
 		return nil, err
@@ -298,16 +309,28 @@ func (c *container) NewTask(ctx context.Context, ioCreate cio.Creator, opts ...N
 	if info.Checkpoint != nil {
 		request.Checkpoint = info.Checkpoint
 	}
+
+	span.SetAttributes(
+		tracing.Attribute("task.container.id", request.ContainerID),
+		tracing.Attribute("task.request.options", request.Options.String()),
+		tracing.Attribute("task.runtime.name", info.runtime),
+	)
 	response, err := c.client.TaskService().Create(ctx, request)
 	if err != nil {
 		return nil, errdefs.FromGRPC(err)
 	}
+
+	span.AddEvent("task created",
+		tracing.Attribute("task.process.id", int(response.Pid)),
+	)
 	t.pid = response.Pid
 	return t, nil
 }
 
 func (c *container) Update(ctx context.Context, opts ...UpdateContainerOpts) error {
 	// fetch the current container config before updating it
+	ctx, span := tracing.StartSpan(ctx, "container.Update")
+	defer span.End()
 	r, err := c.get(ctx)
 	if err != nil {
 		return err

client/process.go

@@ -26,6 +26,7 @@ import (
 	"github.com/containerd/containerd/api/services/tasks/v1"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/protobuf"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 )
 
@@ -118,6 +119,11 @@ func (p *process) Pid() uint32 {
 
 // Start starts the exec process
 func (p *process) Start(ctx context.Context) error {
+	ctx, span := tracing.StartSpan(ctx, "process.Start",
+		tracing.WithAttribute("process.id", p.ID()),
+		tracing.WithAttribute("process.task.id", p.task.ID()),
+	)
+	defer span.End()
 	r, err := p.task.client.TaskService().Start(ctx, &tasks.StartRequest{
 		ContainerID: p.task.id,
 		ExecID:      p.id,
@@ -130,11 +136,18 @@ func (p *process) Start(ctx context.Context) error {
 		}
 		return errdefs.FromGRPC(err)
 	}
+	span.SetAttributes(tracing.Attribute("process.pid", int(r.Pid)))
 	p.pid = r.Pid
 	return nil
 }
 
 func (p *process) Kill(ctx context.Context, s syscall.Signal, opts ...KillOpts) error {
+	ctx, span := tracing.StartSpan(ctx, "process.Kill",
+		tracing.WithAttribute("process.id", p.ID()),
+		tracing.WithAttribute("process.pid", int(p.Pid())),
+		tracing.WithAttribute("process.task.id", p.task.ID()),
+	)
+	defer span.End()
 	var i KillInfo
 	for _, o := range opts {
 		if err := o(ctx, &i); err != nil {
@@ -154,6 +167,11 @@ func (p *process) Wait(ctx context.Context) (<-chan ExitStatus, error) {
 	c := make(chan ExitStatus, 1)
 	go func() {
 		defer close(c)
+		ctx, span := tracing.StartSpan(ctx, "process.Wait",
+			tracing.WithAttribute("process.id", p.ID()),
+			tracing.WithAttribute("process.task.id", p.task.ID()),
+		)
+		defer span.End()
 		r, err := p.task.client.TaskService().Wait(ctx, &tasks.WaitRequest{
 			ContainerID: p.task.id,
 			ExecID:      p.id,
@@ -174,6 +192,10 @@ func (p *process) Wait(ctx context.Context) (<-chan ExitStatus, error) {
 }
 
 func (p *process) CloseIO(ctx context.Context, opts ...IOCloserOpts) error {
+	ctx, span := tracing.StartSpan(ctx, "process.CloseIO",
+		tracing.WithAttribute("process.id", p.ID()),
+	)
+	defer span.End()
 	r := &tasks.CloseIORequest{
 		ContainerID: p.task.id,
 		ExecID:      p.id,
@@ -192,6 +214,10 @@ func (p *process) IO() cio.IO {
 }
 
 func (p *process) Resize(ctx context.Context, w, h uint32) error {
+	ctx, span := tracing.StartSpan(ctx, "process.Resize",
+		tracing.WithAttribute("process.id", p.ID()),
+	)
+	defer span.End()
 	_, err := p.task.client.TaskService().ResizePty(ctx, &tasks.ResizePtyRequest{
 		ContainerID: p.task.id,
 		Width:       w,
@@ -202,6 +228,10 @@ func (p *process) Resize(ctx context.Context, w, h uint32) error {
 }
 
 func (p *process) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStatus, error) {
+	ctx, span := tracing.StartSpan(ctx, "process.Delete",
+		tracing.WithAttribute("process.id", p.ID()),
+	)
+	defer span.End()
 	for _, o := range opts {
 		if err := o(ctx, p); err != nil {
 			return nil, err
@@ -238,8 +268,11 @@ func (p *process) Status(ctx context.Context) (Status, error) {
 	if err != nil {
 		return Status{}, errdefs.FromGRPC(err)
 	}
+	status := ProcessStatus(strings.ToLower(r.Process.Status.String()))
+	exitStatus := r.Process.ExitStatus
+
 	return Status{
-		Status:     ProcessStatus(strings.ToLower(r.Process.Status.String())),
+		Status:     status,
-		ExitStatus: r.Process.ExitStatus,
+		ExitStatus: exitStatus,
 	}, nil
 }

client/task.go

@@ -38,6 +38,7 @@ import (
 	"github.com/containerd/containerd/v2/pkg/protobuf"
 	google_protobuf "github.com/containerd/containerd/v2/pkg/protobuf/types"
 	"github.com/containerd/containerd/v2/pkg/rootfs"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/containerd/v2/plugins"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/typeurl/v2"
@@ -210,6 +211,10 @@ func (t *task) Pid() uint32 {
 }
 
 func (t *task) Start(ctx context.Context) error {
+	ctx, span := tracing.StartSpan(ctx, "task.Start",
+		tracing.WithAttribute("task.id", t.ID()),
+	)
+	defer span.End()
 	r, err := t.client.TaskService().Start(ctx, &tasks.StartRequest{
 		ContainerID: t.id,
 	})
@@ -220,17 +225,28 @@ func (t *task) Start(ctx context.Context) error {
 		}
 		return errdefs.FromGRPC(err)
 	}
+	span.SetAttributes(tracing.Attribute("task.pid", r.Pid))
 	t.pid = r.Pid
 	return nil
 }
 
 func (t *task) Kill(ctx context.Context, s syscall.Signal, opts ...KillOpts) error {
+	ctx, span := tracing.StartSpan(ctx, "task.Kill",
+		tracing.WithAttribute("task.id", t.ID()),
+		tracing.WithAttribute("task.pid", int(t.Pid())),
+	)
+	defer span.End()
 	var i KillInfo
 	for _, o := range opts {
 		if err := o(ctx, &i); err != nil {
 			return err
 		}
 	}
+
+	span.SetAttributes(
+		tracing.Attribute("task.exec.id", i.ExecID),
+		tracing.Attribute("task.exec.killall", i.All),
+	)
 	_, err := t.client.TaskService().Kill(ctx, &tasks.KillRequest{
 		Signal:      uint32(s),
 		ContainerID: t.id,
@@ -244,6 +260,10 @@ func (t *task) Kill(ctx context.Context, s syscall.Signal, opts ...KillOpts) err
 }
 
 func (t *task) Pause(ctx context.Context) error {
+	ctx, span := tracing.StartSpan(ctx, "task.Pause",
+		tracing.WithAttribute("task.id", t.ID()),
+	)
+	defer span.End()
 	_, err := t.client.TaskService().Pause(ctx, &tasks.PauseTaskRequest{
 		ContainerID: t.id,
 	})
@@ -251,6 +271,10 @@ func (t *task) Pause(ctx context.Context) error {
 }
 
 func (t *task) Resume(ctx context.Context) error {
+	ctx, span := tracing.StartSpan(ctx, "task.Resume",
+		tracing.WithAttribute("task.id", t.ID()),
+	)
+	defer span.End()
 	_, err := t.client.TaskService().Resume(ctx, &tasks.ResumeTaskRequest{
 		ContainerID: t.id,
 	})
@@ -264,10 +288,14 @@ func (t *task) Status(ctx context.Context) (Status, error) {
 	if err != nil {
 		return Status{}, errdefs.FromGRPC(err)
 	}
+	status := ProcessStatus(strings.ToLower(r.Process.Status.String()))
+	exitStatus := r.Process.ExitStatus
+	exitTime := protobuf.FromTimestamp(r.Process.ExitedAt)
+
 	return Status{
-		Status:     ProcessStatus(strings.ToLower(r.Process.Status.String())),
+		Status:     status,
-		ExitStatus: r.Process.ExitStatus,
+		ExitStatus: exitStatus,
-		ExitTime:   protobuf.FromTimestamp(r.Process.ExitedAt),
+		ExitTime:   exitTime,
 	}, nil
 }
 
@@ -275,6 +303,10 @@ func (t *task) Wait(ctx context.Context) (<-chan ExitStatus, error) {
 	c := make(chan ExitStatus, 1)
 	go func() {
 		defer close(c)
+		ctx, span := tracing.StartSpan(ctx, "task.Wait",
+			tracing.WithAttribute("task.id", t.ID()),
+		)
+		defer span.End()
 		r, err := t.client.TaskService().Wait(ctx, &tasks.WaitRequest{
 			ContainerID: t.id,
 		})
@@ -297,6 +329,10 @@ func (t *task) Wait(ctx context.Context) (<-chan ExitStatus, error) {
 // it returns the exit status of the task and any errors that were encountered
 // during cleanup
 func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStatus, error) {
+	ctx, span := tracing.StartSpan(ctx, "task.Delete",
+		tracing.WithAttribute("task.id", t.ID()),
+	)
+	defer span.End()
 	for _, o := range opts {
 		if err := o(ctx, t); err != nil {
 			return nil, err
@@ -306,6 +342,7 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStat
 	if err != nil && errdefs.IsNotFound(err) {
 		return nil, err
 	}
+
 	switch status.Status {
 	case Stopped, Unknown, "":
 	case Created:
@@ -350,9 +387,14 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStat
 }
 
 func (t *task) Exec(ctx context.Context, id string, spec *specs.Process, ioCreate cio.Creator) (_ Process, err error) {
+	ctx, span := tracing.StartSpan(ctx, "task.Exec",
+		tracing.WithAttribute("task.id", t.ID()),
+	)
+	defer span.End()
 	if id == "" {
 		return nil, fmt.Errorf("exec id must not be empty: %w", errdefs.ErrInvalidArgument)
 	}
+	span.SetAttributes(tracing.Attribute("task.exec.id", id))
 	i, err := ioCreate(id)
 	if err != nil {
 		return nil, err
@@ -408,6 +450,10 @@ func (t *task) Pids(ctx context.Context) ([]ProcessInfo, error) {
 }
 
 func (t *task) CloseIO(ctx context.Context, opts ...IOCloserOpts) error {
+	ctx, span := tracing.StartSpan(ctx, "task.CloseIO",
+		tracing.WithAttribute("task.id", t.ID()),
+	)
+	defer span.End()
 	r := &tasks.CloseIORequest{
 		ContainerID: t.id,
 	}
@@ -416,6 +462,7 @@ func (t *task) CloseIO(ctx context.Context, opts ...IOCloserOpts) error {
 		o(&i)
 	}
 	r.Stdin = i.Stdin
+
 	_, err := t.client.TaskService().CloseIO(ctx, r)
 	return errdefs.FromGRPC(err)
 }
@@ -425,6 +472,10 @@ func (t *task) IO() cio.IO {
 }
 
 func (t *task) Resize(ctx context.Context, w, h uint32) error {
+	ctx, span := tracing.StartSpan(ctx, "task.Resize",
+		tracing.WithAttribute("task.id", t.ID()),
+	)
+	defer span.End()
 	_, err := t.client.TaskService().ResizePty(ctx, &tasks.ResizePtyRequest{
 		ContainerID: t.id,
 		Width:       w,
@@ -538,6 +589,10 @@ type UpdateTaskInfo struct {
 type UpdateTaskOpts func(context.Context, *Client, *UpdateTaskInfo) error
 
 func (t *task) Update(ctx context.Context, opts ...UpdateTaskOpts) error {
+	ctx, span := tracing.StartSpan(ctx, "task.Update",
+		tracing.WithAttribute("task.id", t.ID()),
+	)
+	defer span.End()
 	request := &tasks.UpdateTaskRequest{
 		ContainerID: t.id,
 	}

core/metadata/containers.go

@@ -31,11 +31,16 @@ import (
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	"github.com/containerd/containerd/v2/pkg/protobuf/proto"
 	"github.com/containerd/containerd/v2/pkg/protobuf/types"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/typeurl/v2"
 	bolt "go.etcd.io/bbolt"
 )
 
+const (
+	spanContainerPrefix = "metadata.containers"
+)
+
 type containerStore struct {
 	db *DB
 }
@@ -116,6 +121,11 @@ func (s *containerStore) List(ctx context.Context, fs ...string) ([]containers.C
 }
 
 func (s *containerStore) Create(ctx context.Context, container containers.Container) (containers.Container, error) {
+	ctx, span := tracing.StartSpan(ctx,
+		tracing.Name(spanContainerPrefix, "Create"),
+		tracing.WithAttribute("container.id", container.ID),
+	)
+	defer span.End()
 	namespace, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return containers.Container{}, err
@@ -145,6 +155,9 @@ func (s *containerStore) Create(ctx context.Context, container containers.Contai
 			return fmt.Errorf("failed to write container %q: %w", container.ID, err)
 		}
 
+		span.SetAttributes(
+			tracing.Attribute("container.createdAt", container.CreatedAt.Format(time.RFC3339)),
+		)
 		return nil
 	}); err != nil {
 		return containers.Container{}, err
@@ -154,6 +167,11 @@ func (s *containerStore) Create(ctx context.Context, container containers.Contai
 }
 
 func (s *containerStore) Update(ctx context.Context, container containers.Container, fieldpaths ...string) (containers.Container, error) {
+	ctx, span := tracing.StartSpan(ctx,
+		tracing.Name(spanContainerPrefix, "Update"),
+		tracing.WithAttribute("container.id", container.ID),
+	)
+	defer span.End()
 	namespace, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return containers.Container{}, err
@@ -245,6 +263,10 @@ func (s *containerStore) Update(ctx context.Context, container containers.Contai
 			return fmt.Errorf("failed to write container %q: %w", container.ID, err)
 		}
 
+		span.SetAttributes(
+			tracing.Attribute("container.createdAt", updated.CreatedAt.Format(time.RFC3339)),
+			tracing.Attribute("container.updatedAt", updated.UpdatedAt.Format(time.RFC3339)),
+		)
 		return nil
 	}); err != nil {
 		return containers.Container{}, err
@@ -254,6 +276,12 @@ func (s *containerStore) Update(ctx context.Context, container containers.Contai
 }
 
 func (s *containerStore) Delete(ctx context.Context, id string) error {
+	ctx, span := tracing.StartSpan(ctx,
+		tracing.Name(spanContainerPrefix, "Delete"),
+		tracing.WithAttribute("container.id", id),
+	)
+	defer span.End()
+
 	namespace, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return err

core/metadata/sandbox.go

@@ -28,11 +28,16 @@ import (
 	"github.com/containerd/containerd/v2/pkg/filters"
 	"github.com/containerd/containerd/v2/pkg/identifiers"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/typeurl/v2"
 	"go.etcd.io/bbolt"
 )
 
+const (
+	spanSandboxPrefix = "metadata.sandbox"
+)
+
 type sandboxStore struct {
 	db *DB
 }
@@ -46,6 +51,11 @@ func NewSandboxStore(db *DB) api.Store {
 
 // Create a sandbox record in the store
 func (s *sandboxStore) Create(ctx context.Context, sandbox api.Sandbox) (api.Sandbox, error) {
+	ctx, span := tracing.StartSpan(ctx,
+		tracing.Name(spanSandboxPrefix, "Create"),
+		tracing.WithAttribute("sandbox.id", sandbox.ID),
+	)
+	defer span.End()
 	ns, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return api.Sandbox{}, err
@@ -68,6 +78,9 @@ func (s *sandboxStore) Create(ctx context.Context, sandbox api.Sandbox) (api.San
 			return fmt.Errorf("write error: %w", err)
 		}
 
+		span.SetAttributes(
+			tracing.Attribute("sandbox.CreatedAt", sandbox.CreatedAt.Format(time.RFC3339)),
+		)
 		return nil
 	}); err != nil {
 		return api.Sandbox{}, err
@@ -78,6 +91,11 @@ func (s *sandboxStore) Create(ctx context.Context, sandbox api.Sandbox) (api.San
 
 // Update the sandbox with the provided sandbox object and fields
 func (s *sandboxStore) Update(ctx context.Context, sandbox api.Sandbox, fieldpaths ...string) (api.Sandbox, error) {
+	ctx, span := tracing.StartSpan(ctx,
+		tracing.Name(spanSandboxPrefix, "Update"),
+		tracing.WithAttribute("sandbox.id", sandbox.ID),
+	)
+	defer span.End()
 	ns, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return api.Sandbox{}, err
@@ -142,6 +160,10 @@ func (s *sandboxStore) Update(ctx context.Context, sandbox api.Sandbox, fieldpat
 			return err
 		}
 
+		span.SetAttributes(
+			tracing.Attribute("sandbox.CreatedAt", updated.CreatedAt.Format(time.RFC3339)),
+			tracing.Attribute("sandbox.UpdatedAt", updated.UpdatedAt.Format(time.RFC3339)),
+		)
 		ret = updated
 		return nil
 	}); err != nil {
@@ -227,6 +249,11 @@ func (s *sandboxStore) List(ctx context.Context, fields ...string) ([]api.Sandbo
 
 // Delete a sandbox from metadata store using the id
 func (s *sandboxStore) Delete(ctx context.Context, id string) error {
+	ctx, span := tracing.StartSpan(ctx,
+		tracing.Name(spanSandboxPrefix, "Delete"),
+		tracing.WithAttribute("sandbox.id", id),
+	)
+	defer span.End()
 	ns, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return err

internal/cri/instrument/instrumented_service.go

@@ -27,11 +27,6 @@ import (
 	ctrdutil "github.com/containerd/containerd/v2/internal/cri/util"
 )
 
-const (
-	// criSpanPrefix is a prefix for CRI server specific spans
-	criSpanPrefix = "pkg.cri.server"
-)
-
 // criService is an CRI server dependency to be wrapped with instrumentation.
 type criService interface {
 	GRPCServices
@@ -69,6 +64,7 @@ func (in *instrumentedService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Infof("RunPodSandbox for %+v", r.GetConfig().GetMetadata())
 	defer func() {
 		if err != nil {
@@ -76,6 +72,7 @@ func (in *instrumentedService) RunPodSandbox(ctx context.Context, r *runtime.Run
 		} else {
 			log.G(ctx).Infof("RunPodSandbox for %+v returns sandbox id %q", r.GetConfig().GetMetadata(), res.GetPodSandboxId())
 		}
+		span.RecordError(err)
 	}()
 	res, err = in.c.RunPodSandbox(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -117,6 +114,7 @@ func (in *instrumentedService) StopPodSandbox(ctx context.Context, r *runtime.St
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Infof("StopPodSandbox for %q", r.GetPodSandboxId())
 	defer func() {
 		if err != nil {
@@ -124,6 +122,7 @@ func (in *instrumentedService) StopPodSandbox(ctx context.Context, r *runtime.St
 		} else {
 			log.G(ctx).Infof("StopPodSandbox for %q returns successfully", r.GetPodSandboxId())
 		}
+		span.RecordError(err)
 	}()
 	res, err := in.c.StopPodSandbox(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -133,6 +132,7 @@ func (in *instrumentedService) RemovePodSandbox(ctx context.Context, r *runtime.
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Infof("RemovePodSandbox for %q", r.GetPodSandboxId())
 	defer func() {
 		if err != nil {
@@ -140,6 +140,7 @@ func (in *instrumentedService) RemovePodSandbox(ctx context.Context, r *runtime.
 		} else {
 			log.G(ctx).Infof("RemovePodSandbox %q returns successfully", r.GetPodSandboxId())
 		}
+		span.RecordError(err)
 	}()
 	res, err := in.c.RemovePodSandbox(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -165,6 +166,7 @@ func (in *instrumentedService) CreateContainer(ctx context.Context, r *runtime.C
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Infof("CreateContainer within sandbox %q for container %+v",
 		r.GetPodSandboxId(), r.GetConfig().GetMetadata())
 	defer func() {
@@ -175,6 +177,7 @@ func (in *instrumentedService) CreateContainer(ctx context.Context, r *runtime.C
 			log.G(ctx).Infof("CreateContainer within sandbox %q for %+v returns container id %q",
 				r.GetPodSandboxId(), r.GetConfig().GetMetadata(), res.GetContainerId())
 		}
+		span.RecordError(err)
 	}()
 	res, err = in.c.CreateContainer(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -184,6 +187,7 @@ func (in *instrumentedService) StartContainer(ctx context.Context, r *runtime.St
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Infof("StartContainer for %q", r.GetContainerId())
 	defer func() {
 		if err != nil {
@@ -191,6 +195,7 @@ func (in *instrumentedService) StartContainer(ctx context.Context, r *runtime.St
 		} else {
 			log.G(ctx).Infof("StartContainer for %q returns successfully", r.GetContainerId())
 		}
+		span.RecordError(err)
 	}()
 	res, err := in.c.StartContainer(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -233,6 +238,7 @@ func (in *instrumentedService) StopContainer(ctx context.Context, r *runtime.Sto
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Infof("StopContainer for %q with timeout %d (s)", r.GetContainerId(), r.GetTimeout())
 	defer func() {
 		if err != nil {
@@ -240,6 +246,7 @@ func (in *instrumentedService) StopContainer(ctx context.Context, r *runtime.Sto
 		} else {
 			log.G(ctx).Infof("StopContainer for %q returns successfully", r.GetContainerId())
 		}
+		span.RecordError(err)
 	}()
 	res, err = in.c.StopContainer(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -249,6 +256,7 @@ func (in *instrumentedService) RemoveContainer(ctx context.Context, r *runtime.R
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Infof("RemoveContainer for %q", r.GetContainerId())
 	defer func() {
 		if err != nil {
@@ -256,6 +264,7 @@ func (in *instrumentedService) RemoveContainer(ctx context.Context, r *runtime.R
 		} else {
 			log.G(ctx).Infof("RemoveContainer for %q returns successfully", r.GetContainerId())
 		}
+		span.RecordError(err)
 	}()
 	res, err = in.c.RemoveContainer(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -265,6 +274,7 @@ func (in *instrumentedService) ExecSync(ctx context.Context, r *runtime.ExecSync
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Debugf("ExecSync for %q with command %+v and timeout %d (s)", r.GetContainerId(), r.GetCmd(), r.GetTimeout())
 	defer func() {
 		if err != nil {
@@ -272,6 +282,7 @@ func (in *instrumentedService) ExecSync(ctx context.Context, r *runtime.ExecSync
 		} else {
 			log.G(ctx).Tracef("ExecSync for %q returns with exit code %d", r.GetContainerId(), res.GetExitCode())
 		}
+		span.RecordError(err)
 	}()
 	res, err = in.c.ExecSync(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -281,6 +292,7 @@ func (in *instrumentedService) Exec(ctx context.Context, r *runtime.ExecRequest)
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Debugf("Exec for %q with command %+v, tty %v and stdin %v",
 		r.GetContainerId(), r.GetCmd(), r.GetTty(), r.GetStdin())
 	defer func() {
@@ -289,6 +301,7 @@ func (in *instrumentedService) Exec(ctx context.Context, r *runtime.ExecRequest)
 		} else {
 			log.G(ctx).Debugf("Exec for %q returns URL %q", r.GetContainerId(), res.GetUrl())
 		}
+		span.RecordError(err)
 	}()
 	res, err = in.c.Exec(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -298,6 +311,7 @@ func (in *instrumentedService) Attach(ctx context.Context, r *runtime.AttachRequ
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
+	span := tracing.SpanFromContext(ctx)
 	log.G(ctx).Debugf("Attach for %q with tty %v and stdin %v", r.GetContainerId(), r.GetTty(), r.GetStdin())
 	defer func() {
 		if err != nil {
@@ -305,6 +319,7 @@ func (in *instrumentedService) Attach(ctx context.Context, r *runtime.AttachRequ
 		} else {
 			log.G(ctx).Debugf("Attach for %q returns URL %q", r.GetContainerId(), res.Url)
 		}
+		span.RecordError(err)
 	}()
 	res, err = in.c.Attach(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -330,8 +345,7 @@ func (in *instrumentedService) PullImage(ctx context.Context, r *runtime.PullIma
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
-	ctx, span := tracing.StartSpan(ctx, tracing.Name(criSpanPrefix, "PullImage"))
+	span := tracing.SpanFromContext(ctx)
-	defer span.End()
 	log.G(ctx).Infof("PullImage %q", r.GetImage().GetImage())
 	defer func() {
 		if err != nil {
@@ -340,7 +354,7 @@ func (in *instrumentedService) PullImage(ctx context.Context, r *runtime.PullIma
 			log.G(ctx).Infof("PullImage %q returns image reference %q",
 				r.GetImage().GetImage(), res.GetImageRef())
 		}
-		span.SetStatus(err)
+		span.RecordError(err)
 	}()
 	res, err = in.c.PullImage(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -350,8 +364,6 @@ func (in *instrumentedService) ListImages(ctx context.Context, r *runtime.ListIm
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
-	ctx, span := tracing.StartSpan(ctx, tracing.Name(criSpanPrefix, "ListImages"))
-	defer span.End()
 	log.G(ctx).Tracef("ListImages with filter %+v", r.GetFilter())
 	defer func() {
 		if err != nil {
@@ -360,7 +372,6 @@ func (in *instrumentedService) ListImages(ctx context.Context, r *runtime.ListIm
 			log.G(ctx).Tracef("ListImages with filter %+v returns image list %+v",
 				r.GetFilter(), res.GetImages())
 		}
-		span.SetStatus(err)
 	}()
 	res, err = in.c.ListImages(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -370,8 +381,6 @@ func (in *instrumentedService) ImageStatus(ctx context.Context, r *runtime.Image
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
-	ctx, span := tracing.StartSpan(ctx, tracing.Name(criSpanPrefix, "ImageStatus"))
-	defer span.End()
 	log.G(ctx).Tracef("ImageStatus for %q", r.GetImage().GetImage())
 	defer func() {
 		if err != nil {
@@ -380,7 +389,6 @@ func (in *instrumentedService) ImageStatus(ctx context.Context, r *runtime.Image
 			log.G(ctx).Tracef("ImageStatus for %q returns image status %+v",
 				r.GetImage().GetImage(), res.GetImage())
 		}
-		span.SetStatus(err)
 	}()
 	res, err = in.c.ImageStatus(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -390,8 +398,7 @@ func (in *instrumentedService) RemoveImage(ctx context.Context, r *runtime.Remov
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
-	ctx, span := tracing.StartSpan(ctx, tracing.Name(criSpanPrefix, "RemoveImage"))
+	span := tracing.SpanFromContext(ctx)
-	defer span.End()
 	log.G(ctx).Infof("RemoveImage %q", r.GetImage().GetImage())
 	defer func() {
 		if err != nil {
@@ -399,7 +406,7 @@ func (in *instrumentedService) RemoveImage(ctx context.Context, r *runtime.Remov
 		} else {
 			log.G(ctx).Infof("RemoveImage %q returns successfully", r.GetImage().GetImage())
 		}
-		span.SetStatus(err)
+		span.RecordError(err)
 	}()
 	res, err := in.c.RemoveImage(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)
@@ -409,8 +416,6 @@ func (in *instrumentedService) ImageFsInfo(ctx context.Context, r *runtime.Image
 	if err := in.checkInitialized(); err != nil {
 		return nil, err
 	}
-	ctx, span := tracing.StartSpan(ctx, tracing.Name(criSpanPrefix, "ImageFsInfo"))
-	defer span.End()
 	log.G(ctx).Tracef("ImageFsInfo")
 	defer func() {
 		if err != nil {
@@ -418,7 +423,6 @@ func (in *instrumentedService) ImageFsInfo(ctx context.Context, r *runtime.Image
 		} else {
 			log.G(ctx).Tracef("ImageFsInfo returns filesystem info %+v", res.ImageFilesystems)
 		}
-		span.SetStatus(err)
 	}()
 	res, err = in.c.ImageFsInfo(ctrdutil.WithNamespace(ctx), r)
 	return res, errdefs.ToGRPC(err)

internal/cri/server/container_attach.go

@@ -22,6 +22,7 @@ import (
 	"io"
 
 	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/log"
 	"k8s.io/client-go/tools/remotecommand"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
@@ -31,10 +32,12 @@ import (
 
 // Attach prepares a streaming endpoint to attach to a running container, and returns the address.
 func (c *criService) Attach(ctx context.Context, r *runtime.AttachRequest) (*runtime.AttachResponse, error) {
+	span := tracing.SpanFromContext(ctx)
 	cntr, err := c.containerStore.Get(r.GetContainerId())
 	if err != nil {
 		return nil, fmt.Errorf("failed to find container in store: %w", err)
 	}
+	span.SetAttributes(tracing.Attribute("container.id", cntr.ID))
 	state := cntr.Status.Get().State()
 	if state != runtime.ContainerState_CONTAINER_RUNNING {
 		return nil, fmt.Errorf("container is in %s state", criContainerStateToString(state))

internal/cri/server/container_create.go

@@ -45,6 +45,7 @@ import (
 	"github.com/containerd/containerd/v2/internal/cri/util"
 	"github.com/containerd/containerd/v2/pkg/blockio"
 	"github.com/containerd/containerd/v2/pkg/oci"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/platforms"
 )
 
@@ -55,6 +56,7 @@ func init() {
 
 // CreateContainer creates a new container in the given PodSandbox.
 func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) (_ *runtime.CreateContainerResponse, retErr error) {
+	span := tracing.SpanFromContext(ctx)
 	config := r.GetConfig()
 	log.G(ctx).Debugf("Container config %+v", config)
 	sandboxConfig := r.GetSandboxConfig()
@@ -72,7 +74,10 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 		sandboxID  = cstatus.SandboxID
 		sandboxPid = cstatus.Pid
 	)
-
+	span.SetAttributes(
+		tracing.Attribute("sandbox.id", sandboxID),
+		tracing.Attribute("sandbox.pid", sandboxPid),
+	)
 	// Generate unique id and name for the container and reserve the name.
 	// Reserve the container name to avoid concurrent `CreateContainer` request creating
 	// the same container.
@@ -87,6 +92,10 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	if err = c.containerNameIndex.Reserve(name, id); err != nil {
 		return nil, fmt.Errorf("failed to reserve container name %q: %w", name, err)
 	}
+	span.SetAttributes(
+		tracing.Attribute("container.id", id),
+		tracing.Attribute("container.name", name),
+	)
 	defer func() {
 		// Release the name if the function returns with an error.
 		if retErr != nil {
@@ -112,7 +121,9 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	if err != nil {
 		return nil, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err)
 	}
-
+	span.SetAttributes(
+		tracing.Attribute("container.image.ref", containerdImage.Name()),
+	)
 	start := time.Now()
 
 	// Create container root directory.
@@ -345,6 +356,9 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 
 	containerCreateTimer.WithValues(ociRuntime.Type).UpdateSince(start)
 
+	span.AddEvent("container created",
+		tracing.Attribute("container.create.duration", time.Since(start).String()),
+	)
 	return &runtime.CreateContainerResponse{ContainerId: id}, nil
 }
 

internal/cri/server/container_exec.go

@@ -20,15 +20,18 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
 // Exec prepares a streaming endpoint to execute a command in the container, and returns the address.
 func (c *criService) Exec(ctx context.Context, r *runtime.ExecRequest) (*runtime.ExecResponse, error) {
+	span := tracing.SpanFromContext(ctx)
 	cntr, err := c.containerStore.Get(r.GetContainerId())
 	if err != nil {
 		return nil, fmt.Errorf("failed to find container %q in store: %w", r.GetContainerId(), err)
 	}
+	span.SetAttributes(tracing.Attribute("container.id", cntr.ID))
 	state := cntr.Status.Get().State()
 	if state != runtime.ContainerState_CONTAINER_RUNNING {
 		return nil, fmt.Errorf("container is in %s state", criContainerStateToString(state))

internal/cri/server/container_execsync.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/v2/pkg/oci"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	"k8s.io/client-go/tools/remotecommand"
@@ -271,6 +272,7 @@ func (c *criService) execInternal(ctx context.Context, container containerd.Cont
 // this case, the CRI plugin will still have a goroutine waiting for the exec process
 // to exit and log the exit code, but dockershim won't.
 func (c *criService) execInContainer(ctx context.Context, id string, opts execOptions) (*uint32, error) {
+	span := tracing.SpanFromContext(ctx)
 	// Get container from our container store.
 	cntr, err := c.containerStore.Get(id)
 
@@ -278,6 +280,7 @@ func (c *criService) execInContainer(ctx context.Context, id string, opts execOp
 		return nil, fmt.Errorf("failed to find container %q in store: %w", id, err)
 	}
 	id = cntr.ID
+	span.SetAttributes(tracing.Attribute("container.id", id))
 
 	state := cntr.Status.Get().State()
 	if state != runtime.ContainerState_CONTAINER_RUNNING {

internal/cri/server/container_remove.go

@@ -24,6 +24,7 @@ import (
 
 	containerd "github.com/containerd/containerd/v2/client"
 	containerstore "github.com/containerd/containerd/v2/internal/cri/store/container"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
@@ -31,6 +32,7 @@ import (
 
 // RemoveContainer removes the container.
 func (c *criService) RemoveContainer(ctx context.Context, r *runtime.RemoveContainerRequest) (_ *runtime.RemoveContainerResponse, retErr error) {
+	span := tracing.SpanFromContext(ctx)
 	start := time.Now()
 	ctrID := r.GetContainerId()
 	container, err := c.containerStore.Get(ctrID)
@@ -43,6 +45,7 @@ func (c *criService) RemoveContainer(ctx context.Context, r *runtime.RemoveConta
 		return &runtime.RemoveContainerResponse{}, nil
 	}
 	id := container.ID
+	span.SetAttributes(tracing.Attribute("container.id", id))
 	i, err := container.Container.Info(ctx)
 	if err != nil {
 		if !errdefs.IsNotFound(err) {
@@ -129,6 +132,11 @@ func (c *criService) RemoveContainer(ctx context.Context, r *runtime.RemoveConta
 
 	containerRemoveTimer.WithValues(i.Runtime.Name).UpdateSince(start)
 
+	span.AddEvent("container removed",
+		tracing.Attribute("container.id", container.ID),
+		tracing.Attribute("container.remove.duration", time.Since(start).String()),
+	)
+
 	return &runtime.RemoveContainerResponse{}, nil
 }
 

internal/cri/server/container_start.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"time"
 
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
@@ -38,12 +39,13 @@ import (
 
 // StartContainer starts the container.
 func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContainerRequest) (retRes *runtime.StartContainerResponse, retErr error) {
+	span := tracing.SpanFromContext(ctx)
 	start := time.Now()
 	cntr, err := c.containerStore.Get(r.GetContainerId())
 	if err != nil {
 		return nil, fmt.Errorf("an error occurred when try to find container %q: %w", r.GetContainerId(), err)
 	}
-
+	span.SetAttributes(tracing.Attribute("container.id", cntr.ID))
 	info, err := cntr.Container.Info(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("get container info: %w", err)
@@ -87,6 +89,7 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain
 	if sandbox.Status.Get().State != sandboxstore.StateReady {
 		return nil, fmt.Errorf("sandbox container %q is not running", sandboxID)
 	}
+	span.SetAttributes(tracing.Attribute("sandbox.id", sandboxID))
 
 	// Recheck target container validity in Linux namespace options.
 	if linux := config.GetLinux(); linux != nil {
@@ -190,6 +193,10 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain
 
 	containerStartTimer.WithValues(info.Runtime.Name).UpdateSince(start)
 
+	span.AddEvent("container started",
+		tracing.Attribute("container.start.duration", time.Since(start).String()),
+	)
+
 	return &runtime.StartContainerResponse{}, nil
 }
 

internal/cri/server/container_stop.go

@@ -27,6 +27,7 @@ import (
 	containerstore "github.com/containerd/containerd/v2/internal/cri/store/container"
 	ctrdutil "github.com/containerd/containerd/v2/internal/cri/util"
 	"github.com/containerd/containerd/v2/pkg/protobuf"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 
@@ -36,6 +37,7 @@ import (
 
 // StopContainer stops a running container with a grace period (i.e., timeout).
 func (c *criService) StopContainer(ctx context.Context, r *runtime.StopContainerRequest) (*runtime.StopContainerResponse, error) {
+	span := tracing.SpanFromContext(ctx)
 	start := time.Now()
 	// Get container config from container store.
 	container, err := c.containerStore.Get(r.GetContainerId())
@@ -49,7 +51,7 @@ func (c *criService) StopContainer(ctx context.Context, r *runtime.StopContainer
 		// https://github.com/kubernetes/cri-api/blob/c20fa40/pkg/apis/runtime/v1/api.proto#L67-L68
 		return &runtime.StopContainerResponse{}, nil
 	}
-
+	span.SetAttributes(tracing.Attribute("container.id", container.ID))
 	if err := c.stopContainer(ctx, container, time.Duration(r.GetTimeout())*time.Second); err != nil {
 		return nil, err
 	}
@@ -76,6 +78,8 @@ func (c *criService) StopContainer(ctx context.Context, r *runtime.StopContainer
 
 // stopContainer stops a container based on the container metadata.
 func (c *criService) stopContainer(ctx context.Context, container containerstore.Container, timeout time.Duration) error {
+	span := tracing.SpanFromContext(ctx)
+	start := time.Now()
 	id := container.ID
 	sandboxID := container.SandboxID
 
@@ -199,6 +203,12 @@ func (c *criService) stopContainer(ctx context.Context, container containerstore
 	if err != nil {
 		return fmt.Errorf("an error occurs during waiting for container %q to be killed: %w", id, err)
 	}
+
+	span.AddEvent("container stopped",
+		tracing.Attribute("container.id", id),
+		tracing.Attribute("container.stop.duration", time.Since(start).String()),
+	)
+
 	return nil
 }
 

internal/cri/server/images/image_status.go

@@ -25,7 +25,6 @@ import (
 
 	imagestore "github.com/containerd/containerd/v2/internal/cri/store/image"
 	"github.com/containerd/containerd/v2/internal/cri/util"
-	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 
@@ -37,17 +36,14 @@ import (
 // TODO(random-liu): We should change CRI to distinguish image id and image spec. (See
 // kubernetes/kubernetes#46255)
 func (c *CRIImageService) ImageStatus(ctx context.Context, r *runtime.ImageStatusRequest) (*runtime.ImageStatusResponse, error) {
-	span := tracing.SpanFromContext(ctx)
 	image, err := c.LocalResolve(r.GetImage().GetImage())
 	if err != nil {
 		if errdefs.IsNotFound(err) {
-			span.AddEvent(err.Error())
 			// return empty without error when image not found.
 			return &runtime.ImageStatusResponse{}, nil
 		}
 		return nil, fmt.Errorf("can not resolve %q locally: %w", r.GetImage().GetImage(), err)
 	}
-	span.SetAttributes(tracing.Attribute("image.id", image.ID))
 	// TODO(random-liu): [P0] Make sure corresponding snapshot exists. What if snapshot
 	// doesn't exist?
 

internal/cri/server/sandbox_remove.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 
@@ -30,6 +31,7 @@ import (
 // RemovePodSandbox removes the sandbox. If there are running containers in the
 // sandbox, they should be forcibly removed.
 func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodSandboxRequest) (*runtime.RemovePodSandboxResponse, error) {
+	span := tracing.SpanFromContext(ctx)
 	start := time.Now()
 	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
 	if err != nil {
@@ -44,6 +46,7 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS
 	}
 	// Use the full sandbox id.
 	id := sandbox.ID
+	span.SetAttributes(tracing.Attribute("sandbox.id", id))
 
 	// If the sandbox is still running, not ready, or in an unknown state, forcibly stop it.
 	// Even if it's in a NotReady state, this will close its network namespace, if open.
@@ -110,5 +113,9 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS
 
 	sandboxRemoveTimer.WithValues(sandbox.RuntimeHandler).UpdateSince(start)
 
+	span.AddEvent("pod sandbox removed",
+		tracing.Attribute("sandbox.remove.duration", time.Since(start).String()),
+	)
+
 	return &runtime.RemovePodSandboxResponse{}, nil
 }

internal/cri/server/sandbox_run.go

@@ -39,6 +39,7 @@ import (
 	sandboxstore "github.com/containerd/containerd/v2/internal/cri/store/sandbox"
 	"github.com/containerd/containerd/v2/internal/cri/util"
 	"github.com/containerd/containerd/v2/pkg/netns"
+	"github.com/containerd/containerd/v2/pkg/tracing"
 )
 
 func init() {
@@ -49,6 +50,7 @@ func init() {
 // RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
 // the sandbox is in ready state.
 func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) {
+	span := tracing.SpanFromContext(ctx)
 	config := r.GetConfig()
 	log.G(ctx).Debugf("Sandbox config %+v", config)
 
@@ -59,6 +61,11 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		return nil, errors.New("sandbox config must include metadata")
 	}
 	name := makeSandboxName(metadata)
+
+	span.SetAttributes(
+		tracing.Attribute("sandbox.id", id),
+		tracing.Attribute("sandbox.name", name),
+	)
 	log.G(ctx).WithField("podsandboxid", id).Debugf("generated id for sandbox name %q", name)
 
 	// cleanupErr records the last error returned by the critical cleanup operations in deferred functions,
@@ -172,6 +179,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		//
 		// To simplify this, in the future, we should just remove this case (podNetwork &&
 		// !userNsEnabled) and just keep the other case (podNetwork && userNsEnabled).
+		span.AddEvent("setup pod network")
 		netStart := time.Now()
 		// If it is not in host network namespace then create a namespace and set the sandbox
 		// handle. NetNSPath in sandbox metadata and NetNS is non empty only for non host network
@@ -356,6 +364,10 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 			return nil, fmt.Errorf("failed to setup network for sandbox %q: %w", id, err)
 		}
 		sandboxCreateNetworkTimer.UpdateSince(netStart)
+
+		span.AddEvent("finished pod network setup",
+			tracing.Attribute("pod.network.setup.duration", time.Since(netStart).String()),
+		)
 	}
 
 	// TODO: get rid of this. sandbox object should no longer have Container field.

internal/cri/server/sandbox_stop.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/containerd/containerd/v2/pkg/tracing"
 	"github.com/containerd/log"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 
@@ -32,6 +33,7 @@ import (
 // StopPodSandbox stops the sandbox. If there are any running containers in the
 // sandbox, they should be forcibly terminated.
 func (c *criService) StopPodSandbox(ctx context.Context, r *runtime.StopPodSandboxRequest) (*runtime.StopPodSandboxResponse, error) {
+	span := tracing.SpanFromContext(ctx)
 	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
 	if err != nil {
 		if !errdefs.IsNotFound(err) {
@@ -44,7 +46,7 @@ func (c *criService) StopPodSandbox(ctx context.Context, r *runtime.StopPodSandb
 		// https://github.com/kubernetes/cri-api/blob/c20fa40/pkg/apis/runtime/v1/api.proto#L45-L46
 		return &runtime.StopPodSandboxResponse{}, nil
 	}
-
+	span.SetAttributes(tracing.Attribute("sandbox.id", sandbox.ID))
 	if err := c.stopPodSandbox(ctx, sandbox); err != nil {
 		return nil, err
 	}
@@ -53,12 +55,14 @@ func (c *criService) StopPodSandbox(ctx context.Context, r *runtime.StopPodSandb
 }
 
 func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sandbox) error {
+	span := tracing.SpanFromContext(ctx)
 	// Use the full sandbox id.
 	id := sandbox.ID
 
 	// Stop all containers inside the sandbox. This terminates the container forcibly,
 	// and container may still be created, so production should not rely on this behavior.
 	// TODO(random-liu): Introduce a state in sandbox to avoid future container creation.
+	span.AddEvent("stopping containers in the sandbox")
 	stop := time.Now()
 	containers := c.containerStore.List()
 	for _, container := range containers {
@@ -87,6 +91,10 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa
 
 	sandboxRuntimeStopTimer.WithValues(sandbox.RuntimeHandler).UpdateSince(stop)
 
+	span.AddEvent("sandbox container stopped",
+		tracing.Attribute("sandbox.stop.duration", time.Since(stop).String()),
+	)
+
 	err := c.nri.StopPodSandbox(ctx, &sandbox)
 	if err != nil {
 		log.G(ctx).WithError(err).Errorf("NRI sandbox stop notification failed")
@@ -94,6 +102,7 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa
 
 	// Teardown network for sandbox.
 	if sandbox.NetNS != nil {
+		span.AddEvent("start pod network teardown")
 		netStop := time.Now()
 		// Use empty netns path if netns is not available. This is defined in:
 		// https://github.com/containernetworking/cni/blob/v0.7.0-alpha1/SPEC.md
@@ -109,10 +118,13 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa
 			return fmt.Errorf("failed to remove network namespace for sandbox %q: %w", id, err)
 		}
 		sandboxDeleteNetwork.UpdateSince(netStop)
+
+		span.AddEvent("finished pod network teardown",
+			tracing.Attribute("network.teardown.duration", time.Since(netStop).String()),
+		)
 	}
 
 	log.G(ctx).Infof("TearDown network for sandbox %q successfully", id)
-
 	return nil
 }
 

pkg/tracing/tracing.go

@@ -36,6 +36,14 @@ type StartConfig struct {
 
 type SpanOpt func(config *StartConfig)
 
+// WithAttribute appends attributes to a new created span.
+func WithAttribute(k string, v interface{}) SpanOpt {
+	return func(config *StartConfig) {
+		config.spanOpts = append(config.spanOpts,
+			trace.WithAttributes(Attribute(k, v)))
+	}
+}
+
 // UpdateHTTPClient updates the http client with the necessary otel transport
 func UpdateHTTPClient(client *http.Client, name string) {
 	client.Transport = otelhttp.NewTransport(
@@ -80,8 +88,13 @@ func (s *Span) End() {
 }
 
 // AddEvent adds an event with provided name and options.
-func (s *Span) AddEvent(name string, options ...trace.EventOption) {
+func (s *Span) AddEvent(name string, attributes ...attribute.KeyValue) {
-	s.otelSpan.AddEvent(name, options...)
+	s.otelSpan.AddEvent(name, trace.WithAttributes(attributes...))
+}
+
+// RecordError will record err as an exception span event for this span
+func (s *Span) RecordError(err error, options ...trace.EventOption) {
+	s.otelSpan.RecordError(err, options...)
 }
 
 // SetStatus sets the status of the current span.