From f20c49311d019580eaef59f3718fa731b45225d8 Mon Sep 17 00:00:00 2001 From: Swagat Bora Date: Tue, 13 Dec 2022 20:51:19 +0000 Subject: [PATCH 1/2] Update tracing documentation to add details about manual instrumentation Signed-off-by: Swagat Bora --- docs/tracing.md | 99 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/docs/tracing.md b/docs/tracing.md index 8e00dfd5e..9451a60a8 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -85,3 +85,102 @@ func clientWithTrace() error { defer span.End() ... } +``` +## Manual instrumentation + +OpenTelemetry provides language specific [API](https://pkg.go.dev/go.opentelemetry.io/otel) libraries to instrument parts of your application that are not covered by automatic instrumentation. + +In Containerd, a thin wrapper library defined in `tracing/tracing.go` provides additional functionality and makes it easier to use the OpenTelemetry API for manual instrumentation. + +### Creating a new span + +To create a new span, use the `tracing.StartSpan()` method. You should already have a global TracerProvider set by configuring `io.containerd.tracing.processor.v1.otlp` plugin, else this will only create an instance of a NoopSpan{}. + +``` +func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) error { + ctx, span := tracing.StartSpan(ctx, + tracing.Name(criSpanPrefix, "CreateContainer") // name of the span + tracing.WithAttribute("sandbox.id",r.GetPodSandboxId(), //attributes to be added to the span + ) + defer span.End() // end the span once the function returns + ... +} +``` +Mark the span complete at the end of workflow by calling `Span.End()`. In the above example, we use 'defer' to ensure that the span is properly closed and its duration is recorded. + +### Adding Attributes to a span + +You can add additional attributes to the span using `Span.SetAttributes()`. Attributes can be added during span creation (by passing `tracing.WithAttribute()` to tracing.StartSpan()) or at any other time during the lifecycle of a span before it has completed. + +``` +func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) error { + ctx, span := tracing.StartSpan(ctx, + tracing.Name(criSpanPrefix, "CreateContainer") + tracing.WithAttribute("sandbox.id",r.GetPodSandboxId(), + ) + defer span.End() + ... + containerId := util.GenerateID() + containerName := makeContainerName(metadata, sandboxConfig.GetMetadata()) + + //Add new attributes to the existing span + span.SetAttributes( + tracing.Attribute("container.id", containerId), + tracing.Attribute("container.name", containerName), + ) + ... +} +``` +### Adding an Event to a span +Use `Span.AddEvent()` to add an event to an existing span. A [span event](https://opentelemetry.io/docs/instrumentation/go/manual/#events) is a specific occurrence within a span, such as the completion of an operation or the occurrence of an error. Span events can be used to provide additional information about the operation represented by the span, and can be used for debugging or performance analysis. + +The below example shows how we can add an event to the span to mark the execution of an NRI hook. +``` +func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) error { + span := tracing.SpanFromContext(ctx) // get the current span from context + ... + ... + if c.nri.isEnabled() { + // Add an event to mark start of an NRI api call + span.AddEvent("start NRI postCreateContainer request") + + err = c.nri.postCreateContainer(ctx, &sandbox, &container) + if err != nil { + span.RecordError("NRI postCreateContainer request failed") //record error + log.G(ctx).WithError(err).Errorf("NRI post-create notification failed") + } + + // Add an event to mark completion of the request + span.AddEvent("finished NRI postCreateContainer request") + } + ... + // You can also add additional attributes to an event + span.AddEvent("container created", + tracing.Attribute("container.create.duration", time.Since(start).String()), + ) + + return &runtime.CreateContainerResponse{ContainerId: id}, nil +} +``` +## Naming Convention + +OpenTelemetry maintains a set of recommended [semantic conventions](https://opentelemetry.io/docs/reference/specification/overview/#semantic-conventions) for different types of telemetry data, such as traces and metrics, to help users of the OpenTelemetry libraries and tools to collect and use telemetry data in a consistent and interoperable way. + +Manually instrumented spans in Containerd follow the conventions defined for [Spans](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/) and [Attributes](https://opentelemetry.io/docs/reference/specification/common/attribute-naming/) + +### Span Names +* Dot-separated notation. +* Span Names may include relative path to the package. +* Span Names should include a name that represents the specific component or service performing the operation. +* For example: "pkg.cri.sbserver.CreateContainer" + * "pkg.cri.sbserver" - relative path to the package + * "CreateContainer" - describes the operation that is traced + +### Attribute Names +* Lower-case. +* Dot-separated notation. +* Use a namespace based representation. +* For example: "http.method.get" , http.method.post". + * "http" - represents the general category of the attribute. + * "method" - specific aspect or property of the attribute. + * "get" - additional detail or context. \ No newline at end of file From 88b4cc659ac863a32eb8e4f132f2a4dca1c46cb2 Mon Sep 17 00:00:00 2001 From: Swagat Bora Date: Wed, 22 Feb 2023 20:25:07 +0000 Subject: [PATCH 2/2] address review comments Signed-off-by: Swagat Bora --- docs/tracing.md | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/tracing.md b/docs/tracing.md index 9451a60a8..97fd4bad6 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -96,7 +96,7 @@ In Containerd, a thin wrapper library defined in `tracing/tracing.go` provides a To create a new span, use the `tracing.StartSpan()` method. You should already have a global TracerProvider set by configuring `io.containerd.tracing.processor.v1.otlp` plugin, else this will only create an instance of a NoopSpan{}. -``` +```go func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) error { ctx, span := tracing.StartSpan(ctx, tracing.Name(criSpanPrefix, "CreateContainer") // name of the span @@ -112,7 +112,7 @@ Mark the span complete at the end of workflow by calling `Span.End()`. In the ab You can add additional attributes to the span using `Span.SetAttributes()`. Attributes can be added during span creation (by passing `tracing.WithAttribute()` to tracing.StartSpan()) or at any other time during the lifecycle of a span before it has completed. -``` +```go func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) error { ctx, span := tracing.StartSpan(ctx, tracing.Name(criSpanPrefix, "CreateContainer") @@ -135,7 +135,7 @@ func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) err Use `Span.AddEvent()` to add an event to an existing span. A [span event](https://opentelemetry.io/docs/instrumentation/go/manual/#events) is a specific occurrence within a span, such as the completion of an operation or the occurrence of an error. Span events can be used to provide additional information about the operation represented by the span, and can be used for debugging or performance analysis. The below example shows how we can add an event to the span to mark the execution of an NRI hook. -``` +```go func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) error { span := tracing.SpanFromContext(ctx) // get the current span from context ... @@ -146,7 +146,6 @@ func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) err err = c.nri.postCreateContainer(ctx, &sandbox, &container) if err != nil { - span.RecordError("NRI postCreateContainer request failed") //record error log.G(ctx).WithError(err).Errorf("NRI post-create notification failed") } @@ -162,6 +161,34 @@ func CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) err return &runtime.CreateContainerResponse{ContainerId: id}, nil } ``` + +### Recording errors in a span +You can use `Span.RecordError()` to record an error as an exception span event for this span. The `RecordError` function does not automatically set a span’s status to Error, so if you wish to consider the span tracking a failed operation you should use `Span.SetStatus(err error)` to record the error and also set the span status to Error. + +To record an error: +```go +span := tracing.SpanFromContext(ctx) +defer span.End() +... +err = c.nri.postCreateContainer(ctx, &sandbox, &container) +if err != nil { + span.RecordError(err) //record error + log.G(ctx).WithError(err).Errorf("NRI post-create notification failed") +} +``` + +To record an error and also set the span status: +```go +span := tracing.SpanFromContext(ctx) +defer span.End() +... +err = c.nri.postCreateContainer(ctx, &sandbox, &container) +if err != nil { + span.SetStatus(err) //record error and set status + log.G(ctx).WithError(err).Errorf("NRI post-create notification failed") +} +``` + ## Naming Convention OpenTelemetry maintains a set of recommended [semantic conventions](https://opentelemetry.io/docs/reference/specification/overview/#semantic-conventions) for different types of telemetry data, such as traces and metrics, to help users of the OpenTelemetry libraries and tools to collect and use telemetry data in a consistent and interoperable way.