[release 1.7] Add support for configuring otel from env and config deprecation notice

RELEASES.md

@@ -394,6 +394,9 @@ The deprecated properties in [`config.toml`](./docs/cri/config.md) are shown in
 |`[plugins."io.containerd.grpc.v1.cri".registry]`                      | `auths`                      | containerd v1.3     | containerd v2.0            | Use [`ImagePullSecrets`](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). See also [#8228](https://github.com/containerd/containerd/issues/8228). |
 |`[plugins."io.containerd.grpc.v1.cri".registry]`                      | `configs`                    | containerd v1.5     | containerd v2.0            | Use [`config_path`](./docs/hosts.md)            |
 |`[plugins."io.containerd.grpc.v1.cri".registry]`                      | `mirrors`                    | containerd v1.5     | containerd v2.0            | Use [`config_path`](./docs/hosts.md)            |
+|`[plugins."io.containerd.tracing.processor.v1.otlp"]`                 | `endpoint`, `protocol`, `insecure` | containerd v1.6.29 | containerd v2.0       | Use [OTLP environment variables](https://opentelemetry.io/docs/specs/otel/protocol/exporter/), e.g. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_SDK_DISABLED    |
+|`[plugins."io.containerd.internal.v1.tracing"]`                       | `service_name`, `sampling_ratio`   | containerd v1.6.29 | containerd v2.0       | Instead use [OTel environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/), e.g. OTEL_SERVICE_NAME, OTEL_TRACES_SAMPLER*  |
+
 
 > **Note**
 >

pkg/deprecation/deprecation.go

@@ -53,6 +53,10 @@ const (
 	RuntimeRuncV1 Warning = Prefix + "runtime-runc-v1"
 	// CRICRIUPath is a warning for the use of the `CriuPath` property
 	CRICRIUPath Warning = Prefix + "cri-criu-path"
+	// OTLPTracingConfig is a warning for the use of the `otlp` property
+	TracingOTLPConfig Warning = Prefix + "tracing-processor-config"
+	// TracingServiceConfig is a warning for the use of the `tracing` property
+	TracingServiceConfig Warning = Prefix + "tracing-service-config"
 )
 
 var messages = map[Warning]string{
@@ -82,6 +86,10 @@ var messages = map[Warning]string{
 	RuntimeRuncV1:   "The `io.containerd.runc.v1` runtime is deprecated since containerd v1.4 and removed in containerd v2.0. Use the `io.containerd.runc.v2` runtime instead.",
 	CRICRIUPath: "The `CriuPath` property of `[plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.*.options]` is deprecated since containerd v1.7 and will be removed in containerd v2.0. " +
 		"Use a criu binary in $PATH instead.",
+	TracingOTLPConfig: "The `otlp` property of `[plugins.\"io.containerd.tracing.processor.v1\".otlp]` is deprecated since containerd v1.6 and will be removed in containerd v2.0." +
+		"Use OTLP environment variables instead: https://opentelemetry.io/docs/specs/otel/protocol/exporter/",
+	TracingServiceConfig: "The `tracing` property of `[plugins.\"io.containerd.internal.v1\".tracing]` is deprecated since containerd v1.6 and will be removed in containerd v2.0." +
+		"Use OTEL environment variables instead: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/",
 }
 
 // Valid checks whether a given Warning is valid

tracing/plugin/otlp.go

@@ -20,49 +20,81 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"net/url"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/pkg/deprecation"
 	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/services/warning"
 	"github.com/containerd/containerd/tracing"
 	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
 	"go.opentelemetry.io/otel/propagation"
-	"go.opentelemetry.io/otel/sdk/resource"
 	"go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
 )
 
 const exporterPlugin = "otlp"
 
+// OTEL and OTLP standard env vars
+// See https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
+const (
+	sdkDisabledEnv = "OTEL_SDK_DISABLED"
+
+	otlpEndpointEnv       = "OTEL_EXPORTER_OTLP_ENDPOINT"
+	otlpTracesEndpointEnv = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
+	otlpProtocolEnv       = "OTEL_EXPORTER_OTLP_PROTOCOL"
+	otlpTracesProtocolEnv = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"
+
+	otelTracesExporterEnv = "OTEL_TRACES_EXPORTER"
+	otelServiceNameEnv    = "OTEL_SERVICE_NAME"
+)
+
 func init() {
 	plugin.Register(&plugin.Registration{
 		ID:     exporterPlugin,
 		Type:   plugin.TracingProcessorPlugin,
 		Config: &OTLPConfig{},
 		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
-			cfg := ic.Config.(*OTLPConfig)
-			if cfg.Endpoint == "" {
-				return nil, fmt.Errorf("no OpenTelemetry endpoint: %w", plugin.ErrSkipPlugin)
+			if err := warnOTLPConfig(ic); err != nil {
+				return nil, err
+			}
+			if err := checkDisabled(); err != nil {
+				return nil, err
 			}
-			exp, err := newExporter(ic.Context, cfg)
+
+			// If OTEL_TRACES_EXPORTER is set, it must be "otlp"
+			if v := os.Getenv(otelTracesExporterEnv); v != "" && v != "otlp" {
+				return nil, fmt.Errorf("unsupported traces exporter %q: %w", v, errdefs.ErrInvalidArgument)
+			}
+
+			exp, err := newExporter(ic.Context)
 			if err != nil {
 				return nil, err
 			}
 			return trace.NewBatchSpanProcessor(exp), nil
 		},
 	})
 	plugin.Register(&plugin.Registration{
-		ID:       "tracing",
-		Type:     plugin.InternalPlugin,
-		Requires: []plugin.Type{plugin.TracingProcessorPlugin},
-		Config:   &TraceConfig{ServiceName: "containerd", TraceSamplingRatio: 1.0},
+		ID:     "tracing",
+		Type:   plugin.InternalPlugin,
+		Config: &TraceConfig{},
+		Requires: []plugin.Type{
+			plugin.TracingProcessorPlugin,
+		},
 		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			if err := warnTraceConfig(ic); err != nil {
+				return nil, err
+			}
+			if err := checkDisabled(); err != nil {
+				return nil, err
+			}
+
 			//get TracingProcessorPlugin which is a dependency
 			plugins, err := ic.GetByType(plugin.TracingProcessorPlugin)
 			if err != nil {
@@ -82,7 +114,7 @@ func init() {
 				proc := p.(trace.SpanProcessor)
 				procs = append(procs, proc)
 			}
-			return newTracer(ic.Context, ic.Config.(*TraceConfig), procs)
+			return newTracer(ic.Context, procs)
 		},
 	})
 
@@ -92,106 +124,137 @@ func init() {
 
 // OTLPConfig holds the configurations for the built-in otlp span processor
 type OTLPConfig struct {
-	Endpoint string `toml:"endpoint"`
-	Protocol string `toml:"protocol"`
-	Insecure bool   `toml:"insecure"`
+	Endpoint string `toml:"endpoint,omitempty"`
+	Protocol string `toml:"protocol,omitempty"`
+	Insecure bool   `toml:"insecure,omitempty"`
 }
 
 // TraceConfig is the common configuration for open telemetry.
 type TraceConfig struct {
-	ServiceName        string  `toml:"service_name"`
-	TraceSamplingRatio float64 `toml:"sampling_ratio"`
+	ServiceName        string  `toml:"service_name,omitempty"`
+	TraceSamplingRatio float64 `toml:"sampling_ratio,omitempty"`
 }
 
-type closer struct {
-	close func() error
+func checkDisabled() error {
+	v := os.Getenv(sdkDisabledEnv)
+	if v != "" {
+		disable, err := strconv.ParseBool(v)
+		if err != nil {
+			return fmt.Errorf("invalid value for %s: %w: %w", sdkDisabledEnv, err, errdefs.ErrInvalidArgument)
+		}
+		if disable {
+			return fmt.Errorf("%w: tracing disabled by env %s=%s", plugin.ErrSkipPlugin, sdkDisabledEnv, v)
+		}
+	}
+
+	if os.Getenv(otlpEndpointEnv) == "" && os.Getenv(otlpTracesEndpointEnv) == "" {
+		return fmt.Errorf("%w: tracing endpoint not configured", plugin.ErrSkipPlugin)
+	}
+	return nil
 }
 
-func (c *closer) Close() error {
-	return c.close()
+type closerFunc func() error
+
+func (f closerFunc) Close() error {
+	return f()
 }
 
 // newExporter creates an exporter based on the given configuration.
 //
 // The default protocol is http/protobuf since it is recommended by
 // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.8.0/specification/protocol/exporter.md#specify-protocol.
-func newExporter(ctx context.Context, cfg *OTLPConfig) (*otlptrace.Exporter, error) {
+func newExporter(ctx context.Context) (*otlptrace.Exporter, error) {
 	const timeout = 5 * time.Second
 
+	v := os.Getenv(otlpTracesProtocolEnv)
+	if v == "" {
+		v = os.Getenv(otlpProtocolEnv)
+	}
+
 	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-
-	if cfg.Protocol == "http/protobuf" || cfg.Protocol == "" {
-		u, err := url.Parse(cfg.Endpoint)
-		if err != nil {
-			return nil, fmt.Errorf("OpenTelemetry endpoint %q %w : %v", cfg.Endpoint, errdefs.ErrInvalidArgument, err)
-		}
-		opts := []otlptracehttp.Option{
-			otlptracehttp.WithEndpoint(u.Host),
-		}
-		if u.Scheme == "http" {
-			opts = append(opts, otlptracehttp.WithInsecure())
-		}
-		return otlptracehttp.New(ctx, opts...)
-	} else if cfg.Protocol == "grpc" {
-		opts := []otlptracegrpc.Option{
-			otlptracegrpc.WithEndpoint(cfg.Endpoint),
-		}
-		if cfg.Insecure {
-			opts = append(opts, otlptracegrpc.WithInsecure())
-		}
-		return otlptracegrpc.New(ctx, opts...)
+	switch v {
+	case "", "http/protobuf":
+		return otlptracehttp.New(ctx)
+	case "grpc":
+		return otlptracegrpc.New(ctx)
+	default:
+		// Other protocols such as "http/json" are not supported.
+		return nil, fmt.Errorf("OpenTelemetry protocol %q : %w", v, errdefs.ErrNotImplemented)
 	}
-	// Other protocols such as "http/json" are not supported.
-	return nil, fmt.Errorf("OpenTelemetry protocol %q : %w", cfg.Protocol, errdefs.ErrNotImplemented)
 }
 
 // newTracer configures protocol-agonostic tracing settings such as
 // its sampling ratio and returns io.Closer.
 //
 // Note that this function sets process-wide tracing configuration.
-func newTracer(ctx context.Context, config *TraceConfig, procs []trace.SpanProcessor) (io.Closer, error) {
-
-	res, err := resource.New(ctx,
-		resource.WithHost(),
-		resource.WithAttributes(
-			// Service name used to displace traces in backends
-			semconv.ServiceNameKey.String(config.ServiceName),
-		),
-	)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create resource: %w", err)
+func newTracer(ctx context.Context, procs []trace.SpanProcessor) (io.Closer, error) {
+	// Let otel configure the service name from env
+	if os.Getenv(otelServiceNameEnv) == "" {
+		os.Setenv(otelServiceNameEnv, "containerd")
 	}
 
-	sampler := trace.ParentBased(trace.TraceIDRatioBased(config.TraceSamplingRatio))
-
-	opts := []trace.TracerProviderOption{
-		trace.WithSampler(sampler),
-		trace.WithResource(res),
-	}
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
 
+	opts := make([]trace.TracerProviderOption, 0, len(procs))
 	for _, proc := range procs {
 		opts = append(opts, trace.WithSpanProcessor(proc))
 	}
-
 	provider := trace.NewTracerProvider(opts...)
-
 	otel.SetTracerProvider(provider)
 
-	otel.SetTextMapPropagator(propagators())
+	return closerFunc(func() error {
+		return provider.Shutdown(ctx)
+	}), nil
+}
 
-	return &closer{close: func() error {
-		for _, p := range procs {
-			if err := p.Shutdown(ctx); err != nil {
-				return err
-			}
-		}
+func warnTraceConfig(ic *plugin.InitContext) error {
+	ctx := ic.Context
+	cfg := ic.Config.(*TraceConfig)
+	var warn bool
+	if cfg.ServiceName != "" {
+		warn = true
+	}
+	if cfg.TraceSamplingRatio != 0 {
+		warn = true
+	}
+
+	if !warn {
 		return nil
-	}}, nil
+	}
 
+	wp, err := ic.Get(plugin.WarningPlugin)
+	if err != nil {
+		return err
+	}
+	ws := wp.(warning.Service)
+	ws.Emit(ctx, deprecation.TracingServiceConfig)
+	return nil
 }
 
-// Returns a composite TestMap propagator
-func propagators() propagation.TextMapPropagator {
-	return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
+func warnOTLPConfig(ic *plugin.InitContext) error {
+	ctx := ic.Context
+	cfg := ic.Config.(*OTLPConfig)
+	var warn bool
+	if cfg.Endpoint != "" {
+		warn = true
+	}
+	if cfg.Protocol != "" {
+		warn = true
+	}
+	if cfg.Insecure {
+		warn = true
+	}
+
+	if !warn {
+		return nil
+	}
+
+	wp, err := ic.Get(plugin.WarningPlugin)
+	if err != nil {
+		return err
+	}
+	ws := wp.(warning.Service)
+	ws.Emit(ctx, deprecation.TracingOTLPConfig)
+	return nil
 }