Add opt-in support for metric overflow attribute

src/OpenTelemetry/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 ## Unreleased
 
+* **Experimental Feature** Added an opt-in feature to aggregate any metric
+  measurements that were dropped due to reaching the max MetricPoints limit.
+  When this feature is enabled, SDK would aggregate such measurements using a
+  reserved MetricPoint with a single tag with key as `otel.metric.overflow` amd
+  value as `true`. The feature is turned-off by default. You can enable it by
+  setting the `AppContext` switch: `OTel.Dotnet.EmitMetricOverflowAttribute`
+  before setting up the `MeterProvider`.
+
+  ```csharp
+  AppContext.SetSwitch("OTel.Dotnet.EmitMetricOverflowAttribute", true);
+
+  // Setup MeterProvider
+  ```
+
+  ([#4737](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4737))
+
 ## 1.6.0-alpha.1
 
 Released 2023-Jul-12

src/OpenTelemetry/Metrics/AggregatorStore.cs

@@ -24,7 +24,10 @@ internal sealed class AggregatorStore
 {
     private static readonly string MetricPointCapHitFixMessage = "Modify instrumentation to reduce the number of unique key/value pair combinations. Or use Views to drop unwanted tags. Or use MeterProviderBuilder.SetMaxMetricPointsPerMetricStream to set higher limit.";
     private static readonly Comparison<KeyValuePair<string, object>> DimensionComparisonDelegate = (x, y) => x.Key.CompareTo(y.Key);
+    private static bool emitOverflowAttribute;
+
     private readonly object lockZeroTags = new();
+    private readonly object lockOverflowTag = new();
     private readonly HashSet<string> tagKeysInteresting;
     private readonly int tagsKeysInterestingCount;
 
@@ -44,10 +47,12 @@ internal sealed class AggregatorStore
     private readonly UpdateDoubleDelegate updateDoubleCallback;
     private readonly int maxMetricPoints;
     private readonly ExemplarFilter exemplarFilter;
+
     private int metricPointIndex = 0;
     private int batchSize = 0;
     private int metricCapHitMessageLogged;
     private bool zeroTagMetricPointInitialized;
+    private bool overflowTagMetricPointInitialized;
 
     internal AggregatorStore(
         MetricStreamIdentity metricStreamIdentity,
@@ -81,6 +86,21 @@ internal sealed class AggregatorStore
             this.tagKeysInteresting = hs;
             this.tagsKeysInterestingCount = hs.Count;
         }
+
+        // If the switch is not set at all or if it's explicitly set to false
+        if (AppContext.TryGetSwitch("OTel.Dotnet.EmitMetricOverflowAttribute", out bool shouldEmitOverflowAttribute) == false ||
+            shouldEmitOverflowAttribute == false)
+        {
+            emitOverflowAttribute = false;
+        }
+        else if (this.maxMetricPoints > 1)
+        {
+            emitOverflowAttribute = true;
+
+            // Setting metricPointIndex to 1 as we would reserve the metricPoints[1] for overflow attribute.
+            // Newer attributes should be added starting at the index: 2
+            this.metricPointIndex = 1;
+        }
     }
 
     private delegate void UpdateLongDelegate(long value, ReadOnlySpan<KeyValuePair<string, object>> tags);
@@ -197,6 +217,22 @@ private void InitializeZeroTagPointIfNotInitialized()
         }
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private void InitializeOverflowTagPointIfNotInitialized()
+    {
+        if (!this.overflowTagMetricPointInitialized)
+        {
+            lock (this.lockOverflowTag)
+            {
+                if (!this.overflowTagMetricPointInitialized)
+                {
+                    this.metricPoints[1] = new MetricPoint(this, this.aggType, new KeyValuePair<string, object>[] { new("otel.metric.overflow", true) }, this.histogramBounds, this.exponentialHistogramMaxSize, this.exponentialHistogramMaxScale);
+                    this.overflowTagMetricPointInitialized = true;
+                }
+            }
+        }
+    }
+
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     private int LookupAggregatorStore(KeyValuePair<string, object>[] tagKeysAndValues, int length)
     {
@@ -329,12 +365,21 @@ private void UpdateLong(long value, ReadOnlySpan<KeyValuePair<string, object>> t
             var index = this.FindMetricAggregatorsDefault(tags);
             if (index < 0)
             {
-                if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                if (emitOverflowAttribute)
                 {
-                    OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    this.InitializeOverflowTagPointIfNotInitialized();
+                    this.metricPoints[1].Update(value);
+                    return;
                 }
+                else
+                {
+                    if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                    {
+                        OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    }
 
-                return;
+                    return;
+                }
             }
 
             // TODO: can special case built-in filters to be bit faster.
@@ -361,12 +406,21 @@ private void UpdateLongCustomTags(long value, ReadOnlySpan<KeyValuePair<string,
             var index = this.FindMetricAggregatorsCustomTag(tags);
             if (index < 0)
             {
-                if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                if (emitOverflowAttribute)
                 {
-                    OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    this.InitializeOverflowTagPointIfNotInitialized();
+                    this.metricPoints[1].Update(value);
+                    return;
                 }
+                else
+                {
+                    if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                    {
+                        OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    }
 
-                return;
+                    return;
+                }
             }
 
             // TODO: can special case built-in filters to be bit faster.
@@ -393,12 +447,21 @@ private void UpdateDouble(double value, ReadOnlySpan<KeyValuePair<string, object
             var index = this.FindMetricAggregatorsDefault(tags);
             if (index < 0)
             {
-                if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                if (emitOverflowAttribute)
                 {
-                    OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    this.InitializeOverflowTagPointIfNotInitialized();
+                    this.metricPoints[1].Update(value);
+                    return;
                 }
+                else
+                {
+                    if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                    {
+                        OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    }
 
-                return;
+                    return;
+                }
             }
 
             // TODO: can special case built-in filters to be bit faster.
@@ -425,12 +488,21 @@ private void UpdateDoubleCustomTags(double value, ReadOnlySpan<KeyValuePair<stri
             var index = this.FindMetricAggregatorsCustomTag(tags);
             if (index < 0)
             {
-                if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                if (emitOverflowAttribute)
                 {
-                    OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    this.InitializeOverflowTagPointIfNotInitialized();
+                    this.metricPoints[1].Update(value);
+                    return;
                 }
+                else
+                {
+                    if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                    {
+                        OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    }
 
-                return;
+                    return;
+                }
             }
 
             // TODO: can special case built-in filters to be bit faster.

test/OpenTelemetry.Tests/Metrics/AggregatorTest.cs → test/OpenTelemetry.Tests/Metrics/AggregatorTestBase.cs

@@ -1,4 +1,4 @@
-// <copyright file="AggregatorTest.cs" company="OpenTelemetry Authors">
+// <copyright file="AggregatorTestBase.cs" company="OpenTelemetry Authors">
 // Copyright The OpenTelemetry Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,14 +19,25 @@
 
 namespace OpenTelemetry.Metrics.Tests;
 
-public class AggregatorTest
+#pragma warning disable SA1402
+
+public abstract class AggregatorTestBase : IDisposable
 {
     private static readonly Meter Meter = new("testMeter");
     private static readonly Instrument Instrument = Meter.CreateHistogram<long>("testInstrument");
     private static readonly ExplicitBucketHistogramConfiguration HistogramConfiguration = new() { Boundaries = Metric.DefaultHistogramBounds };
     private static readonly MetricStreamIdentity MetricStreamIdentity = new(Instrument, HistogramConfiguration);
+
     private readonly AggregatorStore aggregatorStore = new(MetricStreamIdentity, AggregationType.HistogramWithBuckets, AggregationTemporality.Cumulative, 1024);
 
+    protected AggregatorTestBase(bool emitOverflowAttribute)
+    {
+        if (emitOverflowAttribute)
+        {
+            AppContext.SetSwitch("OTel.Dotnet.EmitMetricOverflowAttribute", true);
+        }
+    }
+
     [Fact]
     public void HistogramDistributeToAllBucketsDefault()
     {
@@ -224,6 +235,11 @@ public void MultiThreadedHistogramUpdateAndSnapShotTest()
         Assert.Equal(200, argsToThread.SumOfDelta + lastDelta);
     }
 
+    public void Dispose()
+    {
+        AppContext.SetSwitch("OTel.Dotnet.EmitMetricOverflowAttribute", false);
+    }
+
     internal static void AssertExponentialBucketsAreCorrect(Base2ExponentialBucketHistogram expectedHistogram, ExponentialHistogramData data)
     {
         Assert.Equal(expectedHistogram.Scale, data.Scale);
@@ -463,3 +479,19 @@ private class ThreadArguments
         public double SumOfDelta;
     }
 }
+
+public class AggregatorTests : AggregatorTestBase
+{
+    public AggregatorTests()
+        : base(false)
+    {
+    }
+}
+
+public class AggregatorTestsWithOverflowAttribute : AggregatorTestBase
+{
+    public AggregatorTestsWithOverflowAttribute()
+        : base(true)
+    {
+    }
+}

test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs → test/OpenTelemetry.Tests/Metrics/MetricApiTestBase.cs

@@ -1,4 +1,4 @@
-// <copyright file="MetricAPITest.cs" company="OpenTelemetry Authors">
+// <copyright file="MetricApiTestBase.cs" company="OpenTelemetry Authors">
 // Copyright The OpenTelemetry Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,7 +24,9 @@
 
 namespace OpenTelemetry.Metrics.Tests;
 
-public class MetricApiTest : MetricTestsBase
+#pragma warning disable SA1402
+
+public abstract class MetricApiTestBase : MetricTestsBase, IDisposable
 {
     private const int MaxTimeToAllowForFlush = 10000;
     private static readonly int NumberOfThreads = Environment.ProcessorCount;
@@ -33,9 +35,14 @@ public class MetricApiTest : MetricTestsBase
     private static readonly int NumberOfMetricUpdateByEachThread = 100000;
     private readonly ITestOutputHelper output;
 
-    public MetricApiTest(ITestOutputHelper output)
+    protected MetricApiTestBase(ITestOutputHelper output, bool emitOverflowAttribute)
     {
         this.output = output;
+
+        if (emitOverflowAttribute)
+        {
+            AppContext.SetSwitch("OTel.Dotnet.EmitMetricOverflowAttribute", true);
+        }
     }
 
     [Fact]
@@ -1518,6 +1525,11 @@ public void UnsupportedMetricInstrument()
         Assert.Empty(exportedItems);
     }
 
+    public void Dispose()
+    {
+        AppContext.SetSwitch("OTel.Dotnet.EmitMetricOverflowAttribute", false);
+    }
+
     private static void CounterUpdateThread<T>(object obj)
         where T : struct, IComparable
     {
@@ -1689,3 +1701,19 @@ private class UpdateThreadArguments<T>
         public T[] ValuesToRecord;
     }
 }
+
+public class MetricApiTest : MetricApiTestBase
+{
+    public MetricApiTest(ITestOutputHelper output)
+        : base(output, false)
+    {
+    }
+}
+
+public class MetricApiTestWithOverflowAttribute : MetricApiTestBase
+{
+    public MetricApiTestWithOverflowAttribute(ITestOutputHelper output)
+        : base(output, true)
+    {
+    }
+}