Add possibility to send telemetry events by data collectors (#4622)

* Edge1 * Edge2 risky way * Send back events * Fixes * More * More * More * Fix all tests * More tests * More * More tests * Extend more * Extend more --------- Co-authored-by: Jakub Chocholowicz <jachocho@microsoft.com>
microsoft · Jul 31, 2023 · f60b983 · f60b983
1 parent 6719005
commit f60b983
Show file tree

Hide file tree

Showing 36 changed files with 1,738 additions and 123 deletions.
diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs
@@ -53,6 +53,11 @@ internal class DataCollectionManager : IDataCollectionManager
     /// </summary>
     private readonly TestPlatformDataCollectionEvents _events;
 
+    /// <summary>
+    /// Telemetry reporter
+    /// </summary>
+    private readonly ITelemetryReporter _telemetryReporter;
+
     /// <summary>
     /// Specifies whether the object is disposed or not.
     /// </summary>
@@ -74,7 +79,7 @@ internal class DataCollectionManager : IDataCollectionManager
     /// <param name="messageSink">
     /// The message Sink.
     /// </param>
-    internal DataCollectionManager(IMessageSink messageSink, IRequestData requestData) : this(new DataCollectionAttachmentManager(), messageSink, new DataCollectionTelemetryManager(requestData))
+    internal DataCollectionManager(IMessageSink messageSink, IRequestData requestData, ITelemetryReporter telemetryReporter) : this(new DataCollectionAttachmentManager(), messageSink, new DataCollectionTelemetryManager(requestData), telemetryReporter)
     {
     }
 
@@ -90,14 +95,15 @@ internal DataCollectionManager(IMessageSink messageSink, IRequestData requestDat
     /// <remarks>
     /// The constructor is not public because the factory method should be used to get instances of this class.
     /// </remarks>
-    protected DataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink, IDataCollectionTelemetryManager dataCollectionTelemetryManager)
+    protected DataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink, IDataCollectionTelemetryManager dataCollectionTelemetryManager, ITelemetryReporter telemetryReporter)
     {
         _attachmentManager = datacollectionAttachmentManager;
         _messageSink = messageSink;
         _events = new TestPlatformDataCollectionEvents();
         _dataCollectorExtensionManager = null;
         RunDataCollectors = new Dictionary<Type, DataCollectorInformation>();
         _dataCollectionTelemetryManager = dataCollectionTelemetryManager;
+        _telemetryReporter = telemetryReporter;
     }
 
     /// <summary>
@@ -133,13 +139,13 @@ private DataCollectorExtensionManager DataCollectorExtensionManager
     /// <returns>
     /// The <see cref="DataCollectionManager"/>.
     /// </returns>
-    public static DataCollectionManager Create(IMessageSink messageSink, IRequestData requestData)
+    public static DataCollectionManager Create(IMessageSink messageSink, IRequestData requestData, ITelemetryReporter telemetryReporter)
     {
         if (Instance == null)
         {
             lock (SyncObject)
             {
-                Instance ??= new DataCollectionManager(messageSink, requestData);
+                Instance ??= new DataCollectionManager(messageSink, requestData, telemetryReporter);
             }
         }
 
@@ -528,7 +534,7 @@ private void LoadAndInitialize(DataCollectorSettings dataCollectorSettings, stri
 
         try
         {
-            dataCollectorInfo.InitializeDataCollector();
+            dataCollectorInfo.InitializeDataCollector(_telemetryReporter);
             TPDebug.Assert(dataCollectorConfig is not null, "dataCollectorConfig is null");
             lock (RunDataCollectors)
             {

diff --git a/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorInformation.cs b/src/Microsoft.TestPlatform.Common/DataCollection/DataCollectorInformation.cs
@@ -110,11 +110,16 @@ internal DataCollectorInformation(ObjectModel.DataCollection.DataCollector dataC
     /// <summary>
     /// Initializes datacollectors.
     /// </summary>
-    internal void InitializeDataCollector()
+    internal void InitializeDataCollector(ITelemetryReporter telemetryReporter)
     {
         UpdateConfigurationElement();
 
         DataCollector.Initialize(ConfigurationElement, Events, DataCollectionSink, Logger, EnvironmentContext);
+
+        if (DataCollector is ITelemetryInitializer telemetryInitializer)
+        {
+            telemetryInitializer.Initialize(telemetryReporter);
+        }
     }
 
     private void UpdateConfigurationElement()

diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs
@@ -55,28 +55,6 @@ internal class DataCollectionRequestHandler : IDataCollectionRequestHandler, IDi
     /// </summary>
     private readonly CancellationTokenSource _cancellationTokenSource;
 
-    /// <summary>
-    /// Initializes a new instance of the <see cref="DataCollectionRequestHandler"/> class.
-    /// </summary>
-    /// <param name="messageSink">
-    /// The message sink.
-    /// </param>
-    /// <param name="requestData">
-    /// The request data.
-    /// </param>
-    protected DataCollectionRequestHandler(IMessageSink messageSink, IRequestData requestData)
-        : this(
-            new SocketCommunicationManager(),
-            messageSink,
-            DataCollectionManager.Create(messageSink, requestData),
-            new DataCollectionTestCaseEventHandler(messageSink),
-            JsonDataSerializer.Instance,
-            new FileHelper(),
-            requestData)
-    {
-        _messageSink = messageSink;
-    }
-
     /// <summary>
     /// Initializes a new instance of the <see cref="DataCollectionRequestHandler"/> class.
     /// </summary>
@@ -159,11 +137,12 @@ protected DataCollectionRequestHandler(IMessageSink messageSink, IRequestData re
                 if (Instance == null)
                 {
                     var requestData = new RequestData();
+                    var telemetryReporter = new TelemetryReporter(requestData, communicationManager, JsonDataSerializer.Instance);
 
                     Instance = new DataCollectionRequestHandler(
                         communicationManager,
                         messageSink,
-                        DataCollectionManager.Create(messageSink, requestData),
+                        DataCollectionManager.Create(messageSink, requestData, telemetryReporter),
                         new DataCollectionTestCaseEventHandler(messageSink),
                         JsonDataSerializer.Instance,
                         new FileHelper(),

diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestSender.cs
@@ -116,7 +116,10 @@ public void SendTestHostLaunched(TestHostLaunchedPayload testHostLaunchedPayload
 
         while (!isDataCollectionStarted)
         {
-            var message = _communicationManager.ReceiveMessage();
+            var rawMessage = _communicationManager.ReceiveRawMessage();
+            TPDebug.Assert(rawMessage is not null, "rawMessage is null");
+
+            var message = !rawMessage.IsNullOrEmpty() ? _dataSerializer.DeserializeMessage(rawMessage) : null;
             TPDebug.Assert(message is not null, "message is null");
 
             EqtTrace.Verbose("DataCollectionRequestSender.SendBeforeTestRunStartAndGetResult: Received message: {0}", message);
@@ -132,6 +135,10 @@ public void SendTestHostLaunched(TestHostLaunchedPayload testHostLaunchedPayload
                 isDataCollectionStarted = true;
                 result = _dataSerializer.DeserializePayload<BeforeTestRunStartResult>(message);
             }
+            else if (message.MessageType == MessageType.TelemetryEventMessage)
+            {
+                runEventsHandler?.HandleRawMessage(rawMessage);
+            }
         }
 
         return result;
@@ -151,7 +158,10 @@ public void SendTestHostLaunched(TestHostLaunchedPayload testHostLaunchedPayload
         // Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification.
         while (!isDataCollectionComplete && !isCancelled)
         {
-            var message = _communicationManager.ReceiveMessage();
+            var rawMessage = _communicationManager.ReceiveRawMessage();
+            TPDebug.Assert(rawMessage is not null, "rawMessage is null");
+
+            var message = !rawMessage.IsNullOrEmpty() ? _dataSerializer.DeserializeMessage(rawMessage) : null;
             TPDebug.Assert(message is not null, "message is null");
 
             EqtTrace.Verbose("DataCollectionRequestSender.SendAfterTestRunStartAndGetResult: Received message: {0}", message);
@@ -167,6 +177,10 @@ public void SendTestHostLaunched(TestHostLaunchedPayload testHostLaunchedPayload
                 result = _dataSerializer.DeserializePayload<AfterTestRunEndResult>(message);
                 isDataCollectionComplete = true;
             }
+            else if (message.MessageType == MessageType.TelemetryEventMessage)
+            {
+                runEventsHandler?.HandleRawMessage(rawMessage);
+            }
         }
 
         return result;

diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/MessageType.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/MessageType.cs
@@ -281,4 +281,8 @@ public static class MessageType
     [ProtocolVersion(Version7, typeof(EditorAttachDebuggerPayload))]
     public const string EditorAttachDebugger2 = "TestExecution.EditorAttachDebugger2";
 
+    /// <summary>
+    /// Telemetry event.
+    /// </summary>
+    public const string TelemetryEventMessage = "TestPlatform.TelemetryEvent";
 }
diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt
@@ -1 +1,2 @@
 #nullable enable
+const Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.MessageType.TelemetryEventMessage = "TestPlatform.TelemetryEvent" -> string!
diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TelemetryReporter.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TelemetryReporter.cs
@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces;
+using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
+
+namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities;
+
+internal class TelemetryReporter : ITelemetryReporter
+{
+    private readonly IRequestData _requestData;
+    private readonly ICommunicationManager _communicationManager;
+    private readonly IDataSerializer _dataSerializer;
+
+    public TelemetryReporter(IRequestData requestData, ICommunicationManager communicationManager, IDataSerializer dataSerializer)
+    {
+        _requestData = requestData;
+        _communicationManager = communicationManager;
+        _dataSerializer = dataSerializer;
+    }
+
+    public void Report(TelemetryEvent telemetryEvent)
+    {
+        if (_requestData.IsTelemetryOptedIn)
+        {
+            string message = _dataSerializer.SerializePayload(MessageType.TelemetryEventMessage, telemetryEvent);
+            _communicationManager.SendRawMessage(message);
+        }
+    }
+}
diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManagerWithDataCollection.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManagerWithDataCollection.cs
@@ -141,6 +141,17 @@ public override int StartTestRun(TestRunCriteria testRunCriteria, IInternalTestR
             DataCollectionRunEventsHandler.Messages.Clear();
         }
 
+        // Push all raw messages
+        if (DataCollectionRunEventsHandler.RawMessages.Count > 0)
+        {
+            foreach (var message in DataCollectionRunEventsHandler.RawMessages)
+            {
+                currentEventHandler.HandleRawMessage(message);
+            }
+
+            DataCollectionRunEventsHandler.RawMessages.Clear();
+        }
+
         return base.StartTestRun(testRunCriteria, currentEventHandler);
     }
 
@@ -190,7 +201,7 @@ public override TestProcessStartInfo UpdateTestProcessStartInfo(TestProcessStart
 }
 
 /// <summary>
-/// Handles Log events and stores them in list. Messages in the list will be logged after test execution begins.
+/// Handles Log and raw messages and stores them in list. Messages in the list will be logged after test execution begins.
 /// </summary>
 internal class DataCollectionRunEventsHandler : ITestMessageEventHandler
 {
@@ -200,13 +211,19 @@ internal class DataCollectionRunEventsHandler : ITestMessageEventHandler
     public DataCollectionRunEventsHandler()
     {
         Messages = new List<Tuple<TestMessageLevel, string?>>();
+        RawMessages = new List<string>();
     }
 
     /// <summary>
     /// Gets the cached messages.
     /// </summary>
     public List<Tuple<TestMessageLevel, string?>> Messages { get; private set; }
 
+    /// <summary>
+    /// Gets the cached raw messages.
+    /// </summary>
+    public List<string> RawMessages { get; private set; }
+
     /// <inheritdoc />
     public void HandleLogMessage(TestMessageLevel level, string? message)
     {
@@ -216,6 +233,6 @@ public void HandleLogMessage(TestMessageLevel level, string? message)
     /// <inheritdoc />
     public void HandleRawMessage(string rawMessage)
     {
-        throw new NotImplementedException();
+        RawMessages.Add(rawMessage);
     }
 }
diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManagerWithDataCollection.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManagerWithDataCollection.cs
@@ -120,6 +120,17 @@ public override TestProcessStartInfo UpdateTestProcessStartInfo(TestProcessStart
             DataCollectionRunEventsHandler.Messages.Clear();
         }
 
+        // Push all raw messages
+        if (DataCollectionRunEventsHandler.RawMessages.Count > 0)
+        {
+            foreach (var message in DataCollectionRunEventsHandler.RawMessages)
+            {
+                eventHandler.HandleRawMessage(message);
+            }
+
+            DataCollectionRunEventsHandler.RawMessages.Clear();
+        }
+
         return base.SetupChannel(sources, runSettings);
     }
 

diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/IInternalTestRunEventsHandler.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/IInternalTestRunEventsHandler.cs
@@ -8,7 +8,7 @@
 namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
 
 /// <summary>
-/// Interface contract for handling internal test run events during run. This interface should have methods similar to <see cref="ITestRunEventsHandler"/> <see cref="ITestRunEventsHandler2"/> <see cref="ITestRunEventsHandler3"/>,
+/// Interface contract for handling internal test run events during run. This interface should have methods similar to <see cref="ITestRunEventsHandler"/> <see cref="ITestRunEventsHandler2"/>,
 /// but only the newest (most broad) version of each method, so that we only operate on the latest interface in the internals, and adapt on the edges.
 /// </summary>
 public interface IInternalTestRunEventsHandler : ITestMessageEventHandler

diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITelemetryEventsHandler.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/ITelemetryEventsHandler.cs
@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
+
+public interface ITelemetryEventsHandler
+{
+    void HandleTelemetryEvent(TelemetryEvent telemetryEvent);
+}
diff --git a/src/Microsoft.TestPlatform.ObjectModel/ITelemetryInitializer.cs b/src/Microsoft.TestPlatform.ObjectModel/ITelemetryInitializer.cs
@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestPlatform.ObjectModel;
+
+/// <summary>
+/// Interface for extensions that choose to send telemetry events
+/// </summary>
+public interface ITelemetryInitializer
+{
+    /// <summary>
+    /// Initializes telemetry reporter
+    /// </summary>
+    void Initialize(ITelemetryReporter telemetryReporter);
+}
diff --git a/src/Microsoft.TestPlatform.ObjectModel/ITelemetryReporter.cs b/src/Microsoft.TestPlatform.ObjectModel/ITelemetryReporter.cs
@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestPlatform.ObjectModel;
+
+/// <summary>
+/// Interface for extensions that choose to send telemetry events
+/// </summary>
+public interface ITelemetryReporter
+{
+    /// <summary>
+    /// Pushes telemetry event into TP
+    /// </summary>
+    void Report(TelemetryEvent telemetryEvent);
+}
diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt
@@ -1 +1,11 @@
 #nullable enable
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITelemetryEventsHandler
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITelemetryEventsHandler.HandleTelemetryEvent(Microsoft.VisualStudio.TestPlatform.ObjectModel.TelemetryEvent! telemetryEvent) -> void
+Microsoft.VisualStudio.TestPlatform.ObjectModel.ITelemetryInitializer
+Microsoft.VisualStudio.TestPlatform.ObjectModel.ITelemetryInitializer.Initialize(Microsoft.VisualStudio.TestPlatform.ObjectModel.ITelemetryReporter! telemetryReporter) -> void
+Microsoft.VisualStudio.TestPlatform.ObjectModel.ITelemetryReporter
+Microsoft.VisualStudio.TestPlatform.ObjectModel.ITelemetryReporter.Report(Microsoft.VisualStudio.TestPlatform.ObjectModel.TelemetryEvent! telemetryEvent) -> void
+Microsoft.VisualStudio.TestPlatform.ObjectModel.TelemetryEvent
+Microsoft.VisualStudio.TestPlatform.ObjectModel.TelemetryEvent.Name.get -> string!
+Microsoft.VisualStudio.TestPlatform.ObjectModel.TelemetryEvent.Properties.get -> System.Collections.Generic.IDictionary<string!, object!>!
+Microsoft.VisualStudio.TestPlatform.ObjectModel.TelemetryEvent.TelemetryEvent(string! name, System.Collections.Generic.IDictionary<string!, object!>! properties) -> void
diff --git a/src/Microsoft.TestPlatform.ObjectModel/TelemetryEvent.cs b/src/Microsoft.TestPlatform.ObjectModel/TelemetryEvent.cs
@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+
+namespace Microsoft.VisualStudio.TestPlatform.ObjectModel;
+
+public sealed class TelemetryEvent
+{
+    /// <summary>
+    /// Initialize an TelemetryEvent
+    /// </summary>
+    /// <param name="name">Telemetry event name</param>
+    /// <param name="properties">Telemetry event properties</param>
+    public TelemetryEvent(string name, IDictionary<string, object> properties)
+    {
+        Name = name;
+        Properties = properties;
+    }
+
+    /// <summary>
+    /// Telemetry event name.
+    /// </summary>
+    [DataMember]
+    public string Name { get; private set; }
+
+    /// <summary>
+    /// Telemetry event properties.
+    /// </summary>
+    [DataMember]
+    public IDictionary<string, object> Properties { get; private set; }
+}