[rel/17.3] Update Newtonsoft.Json to 13.0.1

eng/Versions.props

@@ -2,7 +2,7 @@
 <Project>
   <PropertyGroup>
     <!-- This repo version -->
-    <VersionPrefix>17.3.1</VersionPrefix>
+    <VersionPrefix>17.3.2</VersionPrefix>
     <PreReleaseVersionLabel>release</PreReleaseVersionLabel>
     <!-- Opt-out repo features -->
     <UsingToolXliff>false</UsingToolXliff>
@@ -38,7 +38,7 @@
     <MoqVersion>4.16.1</MoqVersion>
     <MonoOptionsVersion>5.3.0.1</MonoOptionsVersion>
     <McMasterExtensionsCommandLineUtils>2.3.0</McMasterExtensionsCommandLineUtils>
-    <NewtonsoftJsonVersion>9.0.1</NewtonsoftJsonVersion>
+    <NewtonsoftJsonVersion>13.0.1</NewtonsoftJsonVersion>
     <MicrosoftBclJsonSourcesVersion>4.6.0-preview4.19202.2</MicrosoftBclJsonSourcesVersion>
     <NuGetVersioningVersion>4.4.0</NuGetVersioningVersion>
     <NuGetVersion>5.3.0</NuGetVersion>

samples/Microsoft.TestPlatform.Protocol/Microsoft.TestPlatform.Protocol.csproj

@@ -11,6 +11,6 @@
     <!-- / CVE-2017-11770 -->
 
     <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Version="4.3.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="9.0.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
 </Project>

scripts/build.ps1

@@ -641,11 +641,12 @@ function Publish-Package {
     }
 
     # Copy dependency of Microsoft.TestPlatform.TestHostRuntimeProvider
-    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\9.0.1\lib\net45\Newtonsoft.Json.dll"
+    $newtonsoftJsonVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.NewtonsoftJsonVersion
+    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\$newtonsoftJsonVersion\lib\net45\Newtonsoft.Json.dll"
     Write-Verbose "Copy-Item $newtonsoft $fullCLRPackage451Dir -Force"
     Copy-Item $newtonsoft $fullCLRPackage451Dir -Force
 
-    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\9.0.1\lib\netstandard1.0\Newtonsoft.Json.dll"
+    $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\$newtonsoftJsonVersion\lib\netstandard1.0\Newtonsoft.Json.dll"
     Write-Verbose "Copy-Item $newtonsoft $coreCLR20PackageDir -Force"
     Copy-Item $newtonsoft $coreCLR20PackageDir -Force
 

scripts/build.sh

@@ -150,6 +150,7 @@ TPB_LocalizedBuild=$DISABLE_LOCALIZED_BUILD
 TPB_Verbose=$VERBOSE
 TPB_EXTERNALS_VERSION=$(grep TestPlatformExternalsVersion $TP_ROOT_DIR/scripts/build/TestPlatform.Dependencies.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
 TPB_CC_EXTERNALS_VERSION=$(grep MicrosoftInternalCodeCoverageVersion $TP_ROOT_DIR/eng/Versions.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
+TPB_NEWTONSOFT_JSON_VERSION=$(grep NewtonsoftJsonVersion $TP_ROOT_DIR/eng/Versions.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION)
 
 TPB_BRANCH="$(git -C "." rev-parse --abbrev-ref HEAD 2>/dev/null)" || TPB_BRANCH="LOCALBRANCH" # detached HEAD
 TPB_COMMIT="$(git -C "." rev-parse HEAD 2>/dev/null)"              || TPB_COMMIT="LOCALBUILD" # detached HEAD
@@ -436,7 +437,7 @@ function publish_package()
         done
         #*************************************************************************************************************#
 
-        newtonsoft=$TP_PACKAGES_DIR/newtonsoft.json/9.0.1/lib/netstandard1.0/Newtonsoft.Json.dll
+        newtonsoft=$TP_PACKAGES_DIR/newtonsoft.json/$TPB_NEWTONSOFT_JSON_VERSION/lib/netstandard1.0/Newtonsoft.Json.dll
         cp $newtonsoft $packageDir
     done
 

scripts/build/TestPlatform.Dependencies.props

@@ -11,7 +11,7 @@
     <!-- this version also needs to be "statically" readable because the test fixture will inspect this file for the version
     and because during the test `dotnet test` will run and re-build some of the test projects and at that time the version
     from a build parameter would not be available, so I am writing this version from the build.ps1 script to keep it in sync -->
-    <NETTestSdkVersion>17.3.1-dev</NETTestSdkVersion>
+    <NETTestSdkVersion>17.3.2-dev</NETTestSdkVersion>
 
     <!-- These versions are used for running unit tests, and running acceptance tests. They are also used as the default version for projects
     in TestAssets.sln to allow running and debugging tests in that solution directly in VS without having to run them via AcceptanceTests. -->
@@ -57,7 +57,7 @@
     <!-- This version also needs to be updated in src\package\nuspec\TestPlatform.ObjectModel.nuspec -->
     <NuGetFrameworksVersion>5.11.0</NuGetFrameworksVersion>
     <ILAsmPackageVersion>5.0.0</ILAsmPackageVersion>
-    <JsonNetVersion>9.0.1</JsonNetVersion>
+    <JsonNetVersion>13.0.1</JsonNetVersion>
 
     <TestPlatformExternalsVersion>17.3.0-preview-2-32502-021</TestPlatformExternalsVersion>
     <!-- <TestPlatformMSDiaVersion>$(TestPlatformExternalsVersion)</TestPlatformMSDiaVersion> -->

scripts/build/TestPlatform.Settings.targets

@@ -5,7 +5,7 @@
     <!-- This version is read by vsts-prebuild.ps1 and is a base for the current version, this should be updated
     at the start of new iteration to the goal number. This is also used to version the local packages. This version needs to be statically
     readable when we read the file as xml, don't move it to a .props file, unless you change the build server process -->
-    <TPVersionPrefix>17.3.1</TPVersionPrefix>
+    <TPVersionPrefix>17.3.2</TPVersionPrefix>
     <LangVersion>preview</LangVersion>
     <EnableNETAnalyzers>true</EnableNETAnalyzers>
     <AnalysisLevel>preview</AnalysisLevel>

src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs

@@ -96,7 +96,7 @@ public Message DeserializeMessage(string rawMessage)
             // PERF: This is slow, we deserialize the message, and the payload into JToken just to get the header. We then
             // deserialize the data from the JToken, but that is twice as expensive as deserializing the whole object directly into the final object type.
             // We need this for backward compatibility though.
-            return Deserialize<VersionedMessage>(rawMessage);
+            return Deserialize<VersionedMessage>(rawMessage)!;
         }
 
         // PERF: Try grabbing the version and message type from the string directly, we are pretty certain how the message is serialized
@@ -105,7 +105,7 @@ public Message DeserializeMessage(string rawMessage)
         {
             // PERF: If the fast path fails, deserialize into header object that does not have any Payload. When the message type info
             // is at the start of the message, this is also pretty fast. Again, this won't touch the payload.
-            MessageHeader header = JsonConvert.DeserializeObject<MessageHeader>(rawMessage, JsonSettings);
+            MessageHeader header = JsonConvert.DeserializeObject<MessageHeader>(rawMessage, JsonSettings)!;
             version = header.Version;
             messageType = header.MessageType;
         }
@@ -161,21 +161,27 @@ public Message DeserializeMessage(string rawMessage)
         var messageWithRawMessage = (VersionedMessageWithRawMessage)message;
         var rawMessage = messageWithRawMessage.RawMessage;
 
+        if (rawMessage == null)
+        {
+            return default;
+        }
+
         // The deserialized message can still have a version (0 or 1), that should use the old deserializer
         if (payloadSerializer == PayloadSerializerV2)
         {
             // PERF: Fast path is compatibile only with protocol versions that use serializer_2,
             // and this is faster than deserializing via deserializer_2.
             var messageWithPayload = JsonConvert.DeserializeObject<PayloadedMessage<T>>(rawMessage, FastJsonSettings);
-            return messageWithPayload.Payload;
+
+            return messageWithPayload == null ? default : messageWithPayload.Payload;
         }
         else
         {
             // PERF: When payloadSerializer1 was resolved we need to deserialize JToken, and then deserialize that.
             // This is still better than deserializing the JToken in DeserializeMessage because here we know that the payload
             // will actually be used.
             TPDebug.Assert(rawMessage is not null, "rawMessage should not be null");
-            var rawMessagePayload = Deserialize<Message>(rawMessage).Payload;
+            var rawMessagePayload = Deserialize<Message>(rawMessage)?.Payload;
             TPDebug.Assert(rawMessagePayload is not null, "rawMessagePayload should not be null");
             return Deserialize<T>(payloadSerializer, rawMessagePayload);
         }
@@ -286,7 +292,7 @@ private static bool TryGetSubstringUntilDelimiter(string rawMessage, int start,
     /// <param name="version">Version of serializer to be used.</param>
     /// <typeparam name="T">Target type to deserialize.</typeparam>
     /// <returns>An instance of <see cref="T"/>.</returns>
-    public T Deserialize<T>(string json, int version = 1)
+    public T? Deserialize<T>(string json, int version = 1)
     {
         var payloadSerializer = GetPayloadSerializer(version);
         return Deserialize<T>(payloadSerializer, json);
@@ -327,6 +333,9 @@ public string SerializePayload(string? messageType, object? payload, int version
         // so when we resolved the old serializer we should use non-fast path.
         if (DisableFastJson || payloadSerializer == PayloadSerializerV1)
         {
+            if (payload == null)
+                return string.Empty;
+
             var serializedPayload = JToken.FromObject(payload, payloadSerializer);
 
             return version > 1 ?
@@ -387,7 +396,7 @@ private static string Serialize<T>(JsonSerializer serializer, T data)
     /// <param name="serializer">Serializer.</param>
     /// <param name="data">Data to be deserialized.</param>
     /// <returns>Deserialized data.</returns>
-    private static T Deserialize<T>(JsonSerializer serializer, string data)
+    private static T? Deserialize<T>(JsonSerializer serializer, string data)
     {
         using var stringReader = new StringReader(data);
         using var jsonReader = new JsonTextReader(stringReader);
@@ -403,7 +412,7 @@ private static T Deserialize<T>(JsonSerializer serializer, string data)
     /// <returns>Deserialized data.</returns>
     private static T Deserialize<T>(JsonSerializer serializer, JToken jToken)
     {
-        return jToken.ToObject<T>(serializer);
+        return jToken.ToObject<T>(serializer)!;
     }
 
     private static JsonSerializer GetPayloadSerializer(int? version)

src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestCaseConverter.cs

@@ -22,7 +22,7 @@ public override bool CanConvert(Type objectType)
     }
 
     /// <inheritdoc/>
-    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
     {
         var testCase = new TestCase();
 
@@ -38,11 +38,22 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
         // key value pairs.
         foreach (var property in properties.Values<JToken>())
         {
-            var testProperty = property["Key"].ToObject<TestProperty>(serializer);
+            var testProperty = property?["Key"]?.ToObject<TestProperty>(serializer);
+
+            if (testProperty == null)
+            {
+                return null;
+            }
 
             // Let the null values be passed in as null data
-            var token = property["Value"];
+            var token = property?["Value"];
             string? propertyData = null;
+
+            if (token == null)
+            {
+                return null;
+            }
+
             if (token.Type != JTokenType.Null)
             {
                 // If the property is already a string. No need to convert again.
@@ -86,8 +97,13 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
     }
 
     /// <inheritdoc/>
-    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+    public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
     {
+        if (value == null)
+        {
+            return;
+        }
+
         // P2 to P1
         var testCase = (TestCase)value;
 

src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestObjectConverter.cs

@@ -30,7 +30,7 @@ public override bool CanConvert(Type objectType)
     }
 
     /// <inheritdoc/>
-    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
     {
         if (objectType != typeof(List<KeyValuePair<TestProperty, object>>))
         {
@@ -55,12 +55,17 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
         // key value pairs.
         foreach (var property in properties)
         {
-            var testProperty = property["Key"].ToObject<TestProperty>(serializer);
+            var testProperty = property?["Key"]?.ToObject<TestProperty>(serializer);
+
+            if (testProperty == null)
+            {
+                continue;
+            }
 
             // Let the null values be passed in as null data
-            var token = property["Value"];
+            var token = property?["Value"];
             object? propertyData = null;
-            if (token.Type != JTokenType.Null)
+            if (token != null && token.Type != JTokenType.Null)
             {
                 // If the property is already a string. No need to convert again.
                 if (token.Type == JTokenType.String)
@@ -82,7 +87,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
     }
 
     /// <inheritdoc/>
-    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+    public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
     {
         // Create an array of <Property, Value> dictionary
     }
@@ -122,7 +127,7 @@ public override bool CanConvert(Type objectType)
     }
 
     /// <inheritdoc/>
-    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
     {
         if (objectType != typeof(List<KeyValuePair<TestProperty, object>>))
         {
@@ -135,7 +140,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
             return new List<KeyValuePair<TestProperty, object>>();
         }
 
-        var deserializedProperties = serializer.Deserialize<List<KeyValuePair<TestPropertyTemplate, JToken>>>(reader);
+        var deserializedProperties = serializer.Deserialize<List<KeyValuePair<TestPropertyTemplate, JToken>>>(reader)!;
         // Initialize the list capacity to be the number of properties we might add.
         var propertyList = new List<KeyValuePair<TestProperty, object?>>(deserializedProperties.Count);
 
@@ -177,7 +182,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
     }
 
     /// <inheritdoc/>
-    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+    public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
     {
         // Create an array of <Property, Value> dictionary
     }

src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestResultConverter.cs

@@ -22,11 +22,11 @@ public override bool CanConvert(Type objectType)
     }
 
     /// <inheritdoc/>
-    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
     {
         var data = JObject.Load(reader);
 
-        var testCase = data["TestCase"].ToObject<TestCase>(serializer);
+        var testCase = data["TestCase"]!.ToObject<TestCase>(serializer)!;
         var testResult = new TestResult(testCase);
 
         // Add attachments for the result
@@ -35,9 +35,9 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
         {
             foreach (var attachment in attachments.Values<JToken>())
             {
-                if (attachment.Type != JTokenType.Null)
+                if (attachment!.Type != JTokenType.Null)
                 {
-                    testResult.Attachments.Add(attachment.ToObject<AttachmentSet>(serializer));
+                    testResult.Attachments.Add(attachment!.ToObject<AttachmentSet>(serializer)!);
                 }
             }
         }
@@ -48,14 +48,14 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
         {
             foreach (var message in messages.Values<JToken>())
             {
-                if (message.Type != JTokenType.Null)
+                if (message!.Type != JTokenType.Null)
                 {
-                    testResult.Messages.Add(message.ToObject<TestResultMessage>(serializer));
+                    testResult.Messages.Add(message.ToObject<TestResultMessage>(serializer)!);
                 }
             }
         }
 
-        JToken properties = data["Properties"];
+        JToken properties = data["Properties"]!;
         if (properties == null || !properties.HasValues)
         {
             return testResult;
@@ -65,12 +65,12 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
         // key value pairs.
         foreach (var property in properties.Values<JToken>())
         {
-            var testProperty = property["Key"].ToObject<TestProperty>(serializer);
+            var testProperty = property!["Key"]!.ToObject<TestProperty>(serializer)!;
 
             // Let the null values be passed in as null data
             var token = property["Value"];
             string? propertyData = null;
-            if (token.Type != JTokenType.Null)
+            if (token!.Type != JTokenType.Null)
             {
                 // If the property is already a string. No need to convert again.
                 if (token.Type == JTokenType.String)
@@ -115,8 +115,13 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
     }
 
     /// <inheritdoc/>
-    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+    public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
     {
+        if (value == null)
+        {
+            return;
+        }
+
         // P2 to P1
         var testResult = (TestResult)value;
 

src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestRunStatisticsConverter.cs

@@ -21,13 +21,13 @@ public override bool CanConvert(Type objectType)
     }
 
     /// <inheritdoc/>
-    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
     {
         return serializer.Deserialize<TestRunStatistics>(reader);
     }
 
     /// <inheritdoc/>
-    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+    public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
     {
         serializer.Serialize(writer, value);
     }

src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorAppDomain.cs

@@ -157,7 +157,8 @@ public async Task<ICollection<AttachmentSet>> ProcessAttachmentSetsAsync(XmlElem
         cancellationToken.Register(() => _wrapper.CancelProcessAttachment());
         _processAttachmentSetsLogger = logger;
         _progressReporter = progressReporter;
-        return JsonDataSerializer.Instance.Deserialize<AttachmentSet[]>(await Task.Run(() => _wrapper.ProcessAttachment(configurationElement.OuterXml, JsonDataSerializer.Instance.Serialize(attachments.ToArray()))).ConfigureAwait(false));
+        var result = await Task.Run(() => _wrapper.ProcessAttachment(configurationElement.OuterXml, JsonDataSerializer.Instance.Serialize(attachments.ToArray()))).ConfigureAwait(false);
+        return JsonDataSerializer.Instance.Deserialize<AttachmentSet[]>(result)!;
     }
 
     public void Dispose()