fix deadlog in BaseTest (#1541)

* fix deadlog in BaseTest * remove PackageReference * fix path * copy from Agent.TempDirectory * new location for test results "VSTestResultsDirectory"
coverlet-coverage · Oct 11, 2023 · 89f2f3e · 89f2f3e
1 parent 473777c
commit 89f2f3e
Show file tree

Hide file tree

Showing 6 changed files with 119 additions and 84 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -25,4 +25,14 @@
     <Deterministic>true</Deterministic>
   </PropertyGroup>
 
+  <ItemGroup>
+    <VSTestLogger Include="trx%3BLogFileName=TestResults-$(TargetFramework)-$(MSBuildProjectName).trx" />
+    <VSTestLogger Include="html%3BLogFileName=TestResults-$(TargetFramework)-$(MSBuildProjectName).html" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <VSTestResultsDirectory>$(RepoRoot)/artifacts/tests</VSTestResultsDirectory>
+    <VSTestLogger>@(VSTestLogger)</VSTestLogger>
+  </PropertyGroup>
+
 </Project>
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -10,6 +10,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
     <PackageVersion Include="Microsoft.Build.Utilities.Core" Version="17.5.0"/>
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="2.10.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyModel" Version="7.0.0" />

diff --git a/eng/publish-coverlet-result-files.yml b/eng/publish-coverlet-result-files.yml
@@ -1,30 +1,21 @@
 steps:
-- task: Powershell@2
-  displayName: Prepare log files to Upload
-  inputs:
-    targetType: inline
-    pwsh: true
-    script: |
-      New-Item -ItemType Directory "$(Build.SourcesDirectory)/artifacts/TestLogs/"
-      function Copy-FileKeepPath {
-      param (
-          $Filter,$FileToCopy,$Destination,$StartIndex
-          )
-      Get-ChildItem -Path $FileToCopy -Filter $Filter -Recurse -File | ForEach-Object {
-          $fileName = $_.FullName
-          $newDestination=Join-Path -Path $Destination -ChildPath $fileName.Substring($StartIndex)
-          $folder=Split-Path -Path $newDestination -Parent
-          if (!(Test-Path -Path $folder)) {New-Item $folder -Type Directory -Force -Verbose}
-          Copy-Item -Path $fileName -Destination $newDestination -Recurse -Force -Verbose
-          }
-      }
-      $logfiles = "coverage.opencover.xml", "coverage.cobertura.xml", "coverage.json", "log.txt", "log.datacollector.*.txt", "log.host.*.txt"
-      foreach ($logfile in $logfiles ) {
-      Copy-FileKeepPath -FileToCopy "$(Build.SourcesDirectory)/test/*" -des "$(Build.SourcesDirectory)/artifacts/TestLogs/" -filter $logfile -startIndex "$(Build.SourcesDirectory)/test/coverlet.integration.tests/bin/".Length
-      }
-
+- task: CopyFiles@2
+  displayName: Copy tests results
   continueOnError: true
   condition: always()
+  inputs:
+    SourceFolder: '$(Build.SourcesDirectory)'
+    Contents: |
+       **/*.trx
+       **/*.html
+       **/*.binlog
+       **/coverage.opencover.xml
+       **/coverage.cobertura.xml
+       **/coverage.json
+       **/log.txt
+       **/log.datacollector.*.txt
+       **/log.host.*.txt
+    TargetFolder: '$(Build.SourcesDirectory)/artifacts/TestLogs'
 
 - task: CopyFiles@2
   displayName: Copy trx files

diff --git a/test/coverlet.core.tests/coverlet.core.tests.csproj b/test/coverlet.core.tests/coverlet.core.tests.csproj
@@ -8,29 +8,34 @@
     <MSBuildWarningsAsMessages>NU1702</MSBuildWarningsAsMessages>
     <!--For test TestInstrument_NetstandardAwareAssemblyResolver_PreserveCompilationContext-->
     <PreserveCompilationContext>true</PreserveCompilationContext>
+    <ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="LinqKit.Microsoft.EntityFrameworkCore" />
-    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
-    <PackageReference Include="Moq" />
-    <PackageReference Include="ReportGenerator.Core" />
-    <PackageReference Include="System.Linq.Async"  />
-    <PackageReference Include="Tmds.ExecFunction" />
-    <PackageReference Include="xunit" />
-    <PackageReference Include="xunit.runner.visualstudio" />
+    <PackageReference Include="LinqKit.Microsoft.EntityFrameworkCore" Version="7.1.4" />
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="2.10.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="ReportGenerator.Core" Version="5.1.23" />
+    <PackageReference Include="System.Linq.Async" Version="6.0.1" />
+    <PackageReference Include="Tmds.ExecFunction" Version="0.6.0" />
+    <PackageReference Include="xunit" Version="2.5.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="$(RepoRoot)src\coverlet.core\coverlet.core.csproj" />
     <ProjectReference Include="$(RepoRoot)test\coverlet.tests.xunit.extensions\coverlet.tests.xunit.extensions.csproj" />
-    
+
     <ProjectReference Include="$(RepoRoot)test\coverlet.core.tests.samples.netstandard\coverlet.core.tests.samples.netstandard.csproj" />
     <ProjectReference Include="$(RepoRoot)test\coverlet.tests.projectsample.empty\coverlet.tests.projectsample.empty.csproj" />
     <ProjectReference Include="$(RepoRoot)test\coverlet.tests.projectsample.excludedbyattribute\coverlet.tests.projectsample.excludedbyattribute.csproj" />
-    <ProjectReference Include="$(RepoRoot)test\coverlet.tests.projectsample.fsharp\coverlet.tests.projectsample.fsharp.fsproj" />    
+    <ProjectReference Include="$(RepoRoot)test\coverlet.tests.projectsample.fsharp\coverlet.tests.projectsample.fsharp.fsproj" />
     <ProjectReference Include="$(RepoRoot)test\coverlet.tests.projectsample.netframework\coverlet.tests.projectsample.netframework.csproj" />
     <ProjectReference Include="$(RepoRoot)test\coverlet.tests.projectsample.vbmynamespace\coverlet.tests.projectsample.vbmynamespace.vbproj" />
   </ItemGroup>

diff --git a/test/coverlet.integration.tests/BaseTest.cs b/test/coverlet.integration.tests/BaseTest.cs
@@ -96,19 +96,27 @@ private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispo
         private protected bool RunCommand(string command, string arguments, out string standardOutput, out string standardError, string workingDirectory = "")
         {
             Debug.WriteLine($"BaseTest.RunCommand: {command} {arguments}\nWorkingDirectory: {workingDirectory}");
-            var psi = new ProcessStartInfo(command, arguments);
-            psi.WorkingDirectory = workingDirectory;
-            psi.RedirectStandardError = true;
-            psi.RedirectStandardOutput = true;
-            Process commandProcess = Process.Start(psi)!;
+            // https://learn.microsoft.com/en-us/dotnet/api/system.diagnostics.process.standardoutput?view=net-7.0&redirectedfrom=MSDN#System_Diagnostics_Process_StandardOutput
+            var commandProcess = new Process();
+            commandProcess.StartInfo.FileName = command;
+            commandProcess.StartInfo.Arguments = arguments;
+            commandProcess.StartInfo.WorkingDirectory = workingDirectory;
+            commandProcess.StartInfo.RedirectStandardError = true;
+            commandProcess.StartInfo.RedirectStandardOutput = true;
+            commandProcess.StartInfo.UseShellExecute = false;
+            string eOut = "";
+            commandProcess.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => { eOut += e.Data; });
+            commandProcess.Start();
+            // To avoid deadlocks, use an asynchronous read operation on at least one of the streams.
+            commandProcess.BeginErrorReadLine();
+            standardOutput = commandProcess.StandardOutput.ReadToEnd();
             if (!commandProcess.WaitForExit((int)TimeSpan.FromMinutes(5).TotalMilliseconds))
             {
-                throw new XunitException($"Command 'dotnet {arguments}' didn't end after 5 minute");
+              throw new XunitException($"Command 'dotnet {arguments}' didn't end after 5 minute");
             }
-            standardOutput = commandProcess.StandardOutput.ReadToEnd();
-            standardError = commandProcess.StandardError.ReadToEnd();
+            standardError = eOut;
             return commandProcess.ExitCode == 0;
-        }
+          }
 
         private protected bool DotnetCli(string arguments, out string standardOutput, out string standardError, string workingDirectory = "")
         {
@@ -192,7 +200,9 @@ private protected void AddCoverletMsbuildRef(string projectPath)
             string msbuildPkgVersion = GetPackageVersion("*msbuild*.nupkg");
             xml.Element("Project")!
                .Element("ItemGroup")!
-               .Add(new XElement("PackageReference", new XAttribute("Include", "coverlet.msbuild"), new XAttribute("Version", msbuildPkgVersion)));
+               .Add(new XElement("PackageReference", new XAttribute("Include", "coverlet.msbuild"), new XAttribute("Version", msbuildPkgVersion),
+                    new XElement("PrivateAssets", "all"),
+                    new XElement("IncludeAssets", "runtime; build; native; contentfiles; analyzers")));
             xml.Save(csproj);
         }
 
@@ -211,7 +221,9 @@ private protected void AddCoverletCollectosRef(string projectPath)
             string msbuildPkgVersion = GetPackageVersion("*collector*.nupkg");
             xml.Element("Project")!
                .Element("ItemGroup")!
-               .Add(new XElement("PackageReference", new XAttribute("Include", "coverlet.collector"), new XAttribute("Version", msbuildPkgVersion)));
+               .Add(new XElement("PackageReference", new XAttribute("Include", "coverlet.collector"), new XAttribute("Version", msbuildPkgVersion),
+                    new XElement("PrivateAssets", "all"),
+                    new XElement("IncludeAssets", "runtime; build; native; contentfiles; analyzers")));
             xml.Save(csproj);
         }
 

diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs
@@ -46,7 +46,7 @@ private protected void AssertCoverage(string standardOutput = "", bool checkDete
             {
                 bool coverageChecked = false;
                 string reportFilePath = "";
-                foreach (string coverageFile in Directory.GetFiles(_testProjectPath, "coverage.json", SearchOption.AllDirectories))
+                foreach (string coverageFile in Directory.GetFiles(GetReportPath(standardOutput), "coverage.json", SearchOption.AllDirectories))
                 {
                     Classes? document = JsonConvert.DeserializeObject<Modules>(File.ReadAllText(coverageFile))?.Document("DeepThought.cs");
                     if (document != null)
@@ -65,7 +65,7 @@ private protected void AssertCoverage(string standardOutput = "", bool checkDete
                 if (checkDeterministicReport)
                 {
                     // Verify deterministic report
-                    foreach (string coverageFile in Directory.GetFiles(_testProjectPath, "coverage.cobertura.xml", SearchOption.AllDirectories))
+                    foreach (string coverageFile in Directory.GetFiles(GetReportPath(standardOutput), "coverage.cobertura.xml", SearchOption.AllDirectories))
                     {
                         Assert.Contains("/_/test/coverlet.integration.determisticbuild/DeepThought.cs", File.ReadAllText(coverageFile));
                         File.Delete(coverageFile);
@@ -149,7 +149,7 @@ public void Collectors()
             Assert.Contains("=/_/", File.ReadAllText(sourceRootMappingFilePath));
 
             string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought", deterministicReport: true);
-            Assert.True(DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError), standardOutput);
+            bool result = DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError);
             if (!string.IsNullOrEmpty(standardError))
             {
               _output.WriteLine(standardError);
@@ -158,6 +158,7 @@ public void Collectors()
             {
               _output.WriteLine(standardOutput);
             }
+            Assert.True(result);
             Assert.Contains("Passed!", standardOutput);
             AssertCoverage(standardOutput);
 
@@ -175,42 +176,57 @@ public void Collectors()
 
         [Fact]
         public void Collectors_SourceLink()
-        {
-            CreateDeterministicTestPropsFile();
-            DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath);
-            Assert.Contains("Build succeeded.", standardOutput);
-            string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping_coverletsample.integration.determisticbuild");
-            Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath);
-            Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath));
-            Assert.Contains("=/_/", File.ReadAllText(sourceRootMappingFilePath));
-
-            string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought", sourceLink: true);
-            Assert.True(DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError), standardOutput);
-            if (!string.IsNullOrEmpty(standardError))
-            {
-              _output.WriteLine(standardError);
-            }
-            else
-            {
-              _output.WriteLine(standardOutput);
-            }
-            Assert.Contains("Passed!", standardOutput);
-            AssertCoverage(standardOutput, checkDeterministicReport: false);
-            Assert.Contains("raw.githubusercontent.com", File.ReadAllText(Directory.GetFiles(_testProjectPath, "coverage.cobertura.xml", SearchOption.AllDirectories).Single()));
-
-            // Check out/in process collectors injection
-            string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single());
-            Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent);
-            Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single()));
-            Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent);
+    {
+      CreateDeterministicTestPropsFile();
+      DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath);
+      Assert.Contains("Build succeeded.", standardOutput);
+      string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping_coverletsample.integration.determisticbuild");
+      Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath);
+      Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath));
+      Assert.Contains("=/_/", File.ReadAllText(sourceRootMappingFilePath));
+
+      string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought", sourceLink: true);
+      bool result = DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError);
+      if (!string.IsNullOrEmpty(standardError))
+      {
+        _output.WriteLine(standardError);
+      }
+      else
+      {
+        _output.WriteLine(standardOutput);
+      }
+      Assert.True(result);
+      Assert.Contains("Passed!", standardOutput);
+      AssertCoverage(standardOutput, checkDeterministicReport: false);
+      Assert.Contains("raw.githubusercontent.com", File.ReadAllText(Directory.GetFiles(GetReportPath(standardOutput), "coverage.cobertura.xml", SearchOption.AllDirectories).Single()));
+
+      // Check out/in process collectors injection
+      string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single());
+      Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent);
+      Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single()));
+      Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent);
+
+      // Process exits hang on clean seem that process doesn't close, maybe some msbuild node reuse? btw manually tested
+      // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath);
+      // Assert.False(File.Exists(sourceRootMappingFilePath));
+      RunCommand("git", "clean -fdx", out _, out _, _testProjectPath);
+    }
 
-            // Process exits hang on clean seem that process doesn't close, maybe some msbuild node reuse? btw manually tested
-            // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath);
-            // Assert.False(File.Exists(sourceRootMappingFilePath));
-            RunCommand("git", "clean -fdx", out _, out _, _testProjectPath);
-        }
+    private string GetReportPath(string standardOutput)
+    {
+      string reportPath = "";
+      if (standardOutput.Contains("coverage.json"))
+      {
+#pragma warning disable CS8602 // Dereference of a possibly null reference.
+        reportPath = standardOutput.Split('\n').FirstOrDefault(line => line.Contains("coverage.json")).TrimStart();
+#pragma warning restore CS8602 // Dereference of a possibly null reference.
+        reportPath = reportPath[reportPath.IndexOf(Directory.GetDirectoryRoot(_testProjectPath))..];
+        reportPath = reportPath[..reportPath.IndexOf("coverage.json")];
+      }
+      return reportPath;
+    }
 
-        public void Dispose()
+    public void Dispose()
         {
             File.Delete(Path.Combine(_testProjectPath, PropsFileName));
         }