Skip to content

Commit

Permalink
Cherry-pick "Remove chutzpah (#4249)" (#4282)
Browse files Browse the repository at this point in the history
Co-authored-by: Jakub Jareš <me@jakubjares.com>
  • Loading branch information
Evangelink and nohwnd committed Jan 31, 2023
1 parent e0cf151 commit c1ee51a
Show file tree
Hide file tree
Showing 8 changed files with 113 additions and 21 deletions.
1 change: 0 additions & 1 deletion scripts/build/TestPlatform.Dependencies.props
Expand Up @@ -27,7 +27,6 @@
<NUnit3AdapterVersion>3.11.0</NUnit3AdapterVersion>
<NUnitConsoleRunnerVersion>3.8.0</NUnitConsoleRunnerVersion>

<ChutzpahAdapterVersion>4.4.13</ChutzpahAdapterVersion>
<FluentAssertionsVersion>6.7.0</FluentAssertionsVersion>

<!-- Versions that are used when building projects from TestAssets.sln for compatibility tests. See Invoke-TestAssetsBuild in scripts/build.ps1.
Expand Down
Expand Up @@ -16,16 +16,21 @@ public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase
{
[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo)
public void NonDllRunAllTestExecution(RunnerInfo runnerInfo)
{
// This used to test Chutzpah, to prove that we can run tests that are not shipped in dlls.
// But that framework is not fixing vulnerable dependencies for a long time, so we use our custom, test adapter
// that simply returns 1 discovered test on discovery, and 1 passed test on execution.
// We do not really test that we can run JavaScript tests, but we test that we can trigger tests that are not shipped
// in a dll, and pick up the provided test adapter.
SetTestEnvironment(_testEnvironment, runnerInfo);
string fileName = "test.js";
var testJSFileAbsolutePath = Path.Combine(_testEnvironment.TestAssetsPath, fileName);
string tempPath = Path.Combine(TempDirectory.Path, fileName);
File.Copy(testJSFileAbsolutePath, tempPath);
var arguments = PrepareArguments(tempPath, GetTestAdapterPath(UnitTestFramework.Chutzpah), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path);
var arguments = PrepareArguments(tempPath, GetTestAdapterPath(UnitTestFramework.NonDll), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path);
InvokeVsTest(arguments);
ValidateSummaryStatus(1, 1, 0);
ValidateSummaryStatus(1, 0, 0);
}

[TestMethod]
Expand Down
Expand Up @@ -31,7 +31,6 @@
<ProjectReference Include="$(TestPlatformRoot)test\Microsoft.TestPlatform.TestUtilities\Microsoft.TestPlatform.TestUtilities.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Chutzpah" Version="$(ChutzpahAdapterVersion)" />
<PackageReference Include="FluentAssertions" Version="$(FluentAssertionsVersion)" />
<PackageReference Include="Microsoft.TestPlatform.TestAsset.NativeCPP" Version="2.0.0" />
<PackageReference Include="Microsoft.TestPlatform.QTools.Assets" Version="2.0.0" />
Expand Down
Expand Up @@ -116,20 +116,18 @@ public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo)
[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo)
public void RunTestsWithNonDllAdapter(RunnerInfo runnerInfo)
{
// This used to be test for Chutzpah, but it has long running problem with updating dependencies,
// so we test against our own test framework, to ensure that we can run test files that are not using
// *.dll as extension.
SetTestEnvironment(_testEnvironment, runnerInfo);
Setup();

var jsSource = Path.Combine(_testEnvironment.TestAssetsPath, "test.js");

// Chuzpah adapter creates _Chutzpah temp files, to give data back from the runner.
// But when cleaning up it deletes all the _Chutzpah files, not just the one it owns,
// so when we run in parallel, the slower process will never find it's own file, because it was already deleted:
// https://github.com/mmanela/chutzpah/issues/812
var jsInTemp = TempDirectory.CopyFile(jsSource);

var testAdapterPath = Directory.EnumerateFiles(GetTestAdapterPath(UnitTestFramework.Chutzpah), "*.TestAdapter.dll").ToList();
var testAdapterPath = Directory.EnumerateFiles(GetTestAdapterPath(UnitTestFramework.NonDll), "*.TestAdapter.dll").ToList();
_vstestConsoleWrapper.InitializeExtensions(new List<string>() { testAdapterPath.First() });

_vstestConsoleWrapper.RunTests(
Expand All @@ -140,9 +138,8 @@ public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo)
var testCase = _runEventHandler.TestResults.Where(tr => tr.TestCase.DisplayName.Equals("TestMethod1"));

// Assert
Assert.AreEqual(2, _runEventHandler.TestResults.Count);
Assert.AreEqual(1, _runEventHandler.TestResults.Count);
Assert.AreEqual(1, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed));
Assert.AreEqual(1, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed));
Assert.AreEqual(1, testCase.First().TestCase.LineNumber);
Assert.AreEqual(0, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed));
}
}
13 changes: 7 additions & 6 deletions test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs
Expand Up @@ -48,11 +48,10 @@ public class IntegrationTestBase
private readonly string _testAdapterRelativePath = @"mstest.testadapter\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar);
private readonly string _nUnitTestAdapterRelativePath = @"nunit3testadapter\{0}\build".Replace('\\', Path.DirectorySeparatorChar);
private readonly string _xUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar);
private readonly string _chutzpahTestAdapterRelativePath = @"chutzpah\{0}\tools".Replace('\\', Path.DirectorySeparatorChar);

public enum UnitTestFramework
{
NUnit, XUnit, MSTest, CPP, Chutzpah
NUnit, XUnit, MSTest, CPP, NonDll
}

public IntegrationTestBase()
Expand Down Expand Up @@ -574,6 +573,12 @@ protected string GetProjectAssetFullPath(string projectName, string assetName)

protected string GetTestAdapterPath(UnitTestFramework testFramework = UnitTestFramework.MSTest)
{
if (testFramework == UnitTestFramework.NonDll)
{
var dllPath = _testEnvironment.GetTestAsset("NonDll.TestAdapter.dll", "netstandard2.0");
return Path.GetDirectoryName(dllPath)!;
}

string adapterRelativePath = string.Empty;

if (testFramework == UnitTestFramework.MSTest)
Expand All @@ -588,10 +593,6 @@ protected string GetTestAdapterPath(UnitTestFramework testFramework = UnitTestFr
{
adapterRelativePath = string.Format(CultureInfo.InvariantCulture, _xUnitTestAdapterRelativePath, IntegrationTestEnvironment.DependencyVersions["XUnitAdapterVersion"]);
}
else if (testFramework == UnitTestFramework.Chutzpah)
{
adapterRelativePath = string.Format(CultureInfo.InvariantCulture, _chutzpahTestAdapterRelativePath, IntegrationTestEnvironment.DependencyVersions["ChutzpahAdapterVersion"]);
}

return _testEnvironment.GetNugetPackage(adapterRelativePath);
}
Expand Down
14 changes: 14 additions & 0 deletions test/TestAssets/NonDll.TestAdapter/NonDll.TestAdapter.csproj
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk">
<!-- Imports Common TestAssets props. -->
<Import Project="..\..\..\scripts\build\TestAssets.props" />

<PropertyGroup>
<TargetFrameworks>netstandard2.0</TargetFrameworks>
<LangVersion>latest</LangVersion>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\..\..\src\Microsoft.TestPlatform.ObjectModel\Microsoft.TestPlatform.ObjectModel.csproj" />
</ItemGroup>
</Project>
71 changes: 71 additions & 0 deletions test/TestAssets/NonDll.TestAdapter/TestAdapter.cs
@@ -0,0 +1,71 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Generic;

using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;

namespace NonDll.TestAdapter;
[FileExtension(".js")]
[DefaultExecutorUri(Uri)]
[ExtensionUri(Uri)]
public class TestAdapter : ITestExecutor, ITestDiscoverer
{
public const string Uri = "executor://nondll.testadapter";

public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext,
IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
var count = 1;
foreach (var source in sources)
{
TestCase testCase = new()
{
Source = source,
CodeFilePath = source,
DisplayName = $"Test{count++}",
ExecutorUri = new Uri(Uri),
};
discoverySink.SendTestCase(testCase);
}
}

public void Cancel()
{
}

public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
foreach (var test in tests)
{
TestResult testResult = new(test)
{
Outcome = TestOutcome.Passed,
};
frameworkHandle.RecordResult(testResult);
}
}

public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
var count = 1;
foreach (var source in sources)
{
TestCase testCase = new()
{
Source = source,
CodeFilePath = source,
DisplayName = $"Test{count++}",
ExecutorUri = new Uri(Uri),
};
TestResult testResult = new(testCase)
{
Outcome = TestOutcome.Passed,
};
frameworkHandle.RecordResult(testResult);
}
}
}
6 changes: 6 additions & 0 deletions test/TestAssets/TestAssets.sln
Expand Up @@ -130,6 +130,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SerializeTestRunTestProject
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MultiHostTestExecutionProject", "MultiHostTestExecutionProject\MultiHostTestExecutionProject.csproj", "{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NonDll.TestAdapter", "NonDll.TestAdapter\NonDll.TestAdapter.csproj", "{429552A4-4C18-4355-94C5-80DC88C48405}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -372,6 +374,10 @@ Global
{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}.Release|Any CPU.Build.0 = Release|Any CPU
{429552A4-4C18-4355-94C5-80DC88C48405}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{429552A4-4C18-4355-94C5-80DC88C48405}.Debug|Any CPU.Build.0 = Debug|Any CPU
{429552A4-4C18-4355-94C5-80DC88C48405}.Release|Any CPU.ActiveCfg = Release|Any CPU
{429552A4-4C18-4355-94C5-80DC88C48405}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down

0 comments on commit c1ee51a

Please sign in to comment.