-
Notifications
You must be signed in to change notification settings - Fork 315
/
DifferentTestFrameworkSimpleTests.cs
199 lines (171 loc) · 9.21 KB
/
DifferentTestFrameworkSimpleTests.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Globalization;
using System.IO;
using Microsoft.TestPlatform.TestUtilities;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace Microsoft.TestPlatform.AcceptanceTests;
[TestClass]
[TestCategory("Windows-Review")]
public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase
{
[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
public void NonDllRunAllTestExecution(RunnerInfo runnerInfo)
{
// This used to test Chutzpah, to prove that we can run tests that are not shipped in dlls.
// But that framework is not fixing vulnerable dependencies for a long time, so we use our custom, test adapter
// that simply returns 1 discovered test on discovery, and 1 passed test on execution.
// We do not really test that we can run JavaScript tests, but we test that we can trigger tests that are not shipped
// in a dll, and pick up the provided test adapter.
SetTestEnvironment(_testEnvironment, runnerInfo);
string fileName = "test.js";
var testJSFileAbsolutePath = Path.Combine(_testEnvironment.TestAssetsPath, fileName);
string tempPath = Path.Combine(TempDirectory.Path, fileName);
File.Copy(testJSFileAbsolutePath, tempPath);
var arguments = PrepareArguments(tempPath, GetTestAdapterPath(UnitTestFramework.NonDll), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path);
InvokeVsTest(arguments);
ValidateSummaryStatus(1, 0, 0);
}
[TestMethod]
// vstest.console is x64 now, but x86 run "in process" run should still succeed by being run in x86 testhost
// Skip .NET (Core) tests because we test them below.
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true, useCoreRunner: false)]
public void CPPRunAllTestExecutionNetFramework(RunnerInfo runnerInfo)
{
SetTestEnvironment(_testEnvironment, runnerInfo);
CppRunAllTests("x86");
}
[TestMethod]
[TestCategory("Windows-Review")]
// vstest.console is 64-bit now, run in process to test the 64-bit native dll
// Skip .NET (Core) tests because we test them below.
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true, useCoreRunner: false)]
public void CPPRunAllTestExecutionPlatformx64NetFramework(RunnerInfo runnerInfo)
{
SetTestEnvironment(_testEnvironment, runnerInfo);
CppRunAllTests("x64");
}
[TestMethod]
// C++ tests cannot run in .NET Framework host under .NET Core, because we only ship .NET Standard CPP adapter in .NET Core
// We also don't test x86 for .NET Core, because the resolver there does not switch between x86 and x64 correctly, it just uses the parent process bitness.
// We run this on netcore31 and not the default netcore21 because netcore31 is the minimum tfm that has the runtime features we need, such as additionaldeps.
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false, useCoreRunner: true)]
public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo)
{
SetTestEnvironment(_testEnvironment, runnerInfo);
CppRunAllTests("x64");
}
[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo)
{
if (!IsCI)
{
Assert.Inconclusive("This works on server but not locally, because locally it grabs old dll from GAC, but has version 10.0.0 as the one in our package.");
}
SetTestEnvironment(_testEnvironment, runnerInfo);
var runSettingsFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".runsettings");
//test the iterationCount setting for WebTestRunConfiguration in run settings
var runSettingsXml = $@"<?xml version='1.0' encoding='utf-8'?>
<RunSettings>
<WebTestRunConfiguration iterationCount='5' />
<RunConfiguration>
<TargetFrameworkVersion>{FrameworkArgValue}</TargetFrameworkVersion>
</RunConfiguration>
</RunSettings>";
CreateRunSettingsFile(runSettingsFilePath, runSettingsXml);
//therefore, the test will run for 5 iterations resulting in web test result file size of at least 150 KB
var minWebTestResultFileSizeInKB = 150;
if (runnerInfo.IsNetRunner)
{
Assert.Inconclusive("WebTests tests not supported with .NET Core runner.");
return;
}
string assemblyRelativePath =
@"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\WebTest1.webtest";
var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath);
using var resultsDirectory = TempDirectory;
var arguments = PrepareArguments(
assemblyAbsolutePath,
string.Empty,
runSettingsFilePath, FrameworkArgValue, string.Empty, resultsDirectory.Path);
InvokeVsTest(arguments);
ValidateSummaryStatus(1, 0, 0);
if (minWebTestResultFileSizeInKB > 0)
{
var dirInfo = new DirectoryInfo(resultsDirectory.Path);
var webtestResultFile = "WebTest1.webtestResult";
var files = dirInfo.GetFiles(webtestResultFile, SearchOption.AllDirectories);
Assert.IsTrue(files.Length > 0, $"File {webtestResultFile} not found under results directory {resultsDirectory}");
var fileSizeInKB = files[0].Length / 1024;
Assert.IsTrue(fileSizeInKB > minWebTestResultFileSizeInKB, $"Size of the file {webtestResultFile} is {fileSizeInKB} KB. It is not greater than {minWebTestResultFileSizeInKB} KB indicating iterationCount in run settings not honored.");
}
}
[TestMethod]
[TestCategory("Windows-Review")]
[NetFullTargetFrameworkDataSource]
public void CodedWebTestRunAllTests(RunnerInfo runnerInfo)
{
if (!IsCI)
{
Assert.Inconclusive("This works on server but not locally, because locally it grabs old dll from GAC, but has version 10.0.0 as the one in our package.");
}
SetTestEnvironment(_testEnvironment, runnerInfo);
if (runnerInfo.IsNetRunner)
{
Assert.Inconclusive("WebTests tests not supported with .NET Core runner.");
return;
}
string assemblyRelativePath = @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\BingWebTest.dll";
var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath);
var arguments = PrepareArguments(
assemblyAbsolutePath,
string.Empty,
string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path);
InvokeVsTest(arguments);
ValidateSummaryStatus(1, 0, 0);
}
[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void NUnitRunAllTestExecution(RunnerInfo runnerInfo)
{
SetTestEnvironment(_testEnvironment, runnerInfo);
var arguments = PrepareArguments(
GetAssetFullPath("NUTestProject.dll"),
GetTestAdapterPath(UnitTestFramework.NUnit),
string.Empty, FrameworkArgValue,
runnerInfo.InIsolationValue, TempDirectory.Path);
InvokeVsTest(arguments);
ValidateSummaryStatus(1, 1, 0);
}
[TestMethod]
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
[NetCoreTargetFrameworkDataSource]
public void XUnitRunAllTestExecution(RunnerInfo runnerInfo)
{
SetTestEnvironment(_testEnvironment, runnerInfo);
string testAssemblyPath = _testEnvironment.GetTestAsset("XUTestProject.dll");
var arguments = PrepareArguments(
testAssemblyPath,
GetTestAdapterPath(UnitTestFramework.XUnit),
string.Empty, FrameworkArgValue,
runnerInfo.InIsolationValue, TempDirectory.Path);
InvokeVsTest(arguments);
ValidateSummaryStatus(1, 1, 0);
}
private void CppRunAllTests(string platform)
{
string assemblyRelativePathFormat = @"microsoft.testplatform.testasset.nativecpp\2.0.0\contentFiles\any\any\{0}\Microsoft.TestPlatform.TestAsset.NativeCPP.dll";
var assemblyRelativePath = platform.Equals("x64", StringComparison.OrdinalIgnoreCase)
? string.Format(CultureInfo.CurrentCulture, assemblyRelativePathFormat, platform)
: string.Format(CultureInfo.CurrentCulture, assemblyRelativePathFormat, "");
var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath);
var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path);
InvokeVsTest(arguments);
ValidateSummaryStatus(1, 1, 0);
}
}