Skip to content

Commit 001d484

Browse files
engyebrahimengyebrahim
authored
Add tests to document test suite messages (#1313)
1 parent af4efda commit 001d484

17 files changed

+1226
-19
lines changed

‎TestFx.sln

+6
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestFramework.ForTestingMST
194194
EndProject
195195
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HierarchyProject", "test\E2ETests\TestAssets\HierarchyProject\HierarchyProject.csproj", "{94A4DAA8-9645-4161-91F6-11EB1AD70EFC}"
196196
EndProject
197+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SuiteLifeCycleTestProject", "test\E2ETests\TestAssets\SuiteLifeCycleTestProject\SuiteLifeCycleTestProject.csproj", "{9B5D52F6-EC81-48FA-9F2F-4F5FA7FE70F1}"
197198
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DynamicDataTestProject", "test\E2ETests\TestAssets\DynamicDataTestProject\DynamicDataTestProject.csproj", "{B90E0931-2E90-4CE9-B107-ECD8E40C6B6C}"
198199
EndProject
199200
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "UnitTests", "UnitTests", "{BB874DF1-44FE-415A-B634-A6B829107890}"
@@ -344,6 +345,10 @@ Global
344345
{94A4DAA8-9645-4161-91F6-11EB1AD70EFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
345346
{94A4DAA8-9645-4161-91F6-11EB1AD70EFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
346347
{94A4DAA8-9645-4161-91F6-11EB1AD70EFC}.Release|Any CPU.Build.0 = Release|Any CPU
348+
{9B5D52F6-EC81-48FA-9F2F-4F5FA7FE70F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
349+
{9B5D52F6-EC81-48FA-9F2F-4F5FA7FE70F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
350+
{9B5D52F6-EC81-48FA-9F2F-4F5FA7FE70F1}.Release|Any CPU.ActiveCfg = Release|Any CPU
351+
{9B5D52F6-EC81-48FA-9F2F-4F5FA7FE70F1}.Release|Any CPU.Build.0 = Release|Any CPU
347352
{B90E0931-2E90-4CE9-B107-ECD8E40C6B6C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
348353
{B90E0931-2E90-4CE9-B107-ECD8E40C6B6C}.Debug|Any CPU.Build.0 = Debug|Any CPU
349354
{B90E0931-2E90-4CE9-B107-ECD8E40C6B6C}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -405,6 +410,7 @@ Global
405410
{66608D86-416A-49AF-A937-C47F7E4586AE} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
406411
{0685FBC3-C3A9-43A9-B15C-15BAA39705FE} = {33D3029D-E653-4929-BB31-C714178C4BEE}
407412
{94A4DAA8-9645-4161-91F6-11EB1AD70EFC} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
413+
{9B5D52F6-EC81-48FA-9F2F-4F5FA7FE70F1} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
408414
{B90E0931-2E90-4CE9-B107-ECD8E40C6B6C} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
409415
{BB874DF1-44FE-415A-B634-A6B829107890} = {A9596292-7E67-4566-9096-143DDAA4E8D8}
410416
EndGlobalSection

‎test/E2ETests/Automation.CLI/CLITestBase.common.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ protected string GetTestAdapterPath()
7979
/// <param name="settingsXml">RunSettings provided for discovery/execution.</param>
8080
/// <param name="testAdapterPath">Full path to TestAdapter.</param>
8181
/// <returns>RunSettingXml as string.</returns>
82-
protected string GetRunSettingXml(string settingsXml, string testAdapterPath)
82+
protected string GetRunSettingXml(string settingsXml, string testAdapterPath, string targetFramework = "")
8383
{
8484
if (string.IsNullOrEmpty(settingsXml))
8585
{
@@ -93,7 +93,7 @@ protected string GetRunSettingXml(string settingsXml, string testAdapterPath)
9393
}
9494

9595
XmlElement root = doc.DocumentElement;
96-
RunConfiguration runConfiguration = new(testAdapterPath);
96+
RunConfiguration runConfiguration = new(Path.Combine(testAdapterPath, targetFramework));
9797
XmlElement runConfigElement = runConfiguration.ToXml();
9898
if (root[runConfiguration.SettingsName] == null)
9999
{

‎test/E2ETests/Automation.CLI/CLITestBase.e2e.cs

+18-17
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ public partial class CLITestBase : TestContainer
1717
{
1818
private static VsTestConsoleWrapper s_vsTestConsoleWrapper;
1919
private DiscoveryEventsHandler _discoveryEventsHandler;
20-
private RunEventsHandler _runEventsHandler;
20+
21+
protected RunEventsHandler RunEventsHandler { get; private set; }
2122

2223
public CLITestBase()
2324
{
@@ -46,17 +47,17 @@ public void InvokeVsTestForDiscovery(string[] sources, string runSettings = "")
4647
/// <param name="sources">List of test assemblies.</param>
4748
/// <param name="runSettings">Run settings for execution.</param>
4849
/// <param name="testCaseFilter">Test Case filter for execution.</param>
49-
public void InvokeVsTestForExecution(string[] sources, string runSettings = "", string testCaseFilter = null)
50+
public void InvokeVsTestForExecution(string[] sources, string runSettings = "", string testCaseFilter = null, string targetFramework = "")
5051
{
5152
ExpandTestSourcePaths(sources);
5253

53-
_runEventsHandler = new RunEventsHandler();
54-
string runSettingXml = GetRunSettingXml(runSettings, GetTestAdapterPath());
54+
RunEventsHandler = new RunEventsHandler();
55+
string runSettingXml = GetRunSettingXml(runSettings, GetTestAdapterPath(), targetFramework);
5556

56-
s_vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, _runEventsHandler);
57-
if (_runEventsHandler.Errors.Any())
57+
s_vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, RunEventsHandler);
58+
if (RunEventsHandler.Errors.Any())
5859
{
59-
throw new Exception($"Run failed with {_runEventsHandler.Errors.Count} errors:{Environment.NewLine}{string.Join(Environment.NewLine, _runEventsHandler.Errors)}");
60+
throw new Exception($"Run failed with {RunEventsHandler.Errors.Count} errors:{Environment.NewLine}{string.Join(Environment.NewLine, RunEventsHandler.Errors)}");
6061
}
6162
}
6263

@@ -104,7 +105,7 @@ public void ValidatePassedTests(params string[] passedTests)
104105
public void ValidatePassedTestsCount(int expectedPassedTestsCount)
105106
{
106107
// Make sure only expected number of tests passed and not more.
107-
Assert.AreEqual(expectedPassedTestsCount, _runEventsHandler.PassedTests.Count);
108+
Assert.AreEqual(expectedPassedTestsCount, RunEventsHandler.PassedTests.Count);
108109
}
109110

110111
/// <summary>
@@ -129,7 +130,7 @@ public void ValidateFailedTests(string source, params string[] failedTests)
129130
public void ValidateFailedTestsCount(int expectedFailedTestsCount)
130131
{
131132
// Make sure only expected number of tests failed and not more.
132-
Assert.AreEqual(expectedFailedTestsCount, _runEventsHandler.FailedTests.Count);
133+
Assert.AreEqual(expectedFailedTestsCount, RunEventsHandler.FailedTests.Count);
133134
}
134135

135136
/// <summary>
@@ -140,7 +141,7 @@ public void ValidateFailedTestsCount(int expectedFailedTestsCount)
140141
public void ValidateSkippedTests(params string[] skippedTests)
141142
{
142143
// Make sure only expected number of tests skipped and not more.
143-
Assert.AreEqual(skippedTests.Length, _runEventsHandler.SkippedTests.Count);
144+
Assert.AreEqual(skippedTests.Length, RunEventsHandler.SkippedTests.Count);
144145

145146
ValidateSkippedTestsContain(skippedTests);
146147
}
@@ -153,9 +154,9 @@ public void ValidateSkippedTests(params string[] skippedTests)
153154
[MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)]
154155
public void ValidatePassedTestsContain(params string[] passedTests)
155156
{
156-
var passedTestResults = _runEventsHandler.PassedTests;
157-
var failedTestResults = _runEventsHandler.FailedTests;
158-
var skippedTestsResults = _runEventsHandler.SkippedTests;
157+
var passedTestResults = RunEventsHandler.PassedTests;
158+
var failedTestResults = RunEventsHandler.FailedTests;
159+
var skippedTestsResults = RunEventsHandler.SkippedTests;
159160

160161
foreach (var test in passedTests)
161162
{
@@ -195,7 +196,7 @@ public void ValidateFailedTestsContain(string source, bool validateStackTraceInf
195196
{
196197
foreach (var test in failedTests)
197198
{
198-
var testFound = _runEventsHandler.FailedTests.FirstOrDefault(f => test.Equals(f.TestCase?.FullyQualifiedName) ||
199+
var testFound = RunEventsHandler.FailedTests.FirstOrDefault(f => test.Equals(f.TestCase?.FullyQualifiedName) ||
199200
test.Equals(f.DisplayName));
200201
Assert.IsNotNull(testFound, "Test '{0}' does not appear in failed tests list.", test);
201202

@@ -223,7 +224,7 @@ public void ValidateSkippedTestsContain(params string[] skippedTests)
223224
{
224225
foreach (var test in skippedTests)
225226
{
226-
var testFound = _runEventsHandler.SkippedTests.Any(s => test.Equals(s.TestCase.FullyQualifiedName) ||
227+
var testFound = RunEventsHandler.SkippedTests.Any(s => test.Equals(s.TestCase.FullyQualifiedName) ||
227228
test.Equals(s.DisplayName));
228229
Assert.IsTrue(testFound, "Test '{0}' does not appear in skipped tests list.", test);
229230
}
@@ -232,8 +233,8 @@ public void ValidateSkippedTestsContain(params string[] skippedTests)
232233
public void ValidateTestRunTime(int thresholdTime)
233234
{
234235
Assert.IsTrue(
235-
_runEventsHandler.ElapsedTimeInRunningTests >= 0 && _runEventsHandler.ElapsedTimeInRunningTests < thresholdTime,
236-
$"Test Run was expected to not exceed {thresholdTime} but it took {_runEventsHandler.ElapsedTimeInRunningTests}");
236+
RunEventsHandler.ElapsedTimeInRunningTests >= 0 && RunEventsHandler.ElapsedTimeInRunningTests < thresholdTime,
237+
$"Test Run was expected to not exceed {thresholdTime} but it took {RunEventsHandler.ElapsedTimeInRunningTests}");
237238
}
238239

239240
/// <summary>

0 commit comments

Comments
 (0)