Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix multiple results not returned with custom TestMethod and TestClass (#358) #363

Merged
merged 9 commits into from
Mar 28, 2018
Merged

Fix multiple results not returned with custom TestMethod and TestClass (#358) #363

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f51024e
Fix Multiple results in VS2017 test runner (#336)
Feb 6, 2018
0b48abb
Fixed PR comments
Feb 7, 2018
d80a026
Added tests with data-driven tests
Feb 7, 2018
25bea17
Fixed build
Feb 7, 2018
a9f76bc
Merge branch 'master' into multipletestresultsbug
jayaranigarg Feb 12, 2018
fbee0f1
Merge branch 'master' into multipletestresultsbug
jayaranigarg Feb 12, 2018
b2af498
Merge remote-tracking branch 'upstream/master' into multipletestresul…
jayaranigarg Mar 27, 2018
97bca2b
Merge branch 'multipletestresultsbug' of https://github.com/bignonced…
jayaranigarg Mar 27, 2018
5374612
Merge branch 'master' into multipletestresultsbug
jayaranigarg Mar 28, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 27 additions & 12 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,22 +247,30 @@ internal UnitTestResult[] RunTestMethod()
watch.Start();

this.testContext.SetDataRow(dataRow);
UTF.TestResult currentResult;
UTF.TestResult[] testResults;

try
{
currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
testResults = new[]
{
new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
};
}

currentResult.DatarowIndex = rowIndex++;
watch.Stop();
currentResult.Duration = watch.Elapsed;
foreach (var testResult in testResults)
{
testResult.DatarowIndex = rowIndex;
testResult.Duration = watch.Elapsed;
}

results.Add(currentResult);
rowIndex++;

results.AddRange(testResults);
}
}
finally
Expand Down Expand Up @@ -293,18 +301,25 @@ internal UnitTestResult[] RunTestMethod()
foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
{
this.testMethodInfo.SetArguments(data);
UTF.TestResult currentResult;
UTF.TestResult[] testResults;
try
{
currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
testResults = new[]
{
new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
};
}

foreach (var testResult in testResults)
{
testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
}

currentResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
results.Add(currentResult);
results.AddRange(testResults);
this.testMethodInfo.SetArguments(null);
}
}
Expand All @@ -313,7 +328,7 @@ internal UnitTestResult[] RunTestMethod()
{
try
{
results.Add(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]);
results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo));
}
catch (Exception ex)
{
Expand Down
6 changes: 4 additions & 2 deletions test/E2ETests/Automation.CLI/CLITestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ namespace Microsoft.MSTestV2.CLIAutomation
using System.Linq;
using System.Xml;
using Microsoft.TestPlatform.VsTestConsole.TranslationLayer;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
using Microsoft.VisualStudio.TestTools.UnitTesting;

public class CLITestBase
Expand Down Expand Up @@ -59,7 +60,8 @@ public void InvokeVsTestForDiscovery(string[] sources, string runSettings = "")
/// </summary>
/// <param name="sources">List of test assemblies.</param>
/// <param name="runSettings">Run settings for execution.</param>
public void InvokeVsTestForExecution(string[] sources, string runSettings = "")
/// <param name="testCaseFilter">Test Case filter for execution.</param>
public void InvokeVsTestForExecution(string[] sources, string runSettings = "", string testCaseFilter = null)
{
for (var iterator = 0; iterator < sources.Length; iterator++)
{
Expand All @@ -74,7 +76,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "")

// this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed
vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll"));
vsTestConsoleWrapper.RunTests(sources, runSettingXml, this.runEventsHandler);
vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler);
}

/// <summary>
Expand Down
51 changes: 51 additions & 0 deletions test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace MSTestAdapter.Smoke.E2ETests
{
using Microsoft.MSTestV2.CLIAutomation;
using Microsoft.VisualStudio.TestTools.UnitTesting;

[TestClass]
public class CustomTestExecutionExtensibilityTests : CLITestBase
{
private const string TestAssembly = "FxExtensibilityTestProject.dll";

[TestMethod]
public void ExecuteCustomTestExtensibilityTests()
{
this.InvokeVsTestForExecution(new string[] { TestAssembly });
this.ValidatePassedTestsContain(
"CustomTestMethod1 - Execution number 1",
"CustomTestMethod1 - Execution number 2",
"CustomTestMethod1 - Execution number 4",
"CustomTestMethod1 - Execution number 5",
"CustomTestClass1 - Execution number 1",
"CustomTestClass1 - Execution number 2",
"CustomTestClass1 - Execution number 4",
"CustomTestClass1 - Execution number 5");
this.ValidateFailedTestsContain(
TestAssembly,
"CustomTestMethod1 - Execution number 3",
"CustomTestClass1 - Execution number 3");
}

[TestMethod]
public void ExecuteCustomTestExtensibilityWithTestDataTests()
{
this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~CustomTestExTests.CustomTestMethod2");
this.ValidatePassedTests(
"CustomTestMethod2 (B)",
"CustomTestMethod2 (B)",
"CustomTestMethod2 (B)");
this.ValidateFailedTests(
TestAssembly,
"CustomTestMethod2 (A)",
"CustomTestMethod2 (A)",
"CustomTestMethod2 (A)",
"CustomTestMethod2 (C)",
"CustomTestMethod2 (C)",
"CustomTestMethod2 (C)");
}
}
}
1 change: 1 addition & 0 deletions test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
</ItemGroup>
<ItemGroup>
<Compile Include="AssertExtensibilityTests.cs" />
<Compile Include="CustomTestExecutionExtensibilityTests.cs" />
<Compile Include="DataSourceTests.cs" />
<Compile Include="DesktopCSharpCLITests.cs" />
<Compile Include="ParallelExecutionTests.cs" />
Expand Down
77 changes: 77 additions & 0 deletions test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
namespace FxExtensibilityTestProject
{
using System.Collections.Generic;
using Microsoft.VisualStudio.TestTools.UnitTesting;

[IterativeTestClass(5)]
public class CustomTestExTests
{
private static int customTestMethod1ExecutionCount;
[IterativeTestMethod(5)]
public void CustomTestMethod1()
{
customTestMethod1ExecutionCount++;
Assert.AreNotEqual(3, customTestMethod1ExecutionCount);
}

[IterativeTestMethod(3)]
[DataRow("A")]
[DataRow("B")]
[DataRow("C")]
public void CustomTestMethod2(string value)
{
Assert.AreEqual("B", value);
}

private static int customTestClass1ExecutionCount;
[TestMethod]
public void CustomTestClass1()
{
customTestClass1ExecutionCount++;
Assert.AreNotEqual(3, customTestClass1ExecutionCount);
}
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider adding a data-driven test with [IterativeTestMethod] attribute

public class IterativeTestMethodAttribute : TestMethodAttribute
{
private readonly int stabilityThreshold;

public IterativeTestMethodAttribute(int stabilityThreshold)
{
this.stabilityThreshold = stabilityThreshold;
}

public override TestResult[] Execute(ITestMethod testMethod)
{
var results = new List<TestResult>();
for (int count = 0; count < this.stabilityThreshold; count++)
{
var testResults = base.Execute(testMethod);
foreach (var testResult in testResults)
{
testResult.DisplayName = $"{testMethod.TestMethodName} - Execution number {count + 1}";
}
results.AddRange(testResults);
}

return results.ToArray();
}
}

public class IterativeTestClassAttribute : TestClassAttribute
{
private readonly int stabilityThreshold;

public IterativeTestClassAttribute(int stabilityThreshold)
{
this.stabilityThreshold = stabilityThreshold;
}

public override TestMethodAttribute GetTestMethodAttribute(TestMethodAttribute testMethodAttribute)
{
if (testMethodAttribute is IterativeTestMethodAttribute) return testMethodAttribute;

return new IterativeTestMethodAttribute(this.stabilityThreshold);
}
}
}
1 change: 1 addition & 0 deletions test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="DynamicDataExTests.cs" />
<Compile Include="DynamicDataExMoreTests.cs" />
<Compile Include="CustomTestExTests.cs" />
<Compile Include="TestDataSourceExTests.cs" />
</ItemGroup>
<ItemGroup>
Expand Down
29 changes: 28 additions & 1 deletion test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,10 +407,37 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF
StringAssert.Contains(results[0].ErrorMessage, "Exception thrown while executing test");
}

[TestMethodV1]
public void RunTestMethodForMultipleResultsReturnMultipleResults()
{
var testMethodAttributeMock = new Mock<UTF.TestMethodAttribute>();
testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny<UTF.ITestMethod>())).Returns(new[]
{
new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed },
new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed }
});

var localTestMethodOptions = new TestMethodOptions
{
Timeout = 200,
Executor = testMethodAttributeMock.Object,
TestContext = this.testContextImplementation,
ExpectedException = null
};

var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null);
var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);

var results = testMethodRunner.Execute();
Assert.AreEqual(2, results.Length);
Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome);
Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome);
}

[TestMethodV1]
public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResultWithPassedOutcome()
{
var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);

var results = testMethodRunner.Execute();
Expand Down