Add ability to ignore specific test cases

src/Adapter/MSTest.TestAdapter/Constants.cs

@@ -123,6 +123,8 @@ internal static class Constants
     internal static readonly TestProperty TestDynamicDataProperty = TestProperty.Register("MSTest.DynamicData", "DynamicData", typeof(string[]), TestPropertyAttributes.Hidden, typeof(TestCase));
 
     internal static readonly TestProperty TestIdGenerationStrategyProperty = TestProperty.Register("MSTest.TestIdGenerationStrategy", "TestIdGenerationStrategy", typeof(int), TestPropertyAttributes.Hidden, typeof(TestCase));
+
+    internal static readonly TestProperty TestDataSourceIgnoreReasonProperty = TestProperty.Register("MSTest.TestDataSourceIgnoreReasonProperty", "TestDataSourceIgnoreReasonProperty", typeof(string), TestPropertyAttributes.Hidden, typeof(TestCase));
     #endregion
 
     #region Private Constants

src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs

@@ -342,7 +342,6 @@ static UnitTestElement GetFixtureTest(string classFullName, string assemblyLocat
             return new UnitTestElement(method)
             {
                 DisplayName = $"[{fixtureType}] {methodName}",
-                Ignored = true,
                 Traits = [new Trait(Constants.FixturesTestTrait, fixtureType)],
             };
         }
@@ -424,6 +423,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat
         // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests.
         // Any change made here should be reflected in TestMethodRunner.ExecuteDataSourceBasedTests as well.
         data = dataSource.GetData(methodInfo);
+        string? testDataSourceIgnoreReason = (dataSource as ITestDataSourceIgnoreCapability)?.Ignore;
 
         if (!data.Any())
         {
@@ -435,6 +435,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat
             UnitTestElement discoveredTest = test.Clone();
             // Make the test not data driven, because it had no data.
             discoveredTest.TestMethod.DataType = DynamicDataType.None;
+            discoveredTest.TestMethod.TestDataSourceIgnoreReason = testDataSourceIgnoreReason;
             discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, null) ?? discoveredTest.DisplayName;
 
             tests.Add(discoveredTest);
@@ -468,6 +469,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat
             try
             {
                 discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d);
+                discoveredTest.TestMethod.TestDataSourceIgnoreReason = testDataSourceIgnoreReason;
                 discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource;
             }
             catch (SerializationException ex)

src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs

@@ -155,7 +155,6 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool isDeclaredInT
             TestCategory = _reflectHelper.GetTestCategories(method, _type),
             DoNotParallelize = _reflectHelper.IsDoNotParallelizeSet(method, _type),
             Priority = _reflectHelper.GetPriority(method),
-            Ignored = _reflectHelper.IsNonDerivedAttributeDefined<IgnoreAttribute>(method, inherit: false),
             DeploymentItems = PlatformServiceProvider.Instance.TestDeployment.GetDeploymentItems(method, _type, warnings),
         };
 

src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs

@@ -568,7 +568,8 @@ internal void ExecuteClassCleanup(TestContext testContext)
             {
                 if (classCleanupMethod is not null)
                 {
-                    if (!ReflectHelper.Instance.IsNonDerivedAttributeDefined<IgnoreAttribute>(classCleanupMethod.DeclaringType!, false))
+                    if (ClassAttribute.Ignore is null &&
+                        !ReflectHelper.Instance.IsNonDerivedAttributeDefined<IgnoreAttribute>(classCleanupMethod.DeclaringType!, false))
                     {
                         ClassCleanupException = InvokeCleanupMethod(classCleanupMethod, remainingCleanupCount: BaseClassCleanupMethods.Count, testContext);
                     }
@@ -579,7 +580,8 @@ internal void ExecuteClassCleanup(TestContext testContext)
                     for (int i = 0; i < BaseClassCleanupMethods.Count; i++)
                     {
                         classCleanupMethod = BaseClassCleanupMethods[i];
-                        if (!ReflectHelper.Instance.IsNonDerivedAttributeDefined<IgnoreAttribute>(classCleanupMethod.DeclaringType!, false))
+                        if (ClassAttribute.Ignore is null &&
+                            !ReflectHelper.Instance.IsNonDerivedAttributeDefined<IgnoreAttribute>(classCleanupMethod.DeclaringType!, false))
                         {
                             ClassCleanupException = InvokeCleanupMethod(classCleanupMethod, remainingCleanupCount: BaseClassCleanupMethods.Count - 1 - i, testContext);
                             if (ClassCleanupException is not null)

src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs

@@ -168,6 +168,11 @@ internal UnitTestResult[] RunTestMethod()
         {
             if (_test.DataType == DynamicDataType.ITestDataSource)
             {
+                if (_test.TestDataSourceIgnoreReason is not null)
+                {
+                    return [new(UnitTestOutcome.Ignored, _test.TestDataSourceIgnoreReason)];
+                }
+
                 object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData);
                 TestResult[] testResults = ExecuteTestWithDataSource(null, data);
                 results.AddRange(testResults);
@@ -306,6 +311,20 @@ private bool ExecuteDataSourceBasedTests(List<TestResult> results)
                 foreach (UTF.ITestDataSource testDataSource in testDataSources)
                 {
                     isDataDriven = true;
+
+                    if (testDataSource is ITestDataSourceIgnoreCapability { Ignore: { } ignoreReason })
+                    {
+                        results.Add(new()
+                        {
+                            // This is closest to ignore. This enum doesn't have a value specific to Ignore.
+                            // It may be a better idea to add a value there, but the enum is public and we need to think more carefully before adding the API.
+                            // For now, TestResultExtensions.ToUnitTestResults method will convert this to Ignored value of ObjectModel enum when IgnoreReason is non-null.
+                            Outcome = UTF.UnitTestOutcome.NotRunnable,
+                            IgnoreReason = ignoreReason,
+                        });
+                        continue;
+                    }
+
                     IEnumerable<object?[]>? dataSource;
 
                     // This code is to execute tests. To discover the tests code is in AssemblyEnumerator.ProcessTestDataSourceTests.

src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs

@@ -347,9 +347,17 @@ private bool IsTestMethodRunnable(
             }
         }
 
+        // TODO: Executor should never be null. Is it incorrectly annotated?
+        string? ignoreMessage = testMethodInfo.TestMethodOptions.Executor?.Ignore ?? testMethodInfo.Parent.ClassAttribute.Ignore;
+        if (ignoreMessage is not null)
+        {
+            notRunnableResult = [new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage)];
+            return false;
+        }
+
         IgnoreAttribute? ignoreAttributeOnClass =
             _reflectHelper.GetFirstNonDerivedAttributeOrDefault<IgnoreAttribute>(testMethodInfo.Parent.ClassType, inherit: false);
-        string? ignoreMessage = ignoreAttributeOnClass?.IgnoreMessage;
+        ignoreMessage = ignoreAttributeOnClass?.IgnoreMessage;
 
         IgnoreAttribute? ignoreAttributeOnMethod =
             _reflectHelper.GetFirstNonDerivedAttributeOrDefault<IgnoreAttribute>(testMethodInfo.TestMethod, inherit: false);

src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs

@@ -86,6 +86,7 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string
 
             testMethod.DataType = dataType;
             testMethod.SerializedData = data;
+            testMethod.TestDataSourceIgnoreReason = testCase.GetPropertyValue(Constants.TestDataSourceIgnoreReasonProperty) as string;
         }
 
         if (testCase.GetPropertyValue(Constants.DeclaringClassNameProperty) is string declaringClassName && declaringClassName != testClassName)

src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs

@@ -29,7 +29,9 @@ internal static UnitTestResult[] ToUnitTestResults(this IReadOnlyCollection<UTF.
         int i = 0;
         foreach (UTF.TestResult testResult in testResults)
         {
-            var outcome = testResult.Outcome.ToUnitTestOutcome();
+            UnitTestOutcome outcome = testResult.IgnoreReason is not null
+                ? UnitTestOutcome.Ignored
+                : testResult.Outcome.ToUnitTestOutcome();
 
             UnitTestResult unitTestResult = testResult.TestFailureException is { } testFailureException
                 ? new UnitTestResult(
@@ -38,6 +40,12 @@ internal static UnitTestResult[] ToUnitTestResults(this IReadOnlyCollection<UTF.
                         testFailureException.TryGetMessage(),
                         testFailureException is TestFailedException testException ? testException.StackTraceInformation : testFailureException.TryGetStackTraceInformation()))
                 : new UnitTestResult { Outcome = outcome };
+
+            if (testResult.IgnoreReason is not null)
+            {
+                unitTestResult.ErrorMessage = testResult.IgnoreReason;
+            }
+
             unitTestResult.StandardOut = testResult.LogOutput;
             unitTestResult.StandardError = testResult.LogError;
             unitTestResult.DebugTrace = testResult.DebugTrace;

src/Adapter/MSTest.TestAdapter/ObjectModel/TestMethod.cs

@@ -143,6 +143,8 @@ public string? DeclaringClassFullName
     /// </summary>
     internal string?[]? SerializedData { get; set; }
 
+    internal string? TestDataSourceIgnoreReason { get; set; }
+
     /// <summary>
     /// Gets or sets the test group set during discovery.
     /// </summary>

src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs

@@ -33,11 +33,6 @@ public UnitTestElement(TestMethod testMethod)
     /// </summary>
     public TestMethod TestMethod { get; private set; }
 
-    /// <summary>
-    /// Gets or sets a value indicating whether the unit test should be ignored at run-time.
-    /// </summary>
-    public bool Ignored { get; set; }
-
     /// <summary>
     /// Gets or sets a value indicating whether it is a async test.
     /// </summary>
@@ -211,6 +206,7 @@ internal TestCase ToTestCase()
 
             testCase.SetPropertyValue(Constants.TestDynamicDataTypeProperty, (int)TestMethod.DataType);
             testCase.SetPropertyValue(Constants.TestDynamicDataProperty, data);
+            testCase.SetPropertyValue(Constants.TestDataSourceIgnoreReasonProperty, TestMethod.TestDataSourceIgnoreReason);
         }
 
         SetTestCaseId(testCase, testFullName);

src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs

@@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting;
 /// Attribute to define in-line data for a test method.
 /// </summary>
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
-public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability
+public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability, ITestDataSourceIgnoreCapability
 {
     /// <summary>
     /// Initializes a new instance of the <see cref="DataRowAttribute"/> class.
@@ -53,6 +53,11 @@ public DataRowAttribute(string?[]? stringArrayData)
     /// </summary>
     public string? DisplayName { get; set; }
 
+    /// <summary>
+    /// Gets or sets a reason to ignore the specific test case. Setting the property to non-null value will ignore the test case.
+    /// </summary>
+    public string? Ignore { get; set; }
+
     /// <inheritdoc />
     public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto;
 

src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs

@@ -32,7 +32,7 @@ public enum DynamicDataSourceType
 /// Attribute to define dynamic data for a test method.
 /// </summary>
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
-public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability
+public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability, ITestDataSourceIgnoreCapability
 {
     private readonly string _dynamicDataSourceName;
     private readonly DynamicDataSourceType _dynamicDataSourceType;
@@ -114,6 +114,11 @@ public DynamicDataAttribute(string dynamicDataSourceName, Type dynamicDataDeclar
     /// <inheritdoc />
     public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto;
 
+    /// <summary>
+    /// Gets or sets a reason to ignore this dynamic data source. Setting the property to non-null value will ignore the dynamic data source.
+    /// </summary>
+    public string? Ignore { get; set; }
+
     /// <inheritdoc />
     public IEnumerable<object[]> GetData(MethodInfo methodInfo)
         => DynamicDataProvider.Instance.GetData(_dynamicDataDeclaringType, _dynamicDataSourceType, _dynamicDataSourceName, methodInfo);

src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs

@@ -26,4 +26,9 @@ public class TestClassAttribute : Attribute
     public virtual TestMethodAttribute? GetTestMethodAttribute(TestMethodAttribute? testMethodAttribute) =>
         // If TestMethod is not extended by derived class then return back the original TestMethodAttribute
         testMethodAttribute;
+
+    /// <summary>
+    /// Gets or sets a reason to ignore the test class. Setting the property to non-null value will ignore the test class.
+    /// </summary>
+    public string? Ignore { get; set; }
 }

src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs

@@ -50,6 +50,11 @@ public TestMethodAttribute()
     /// </summary>
     public string? DisplayName { get; }
 
+    /// <summary>
+    /// Gets or sets a reason to ignore the test method. Setting the property to non-null value will ignore the test method.
+    /// </summary>
+    public string? Ignore { get; set; }
+
     /// <summary>
     /// Executes a test method.
     /// </summary>

src/TestFramework/TestFramework/Attributes/TestMethod/TestResult.cs

@@ -24,6 +24,8 @@ public class TestResult
     /// </summary>
     public UnitTestOutcome Outcome { get; set; }
 
+    internal string? IgnoreReason { get; set; }
+
     /// <summary>
     /// Gets or sets the exception thrown when test is failed.
     /// </summary>

src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestTools.UnitTesting;
+
+/// <summary>
+/// Specifies the capability of a test data source to be ignored and define the ignore reason.
+/// </summary>
+public interface ITestDataSourceIgnoreCapability
+{
+    /// <summary>
+    /// Gets or sets a reason to ignore the test data source. Setting the property to non-null value will ignore the test data source.
+    /// </summary>
+    string? Ignore { get; set; }
+}

src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt

@@ -1,11 +1,22 @@
 #nullable enable
+Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.Ignore.get -> string?
+Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.Ignore.set -> void
 Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName) -> void
 Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void
 Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType) -> void
 Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.Ignore.get -> string?
+Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.Ignore.set -> void
 Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.AutoDetect = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType
 *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void
 *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability
+Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.Ignore.get -> string?
+Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.Ignore.set -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.Ignore.get -> string?
+Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.Ignore.set -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.Ignore.get -> string?
+Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.Ignore.set -> void
 static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Throws<TException>(System.Action! action, string! message = "", params object![]! messageArgs) -> TException!
 static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsAsync<TException>(System.Func<System.Threading.Tasks.Task!>! action, string! message = "", params object![]! messageArgs) -> System.Threading.Tasks.Task<TException!>!
 static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactly<TException>(System.Action! action, string! message = "", params object![]! messageArgs) -> TException!

test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs

@@ -291,18 +291,6 @@ public void GetTestFromMethodShouldInitializeAsyncTypeNameCorrectly()
         Verify(expectedAsyncTaskName == testElement.AsyncTypeName);
     }
 
-    public void GetTestFromMethodShouldSetIgnoredPropertyToFalseIfNotSetOnTestClassAndTestMethod()
-    {
-        SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameAssembly: true);
-        TypeEnumerator typeEnumerator = GetTypeEnumeratorInstance(typeof(DummyTestClass), "DummyAssemblyName");
-        MethodInfo methodInfo = typeof(DummyTestClass).GetMethod("MethodWithVoidReturnType");
-
-        MSTest.TestAdapter.ObjectModel.UnitTestElement testElement = typeEnumerator.GetTestFromMethod(methodInfo, true, _warnings);
-
-        Verify(testElement is not null);
-        Verify(!testElement.Ignored);
-    }
-
     public void GetTestFromMethodShouldSetTestCategory()
     {
         SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameAssembly: true);

test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs

@@ -123,7 +123,7 @@ public void RunTestsForTestWithFilterShouldSendResultsForFilteredTests()
 
     public void RunTestsForIgnoredTestShouldSendResultsMarkingIgnoredTestsAsSkipped()
     {
-        TestCase testCase = GetTestCase(typeof(DummyTestClass), "IgnoredTest", ignore: true);
+        TestCase testCase = GetTestCase(typeof(DummyTestClass), "IgnoredTest");
         TestCase[] tests = [testCase];
 
         _testExecutionManager.RunTests(tests, _runContext, _frameworkHandle, _cancellationToken);
@@ -814,14 +814,11 @@ private void RunTestsForTestShouldRunTestsInTheParentDomainsApartmentState()
 
     #region private methods
 
-    private static TestCase GetTestCase(Type typeOfClass, string testName, bool ignore = false)
+    private static TestCase GetTestCase(Type typeOfClass, string testName)
     {
         MethodInfo methodInfo = typeOfClass.GetMethod(testName);
         var testMethod = new TestMethod(methodInfo.Name, typeOfClass.FullName, Assembly.GetExecutingAssembly().Location, isAsync: false);
-        UnitTestElement element = new(testMethod)
-        {
-            Ignored = ignore,
-        };
+        UnitTestElement element = new(testMethod);
         return element.ToTestCase();
     }