Allow to disable test expansion on implementations of ITestDataSource (…

…#4269)

src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs

@@ -25,7 +25,6 @@ internal class TypeEnumerator
     private readonly TypeValidator _typeValidator;
     private readonly TestMethodValidator _testMethodValidator;
     private readonly TestIdGenerationStrategy _testIdGenerationStrategy;
-    private readonly TestDataSourceDiscoveryOption _discoveryOption;
     private readonly ReflectHelper _reflectHelper;
 
     /// <summary>
@@ -37,15 +36,14 @@ internal class TypeEnumerator
     /// <param name="typeValidator"> The validator for test classes. </param>
     /// <param name="testMethodValidator"> The validator for test methods. </param>
     /// <param name="testIdGenerationStrategy"><see cref="TestIdGenerationStrategy"/> to use when generating TestId.</param>
-    internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, TestDataSourceDiscoveryOption discoveryOption, TestIdGenerationStrategy testIdGenerationStrategy)
+    internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, TestIdGenerationStrategy testIdGenerationStrategy)
     {
         _type = type;
         _assemblyFilePath = assemblyFilePath;
         _reflectHelper = reflectHelper;
         _typeValidator = typeValidator;
         _testMethodValidator = testMethodValidator;
         _testIdGenerationStrategy = testIdGenerationStrategy;
-        _discoveryOption = discoveryOption;
     }
 
     /// <summary>
@@ -154,22 +152,6 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool isDeclaredInT
                     method.DeclaringType.Assembly);
         }
 
-        // PERF: When discovery option is set to DuringDiscovery, we will expand data on tests to one test case
-        // per data item. This will happen in AssemblyEnumerator. But AssemblyEnumerator does not have direct access to
-        // the method info or method attributes, so it would create a TestMethodInfo to see if the test is data driven.
-        // Creating TestMethodInfo is expensive and should be done only for a test that we know is data driven.
-        //
-        // So to optimize this we check if we have some data source attribute. Because here we have access to all attributes
-        // and we store that info in DataType. AssemblyEnumerator will pick this up and will get the real test data in the expensive way
-        // or it will skip over getting the data cheaply, when DataType = DynamicDataType.None.
-        //
-        // This needs to be done only when DuringDiscovery is set, because otherwise we would populate the DataType, but we would not populate
-        // and execution would not try to re-populate the data, because DataType is already set to data driven, so it would just throw error about empty data.
-        if (_discoveryOption == TestDataSourceDiscoveryOption.DuringDiscovery)
-        {
-            testMethod.DataType = GetDynamicDataType(method);
-        }
-
         var testElement = new UnitTestElement(testMethod)
         {
             // Get compiler generated type name for async test method (either void returning or task returning).
@@ -238,22 +220,4 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool isDeclaredInT
 
         return testElement;
     }
-
-    private DynamicDataType GetDynamicDataType(MethodInfo method)
-    {
-        foreach (Attribute attribute in _reflectHelper.GetDerivedAttributes<Attribute>(method, inherit: true))
-        {
-            if (AttributeComparer.IsDerived<ITestDataSource>(attribute))
-            {
-                return DynamicDataType.ITestDataSource;
-            }
-
-            if (AttributeComparer.IsDerived<DataSourceAttribute>(attribute))
-            {
-                return DynamicDataType.DataSourceAttribute;
-            }
-        }
-
-        return DynamicDataType.None;
-    }
 }

src/Adapter/MSTest.TestAdapter/Execution/TypeCache.cs

@@ -89,9 +89,6 @@ public IEnumerable<TestAssemblyInfo> AssemblyInfoListWithExecutableCleanupMethod
     /// <summary>
     /// Get the test method info corresponding to the parameter test Element.
     /// </summary>
-    /// <param name="testMethod"> The test Method. </param>
-    /// <param name="testContext"> The test Context. </param>
-    /// <param name="captureDebugTraces"> Indicates whether the test method should capture debug traces.</param>
     /// <returns> The <see cref="TestMethodInfo"/>. </returns>
     public TestMethodInfo? GetTestMethodInfo(TestMethod testMethod, ITestContext testContext, bool captureDebugTraces)
     {
@@ -109,7 +106,29 @@ public IEnumerable<TestAssemblyInfo> AssemblyInfoListWithExecutableCleanupMethod
         }
 
         // Get the testMethod
-        return ResolveTestMethod(testMethod, testClassInfo, testContext, captureDebugTraces);
+        return ResolveTestMethodInfo(testMethod, testClassInfo, testContext, captureDebugTraces);
+    }
+
+    /// <summary>
+    /// Get the test method info corresponding to the parameter test Element.
+    /// </summary>
+    /// <returns> The <see cref="TestMethodInfo"/>. </returns>
+    public TestMethodInfo? GetTestMethodInfoForDiscovery(TestMethod testMethod)
+    {
+        Guard.NotNull(testMethod);
+
+        // Get the classInfo (This may throw as GetType calls assembly.GetType(..,true);)
+        TestClassInfo? testClassInfo = GetClassInfo(testMethod);
+
+        if (testClassInfo == null)
+        {
+            // This means the class containing the test method could not be found.
+            // Return null so we return a not found result.
+            return null;
+        }
+
+        // Get the testMethod
+        return ResolveTestMethodInfoForDiscovery(testMethod, testClassInfo);
     }
 
     /// <summary>
@@ -704,23 +723,18 @@ private void UpdateInfoIfTestInitializeOrCleanupMethod(
     /// cannot be found, or a function is found that returns non-void, the result is
     /// set to error.
     /// </summary>
-    /// <param name="testMethod"> The test Method. </param>
-    /// <param name="testClassInfo"> The test Class Info. </param>
-    /// <param name="testContext"> The test Context. </param>
-    /// <param name="captureDebugTraces"> Indicates whether the test method should capture debug traces.</param>
     /// <returns>
     /// The TestMethodInfo for the given test method. Null if the test method could not be found.
     /// </returns>
-    private TestMethodInfo? ResolveTestMethod(TestMethod testMethod, TestClassInfo testClassInfo, ITestContext testContext, bool captureDebugTraces)
+    private TestMethodInfo ResolveTestMethodInfo(TestMethod testMethod, TestClassInfo testClassInfo, ITestContext testContext, bool captureDebugTraces)
     {
         DebugEx.Assert(testMethod != null, "testMethod is Null");
         DebugEx.Assert(testClassInfo != null, "testClassInfo is Null");
 
         MethodInfo methodInfo = GetMethodInfoForTestMethod(testMethod, testClassInfo);
 
-        ExpectedExceptionBaseAttribute? expectedExceptionAttribute = _reflectionHelper.ResolveExpectedExceptionHelper(methodInfo, testMethod);
         TimeoutInfo timeout = GetTestTimeout(methodInfo, testMethod);
-
+        ExpectedExceptionBaseAttribute? expectedExceptionAttribute = _reflectionHelper.ResolveExpectedExceptionHelper(methodInfo, testMethod);
         var testMethodOptions = new TestMethodOptions(timeout, expectedExceptionAttribute, testContext, captureDebugTraces, GetTestMethodAttribute(methodInfo, testClassInfo));
         var testMethodInfo = new TestMethodInfo(methodInfo, testClassInfo, testMethodOptions);
 
@@ -729,6 +743,14 @@ private void UpdateInfoIfTestInitializeOrCleanupMethod(
         return testMethodInfo;
     }
 
+    private TestMethodInfo ResolveTestMethodInfoForDiscovery(TestMethod testMethod, TestClassInfo testClassInfo)
+    {
+        MethodInfo methodInfo = GetMethodInfoForTestMethod(testMethod, testClassInfo);
+
+        // Let's build a fake options type as it won't be used.
+        return new TestMethodInfo(methodInfo, testClassInfo, new(TimeoutInfo.FromTimeout(-1), null, null, false, null));
+    }
+
     /// <summary>
     /// Provides the Test Method Extension Attribute of the TestClass.
     /// </summary>

src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs

@@ -278,11 +278,22 @@ internal static TestIdGenerationStrategy GetTestIdGenerationStrategy(Assembly as
     /// Gets TestDataSourceDiscovery assembly level attribute.
     /// </summary>
     /// <param name="assembly"> The test assembly. </param>
+    [Obsolete]
     internal static TestDataSourceDiscoveryOption? GetTestDataSourceDiscoveryOption(Assembly assembly)
         => PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(assembly, typeof(TestDataSourceDiscoveryAttribute))
             .OfType<TestDataSourceDiscoveryAttribute>()
             .FirstOrDefault()?.DiscoveryOption;
 
+    /// <summary>
+    /// Gets TestDataSourceOptions assembly level attribute.
+    /// </summary>
+    /// <param name="assembly"> The test assembly. </param>
+    /// <returns> The TestDataSourceOptionsAttribute if set. Null otherwise. </returns>
+    internal static TestDataSourceOptionsAttribute? GetTestDataSourceOptions(Assembly assembly)
+        => PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(assembly, typeof(TestDataSourceOptionsAttribute))
+            .OfType<TestDataSourceOptionsAttribute>()
+            .FirstOrDefault();
+
     /// <summary>
     /// Get the parallelization behavior for a test method.
     /// </summary>

src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs

@@ -11,7 +11,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting;
 /// Attribute to define in-line data for a test method.
 /// </summary>
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
-public class DataRowAttribute : Attribute, ITestDataSource
+public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability
 {
     /// <summary>
     /// Initializes a new instance of the <see cref="DataRowAttribute"/> class.
@@ -55,6 +55,9 @@ public DataRowAttribute(string?[]? stringArrayData)
     /// </summary>
     public string? DisplayName { get; set; }
 
+    /// <inheritdoc />
+    public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto;
+
     /// <inheritdoc />
     public IEnumerable<object?[]> GetData(MethodInfo methodInfo) => [Data];
 

src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs

@@ -33,7 +33,7 @@ public enum DynamicDataSourceType
 /// Attribute to define dynamic data for a test method.
 /// </summary>
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
-public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo
+public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability
 {
     private readonly string _dynamicDataSourceName;
     private readonly DynamicDataSourceType _dynamicDataSourceType;
@@ -84,6 +84,9 @@ public DynamicDataAttribute(string dynamicDataSourceName, Type dynamicDataDeclar
     /// </summary>
     public Type? DynamicDataDisplayNameDeclaringType { get; set; }
 
+    /// <inheritdoc />
+    public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto;
+
     /// <inheritdoc />
     public IEnumerable<object[]> GetData(MethodInfo methodInfo) => DynamicDataProvider.Instance.GetData(_dynamicDataDeclaringType, _dynamicDataSourceType, _dynamicDataSourceName, methodInfo);
 

src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryAttribute.cs

@@ -7,6 +7,11 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting;
 /// Specifies how to discover <see cref="ITestDataSource"/> tests.
 /// </summary>
 [AttributeUsage(AttributeTargets.Assembly)]
+#if NET6_0_OR_GREATER
+[Obsolete("Attribute is obsolete and will be removed in v4, instead use 'TestDataSourceOptionsAttribute'.", DiagnosticId = "MSTESTOBS")]
+#else
+[Obsolete("Attribute is obsolete and will be removed in v4, instead use 'TestDataSourceOptionsAttribute'.")]
+#endif
 public class TestDataSourceDiscoveryAttribute : Attribute
 {
     /// <summary>
@@ -15,7 +20,8 @@ public class TestDataSourceDiscoveryAttribute : Attribute
     /// <param name="discoveryOption">
     /// The <see cref="TestDataSourceDiscoveryOption"/> to use when discovering <see cref="ITestDataSource"/> tests.
     /// </param>
-    public TestDataSourceDiscoveryAttribute(TestDataSourceDiscoveryOption discoveryOption) => DiscoveryOption = discoveryOption;
+    public TestDataSourceDiscoveryAttribute(TestDataSourceDiscoveryOption discoveryOption)
+        => DiscoveryOption = discoveryOption;
 
     /// <summary>
     /// Gets the discovery option.

src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryOption.cs

@@ -6,6 +6,11 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting;
 /// <summary>
 /// The supported discovery modes for <see cref="ITestDataSource"/> tests.
 /// </summary>
+#if NET6_0_OR_GREATER
+[Obsolete("Type is obsolete and will be removed in v4, instead use 'TestDataSourceUnfoldingStrategy'.", DiagnosticId = "MSTESTOBS")]
+#else
+[Obsolete("Type is obsolete and will be removed in v4, instead use 'TestDataSourceUnfoldingStrategy'.")]
+#endif
 public enum TestDataSourceDiscoveryOption
 {
     /// <summary>

src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestTools.UnitTesting;
+
+/// <summary>
+/// Specifies options for all <see cref="ITestDataSource"/> of the current assembly.
+/// </summary>
+/// <remarks>
+/// These options can be override by individual <see cref="ITestDataSource"/> attribute.</remarks>
+[AttributeUsage(AttributeTargets.Assembly, Inherited = false)]
+public sealed class TestDataSourceOptionsAttribute : Attribute
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TestDataSourceOptionsAttribute"/> class.
+    /// </summary>
+    /// <param name="unfoldingStrategy">
+    /// The <see cref="UnfoldingStrategy"/> to use when executing parameterized tests.
+    /// </param>
+    public TestDataSourceOptionsAttribute(TestDataSourceUnfoldingStrategy unfoldingStrategy)
+        => UnfoldingStrategy = unfoldingStrategy;
+
+    /// <summary>
+    /// Gets the test unfolding strategy.
+    /// </summary>
+    public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; }
+}

src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestTools.UnitTesting;
+
+/// <summary>
+/// Specifies the capability of a test data source to define how parameterized tests should be executed, either as
+/// individual test cases for each data row or as a single test case. This affects the test results and the UI
+/// representation of the tests.
+/// </summary>
+public interface ITestDataSourceUnfoldingCapability
+{
+    TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; }
+}
+
+/// <summary>
+/// Specifies how parameterized tests should be executed, either as individual test cases for each data row or as a
+/// single test case. This affects the test results and the UI representation of the tests.
+/// </summary>
+public enum TestDataSourceUnfoldingStrategy : byte
+{
+    /// <summary>
+    /// MSTest will decide whether to unfold the parameterized test based on value from the assembly level attribute
+    /// <see cref="TestDataSourceOptionsAttribute" />. If no assembly level attribute is specified, then the default
+    /// configuration is to unfold.
+    /// </summary>
+    Auto,
+
+    /// <summary>
+    /// Each data row is treated as a separate test case.
+    /// </summary>
+    Unfold,
+
+    /// <summary>
+    /// The parameterized test is not unfolded; all data rows are treated as a single test case.
+    /// </summary>
+    Fold,
+}

src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt

@@ -1 +1,14 @@
 #nullable enable
+Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.UnfoldingStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
+Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.UnfoldingStrategy.set -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.UnfoldingStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
+Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.UnfoldingStrategy.set -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceUnfoldingCapability
+Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceUnfoldingCapability.UnfoldingStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
+Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute
+Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.TestDataSourceOptionsAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy unfoldingStrategy) -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.UnfoldingStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
+Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
+Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.Auto = 0 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
+Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.Fold = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy
+Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.Unfold = 1 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy