From 33f94f8e54d6caf157b593fb5f455a4b38121945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 11 Dec 2024 16:44:54 +0100 Subject: [PATCH] Allow to disable test expansion on implementations of ITestDataSource (#4269) --- .../Discovery/AssemblyEnumerator.cs | 248 +++++++++--------- .../Discovery/TypeEnumerator.cs | 38 +-- .../MSTest.TestAdapter/Execution/TypeCache.cs | 44 +++- .../Helpers/ReflectHelper.cs | 11 + .../Attributes/DataSource/DataRowAttribute.cs | 5 +- .../DataSource/DynamicDataAttribute.cs | 5 +- .../TestDataSourceDiscoveryAttribute.cs | 8 +- .../TestDataSourceDiscoveryOption.cs | 5 + .../TestDataSourceOptionsAttribute.cs | 27 ++ .../ITestDataSourceUnfoldingCapability.cs | 38 +++ .../PublicAPI/PublicAPI.Unshipped.txt | 13 + .../DataExtensibilityTests.cs | 53 +++- .../Parameterized tests/DynamicDataTests.cs | 44 +++- .../DataExtensibilityTests.cs | 2 +- .../Parameterized tests/DynamicDataTests.cs | 2 +- .../DisableExpansionTests.cs | 67 +++++ .../TestDataSourceExTests.cs | 26 +- .../Discovery/AssemblyEnumeratorTests.cs | 12 +- .../Discovery/TypeEnumeratorTests.cs | 5 +- 19 files changed, 462 insertions(+), 191 deletions(-) create mode 100644 src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs create mode 100644 src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs create mode 100644 test/IntegrationTests/TestAssets/DynamicDataTestProject/DisableExpansionTests.cs diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index dc95277fea..e810b570de 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -90,15 +90,26 @@ internal ICollection EnumerateAssembly( DataRowAttribute.TestIdGenerationStrategy = testIdGenerationStrategy; DynamicDataAttribute.TestIdGenerationStrategy = testIdGenerationStrategy; - TestDataSourceDiscoveryOption testDataSourceDiscovery = ReflectHelper.GetTestDataSourceDiscoveryOption(assembly) + TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy = ReflectHelper.GetTestDataSourceOptions(assembly)?.UnfoldingStrategy switch + { + // When strategy is auto we want to unfold + TestDataSourceUnfoldingStrategy.Auto => TestDataSourceUnfoldingStrategy.Unfold, + // When strategy is set, let's use it + { } value => value, + // When the attribute is not set, let's look at the legacy attribute +#pragma warning disable CS0612 // Type or member is obsolete #pragma warning disable CS0618 // Type or member is obsolete - - // When using legacy strategy, there is no point in trying to "read" data during discovery - // as the ID generator will ignore it. - ?? (testIdGenerationStrategy == TestIdGenerationStrategy.Legacy - ? TestDataSourceDiscoveryOption.DuringExecution - : TestDataSourceDiscoveryOption.DuringDiscovery); + null => (ReflectHelper.GetTestDataSourceDiscoveryOption(assembly), testIdGenerationStrategy) switch +#pragma warning restore CS0612 // Type or member is obsolete + { + (TestDataSourceDiscoveryOption.DuringExecution, _) => TestDataSourceUnfoldingStrategy.Fold, + // When using legacy strategy, there is no point in trying to "read" data during discovery + // as the ID generator will ignore it. + (null, TestIdGenerationStrategy.Legacy) => TestDataSourceUnfoldingStrategy.Fold, #pragma warning restore CS0618 // Type or member is obsolete + _ => TestDataSourceUnfoldingStrategy.Unfold, + }, + }; Dictionary? testRunParametersFromRunSettings = RunSettingsUtilities.GetTestRunParameters(runSettingsXml); foreach (Type type in types) @@ -109,7 +120,7 @@ internal ICollection EnumerateAssembly( } List testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals, - testDataSourceDiscovery, testIdGenerationStrategy, fixturesTests); + dataSourcesUnfoldingStrategy, testIdGenerationStrategy, fixturesTests); tests.AddRange(testsInType); } @@ -192,15 +203,14 @@ internal static string GetLoadExceptionDetails(ReflectionTypeLoadException ex) /// The reflected assembly name. /// True to discover test classes which are declared internal in /// addition to test classes which are declared public. - /// to use when generating tests. /// to use when generating TestId. /// a TypeEnumerator instance. - internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFileName, bool discoverInternals, TestDataSourceDiscoveryOption discoveryOption, TestIdGenerationStrategy testIdGenerationStrategy) + internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFileName, bool discoverInternals, TestIdGenerationStrategy testIdGenerationStrategy) { var typeValidator = new TypeValidator(ReflectHelper, discoverInternals); var testMethodValidator = new TestMethodValidator(ReflectHelper, discoverInternals); - return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator, discoveryOption, testIdGenerationStrategy); + return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator, testIdGenerationStrategy); } private List DiscoverTestsInType( @@ -209,7 +219,7 @@ private List DiscoverTestsInType( Type type, List warningMessages, bool discoverInternals, - TestDataSourceDiscoveryOption discoveryOption, + TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, TestIdGenerationStrategy testIdGenerationStrategy, HashSet fixturesTests) { @@ -225,24 +235,22 @@ private List DiscoverTestsInType( try { typeFullName = type.FullName; - TypeEnumerator testTypeEnumerator = GetTypeEnumerator(type, assemblyFileName, discoverInternals, discoveryOption, testIdGenerationStrategy); + TypeEnumerator testTypeEnumerator = GetTypeEnumerator(type, assemblyFileName, discoverInternals, testIdGenerationStrategy); List? unitTestCases = testTypeEnumerator.Enumerate(warningMessages); if (unitTestCases != null) { foreach (UnitTestElement test in unitTestCases) { - if (discoveryOption == TestDataSourceDiscoveryOption.DuringDiscovery) + if (_typeCache.GetTestMethodInfoForDiscovery(test.TestMethod) is { } testMethodInfo) { - Lazy testMethodInfo = GetTestMethodInfo(sourceLevelParameters, test); - // Add fixture tests like AssemblyInitialize, AssemblyCleanup, ClassInitialize, ClassCleanup. - if (MSTestSettings.CurrentSettings.ConsiderFixturesAsSpecialTests && testMethodInfo.Value is not null) + if (MSTestSettings.CurrentSettings.ConsiderFixturesAsSpecialTests) { - AddFixtureTests(testMethodInfo.Value, tests, fixturesTests); + AddFixtureTests(testMethodInfo, tests, fixturesTests); } - if (DynamicDataAttached(test, testMethodInfo, tests)) + if (TryUnfoldITestDataSources(test, testMethodInfo, dataSourcesUnfoldingStrategy, tests)) { continue; } @@ -264,46 +272,6 @@ private List DiscoverTestsInType( return tests; } - private Lazy GetTestMethodInfo(IDictionary sourceLevelParameters, UnitTestElement test) => - new(() => - { - // NOTE: From this place we don't have any path that would let the user write a message on the TestContext and we don't do - // anything with what would be printed anyway so we can simply use a simple StringWriter. - using var writer = new StringWriter(); - TestMethod testMethod = test.TestMethod; - MSTestAdapter.PlatformServices.Interface.ITestContext testContext = PlatformServiceProvider.Instance.GetTestContext(testMethod, writer, sourceLevelParameters); - return _typeCache.GetTestMethodInfo(testMethod, testContext, MSTestSettings.CurrentSettings.CaptureDebugTraces); - }); - - private static bool DynamicDataAttached(UnitTestElement test, Lazy testMethodInfo, List tests) - { - // It should always be `true`, but if any part of the chain is obsolete; it might not contain those. - // Since we depend on those properties, if they don't exist, we bail out early. - if (!test.TestMethod.HasManagedMethodAndTypeProperties) - { - return false; - } - - DynamicDataType originalDataType = test.TestMethod.DataType; - - // PERF: For perf we started setting DataType in TypeEnumerator, so when it is None we will not reach this line. - // But if we do run this code, we still reset it to None, because the code that determines if this is data drive test expects the value to be None - // and only sets it when needed. - // - // If you remove this line and acceptance tests still pass you are okay. - test.TestMethod.DataType = DynamicDataType.None; - - // The data source tests that we can process currently are those using attributes that - // implement ITestDataSource (i.e, DataRow and DynamicData attributes). - // However, for DataSourceAttribute, we currently don't have anyway to process it during discovery. - // (Note: this method is only called under discoveryOption == TestDataSourceDiscoveryOption.DuringDiscovery) - // So we want to return false from this method for non ITestDataSource (whether it's None or DataSourceAttribute). Otherwise, the test - // will be completely skipped which is wrong behavior. - return originalDataType == DynamicDataType.ITestDataSource && - testMethodInfo.Value != null && - TryProcessITestDataSourceTests(test, testMethodInfo.Value, tests); - } - private static void AddFixtureTests(TestMethodInfo testMethodInfo, List tests, HashSet fixtureTests) { string assemblyName = testMethodInfo.Parent.Parent.Assembly.GetName().Name!; @@ -384,100 +352,146 @@ static UnitTestElement GetFixtureTest(string classFullName, string assemblyLocat } } - private static bool TryProcessITestDataSourceTests(UnitTestElement test, TestMethodInfo testMethodInfo, List tests) + private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodInfo testMethodInfo, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, List tests) { + // It should always be `true`, but if any part of the chain is obsolete; it might not contain those. + // Since we depend on those properties, if they don't exist, we bail out early. + if (!test.TestMethod.HasManagedMethodAndTypeProperties) + { + return false; + } + // We don't have a special method to filter attributes that are not derived from Attribute, so we take all // attributes and filter them. We don't have to care if there is one, because this method is only entered when // there is at least one (we determine this in TypeEnumerator.GetTestFromMethod. IEnumerable testDataSources = ReflectHelper.Instance.GetDerivedAttributes(testMethodInfo.MethodInfo, inherit: false).OfType(); + // We need to use a temporary list to avoid adding tests to the main list if we fail to expand any data source. + List tempListOfTests = new(); + try { - return ProcessITestDataSourceTests(test, new(testMethodInfo.MethodInfo, test.DisplayName), testDataSources, tests); + bool isDataDriven = false; + foreach (ITestDataSource dataSource in testDataSources) + { + isDataDriven = true; + if (!TryUnfoldITestDataSource(dataSource, dataSourcesUnfoldingStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests)) + { + // TODO: Improve multi-source design! + // Ideally we would want to consider each data source separately but when one source cannot be expanded, + // we will run all sources from the given method so we need to bail-out "globally". + return false; + } + } + + if (tempListOfTests.Count > 0) + { + tests.AddRange(tempListOfTests); + } + + return isDataDriven; } catch (Exception ex) { string message = string.Format(CultureInfo.CurrentCulture, Resource.CannotEnumerateIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, ex); PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"DynamicDataEnumerator: {message}"); + + if (tempListOfTests.Count > 0) + { + tests.AddRange(tempListOfTests); + } + return false; } } - private static bool ProcessITestDataSourceTests(UnitTestElement test, ReflectionTestMethodInfo methodInfo, IEnumerable testDataSources, - List tests) + private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List tests) { - foreach (ITestDataSource dataSource in testDataSources) + var unfoldingCapability = dataSource as ITestDataSourceUnfoldingCapability; + + // If the global strategy is to fold and local has no strategy or uses Auto then return false + if (dataSourcesUnfoldingStrategy == TestDataSourceUnfoldingStrategy.Fold + && (unfoldingCapability is null || unfoldingCapability.UnfoldingStrategy == TestDataSourceUnfoldingStrategy.Auto)) { - IEnumerable? data; + return false; + } + + // If the data source specifies the unfolding strategy as fold then return false + if (unfoldingCapability?.UnfoldingStrategy == TestDataSourceUnfoldingStrategy.Fold) + { + return false; + } + + // Otherwise, unfold the data source and verify it can be serialized. + IEnumerable? data; - // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests. - // Any change made here should be reflected in TestMethodRunner.ExecuteDataSourceBasedTests as well. - data = dataSource.GetData(methodInfo); + // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests. + // Any change made here should be reflected in TestMethodRunner.ExecuteDataSourceBasedTests as well. + data = dataSource.GetData(methodInfo); - if (!data.Any()) + if (!data.Any()) + { + if (!MSTestSettings.CurrentSettings.ConsiderEmptyDataSourceAsInconclusive) { - if (!MSTestSettings.CurrentSettings.ConsiderEmptyDataSourceAsInconclusive) - { - throw dataSource.GetExceptionForEmptyDataSource(methodInfo); - } + throw dataSource.GetExceptionForEmptyDataSource(methodInfo); + } - UnitTestElement discoveredTest = test.Clone(); - // Make the test not data driven, because it had no data. - discoveredTest.TestMethod.DataType = DynamicDataType.None; - discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, null) ?? discoveredTest.DisplayName; + UnitTestElement discoveredTest = test.Clone(); + // Make the test not data driven, because it had no data. + discoveredTest.TestMethod.DataType = DynamicDataType.None; + discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, null) ?? discoveredTest.DisplayName; - tests.Add(discoveredTest); + tests.Add(discoveredTest); - continue; - } + return true; + } - var testDisplayNameFirstSeen = new Dictionary(); - var discoveredTests = new List(); - int index = 0; + var testDisplayNameFirstSeen = new Dictionary(); + var discoveredTests = new List(); + int index = 0; - foreach (object?[] d in data) - { - UnitTestElement discoveredTest = test.Clone(); - discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, d) ?? discoveredTest.DisplayName; + foreach (object?[] d in data) + { + UnitTestElement discoveredTest = test.Clone(); + discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, d) ?? discoveredTest.DisplayName; - // If strategy is DisplayName and we have a duplicate test name don't expand the test, bail out. + // If strategy is DisplayName and we have a duplicate test name don't expand the test, bail out. #pragma warning disable CS0618 // Type or member is obsolete - if (test.TestMethod.TestIdGenerationStrategy == TestIdGenerationStrategy.DisplayName - && testDisplayNameFirstSeen.TryGetValue(discoveredTest.DisplayName!, out int firstIndexSeen)) - { - string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_DuplicateDisplayName, firstIndexSeen, index, discoveredTest.DisplayName); - warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}"); + if (test.TestMethod.TestIdGenerationStrategy == TestIdGenerationStrategy.DisplayName + && testDisplayNameFirstSeen.TryGetValue(discoveredTest.DisplayName!, out int firstIndexSeen)) + { + string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_DuplicateDisplayName, firstIndexSeen, index, discoveredTest.DisplayName); + warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}"); - // Duplicated display name so bail out. Caller will handle adding the original test. - return false; - } + // Duplicated display name so bail out. Caller will handle adding the original test. + return false; + } #pragma warning restore CS0618 // Type or member is obsolete - try - { - discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d); - discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; - } - catch (SerializationException ex) - { - string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_CannotSerialize, index, discoveredTest.DisplayName); - warning += Environment.NewLine; - warning += ex.ToString(); - warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}"); - - // Serialization failed for the type, bail out. Caller will handle adding the original test. - return false; - } - - discoveredTests.Add(discoveredTest); - testDisplayNameFirstSeen[discoveredTest.DisplayName!] = index++; + try + { + discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d); + discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; + } + catch (SerializationException ex) + { + string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_CannotSerialize, index, discoveredTest.DisplayName); + warning += Environment.NewLine; + warning += ex.ToString(); + warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}"); + + // Serialization failed for the type, bail out. Caller will handle adding the original test. + return false; } - tests.AddRange(discoveredTests); + discoveredTests.Add(discoveredTest); + testDisplayNameFirstSeen[discoveredTest.DisplayName!] = index++; } + tests.AddRange(discoveredTests); + return true; } } diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs index 179dcd9c4b..ca1eaf47a1 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs @@ -25,7 +25,6 @@ internal class TypeEnumerator private readonly TypeValidator _typeValidator; private readonly TestMethodValidator _testMethodValidator; private readonly TestIdGenerationStrategy _testIdGenerationStrategy; - private readonly TestDataSourceDiscoveryOption _discoveryOption; private readonly ReflectHelper _reflectHelper; /// @@ -37,7 +36,7 @@ internal class TypeEnumerator /// The validator for test classes. /// The validator for test methods. /// to use when generating TestId. - internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, TestDataSourceDiscoveryOption discoveryOption, TestIdGenerationStrategy testIdGenerationStrategy) + internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, TestIdGenerationStrategy testIdGenerationStrategy) { _type = type; _assemblyFilePath = assemblyFilePath; @@ -45,7 +44,6 @@ internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflec _typeValidator = typeValidator; _testMethodValidator = testMethodValidator; _testIdGenerationStrategy = testIdGenerationStrategy; - _discoveryOption = discoveryOption; } /// @@ -154,22 +152,6 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool isDeclaredInT method.DeclaringType.Assembly); } - // PERF: When discovery option is set to DuringDiscovery, we will expand data on tests to one test case - // per data item. This will happen in AssemblyEnumerator. But AssemblyEnumerator does not have direct access to - // the method info or method attributes, so it would create a TestMethodInfo to see if the test is data driven. - // Creating TestMethodInfo is expensive and should be done only for a test that we know is data driven. - // - // So to optimize this we check if we have some data source attribute. Because here we have access to all attributes - // and we store that info in DataType. AssemblyEnumerator will pick this up and will get the real test data in the expensive way - // or it will skip over getting the data cheaply, when DataType = DynamicDataType.None. - // - // This needs to be done only when DuringDiscovery is set, because otherwise we would populate the DataType, but we would not populate - // and execution would not try to re-populate the data, because DataType is already set to data driven, so it would just throw error about empty data. - if (_discoveryOption == TestDataSourceDiscoveryOption.DuringDiscovery) - { - testMethod.DataType = GetDynamicDataType(method); - } - var testElement = new UnitTestElement(testMethod) { // Get compiler generated type name for async test method (either void returning or task returning). @@ -238,22 +220,4 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool isDeclaredInT return testElement; } - - private DynamicDataType GetDynamicDataType(MethodInfo method) - { - foreach (Attribute attribute in _reflectHelper.GetDerivedAttributes(method, inherit: true)) - { - if (AttributeComparer.IsDerived(attribute)) - { - return DynamicDataType.ITestDataSource; - } - - if (AttributeComparer.IsDerived(attribute)) - { - return DynamicDataType.DataSourceAttribute; - } - } - - return DynamicDataType.None; - } } diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TypeCache.cs b/src/Adapter/MSTest.TestAdapter/Execution/TypeCache.cs index 0ce373c00e..676b4ce83b 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TypeCache.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TypeCache.cs @@ -89,9 +89,6 @@ public IEnumerable AssemblyInfoListWithExecutableCleanupMethod /// /// Get the test method info corresponding to the parameter test Element. /// - /// The test Method. - /// The test Context. - /// Indicates whether the test method should capture debug traces. /// The . public TestMethodInfo? GetTestMethodInfo(TestMethod testMethod, ITestContext testContext, bool captureDebugTraces) { @@ -109,7 +106,29 @@ public IEnumerable AssemblyInfoListWithExecutableCleanupMethod } // Get the testMethod - return ResolveTestMethod(testMethod, testClassInfo, testContext, captureDebugTraces); + return ResolveTestMethodInfo(testMethod, testClassInfo, testContext, captureDebugTraces); + } + + /// + /// Get the test method info corresponding to the parameter test Element. + /// + /// The . + public TestMethodInfo? GetTestMethodInfoForDiscovery(TestMethod testMethod) + { + Guard.NotNull(testMethod); + + // Get the classInfo (This may throw as GetType calls assembly.GetType(..,true);) + TestClassInfo? testClassInfo = GetClassInfo(testMethod); + + if (testClassInfo == null) + { + // This means the class containing the test method could not be found. + // Return null so we return a not found result. + return null; + } + + // Get the testMethod + return ResolveTestMethodInfoForDiscovery(testMethod, testClassInfo); } /// @@ -704,23 +723,18 @@ private void UpdateInfoIfTestInitializeOrCleanupMethod( /// cannot be found, or a function is found that returns non-void, the result is /// set to error. /// - /// The test Method. - /// The test Class Info. - /// The test Context. - /// Indicates whether the test method should capture debug traces. /// /// The TestMethodInfo for the given test method. Null if the test method could not be found. /// - private TestMethodInfo? ResolveTestMethod(TestMethod testMethod, TestClassInfo testClassInfo, ITestContext testContext, bool captureDebugTraces) + private TestMethodInfo ResolveTestMethodInfo(TestMethod testMethod, TestClassInfo testClassInfo, ITestContext testContext, bool captureDebugTraces) { DebugEx.Assert(testMethod != null, "testMethod is Null"); DebugEx.Assert(testClassInfo != null, "testClassInfo is Null"); MethodInfo methodInfo = GetMethodInfoForTestMethod(testMethod, testClassInfo); - ExpectedExceptionBaseAttribute? expectedExceptionAttribute = _reflectionHelper.ResolveExpectedExceptionHelper(methodInfo, testMethod); TimeoutInfo timeout = GetTestTimeout(methodInfo, testMethod); - + ExpectedExceptionBaseAttribute? expectedExceptionAttribute = _reflectionHelper.ResolveExpectedExceptionHelper(methodInfo, testMethod); var testMethodOptions = new TestMethodOptions(timeout, expectedExceptionAttribute, testContext, captureDebugTraces, GetTestMethodAttribute(methodInfo, testClassInfo)); var testMethodInfo = new TestMethodInfo(methodInfo, testClassInfo, testMethodOptions); @@ -729,6 +743,14 @@ private void UpdateInfoIfTestInitializeOrCleanupMethod( return testMethodInfo; } + private TestMethodInfo ResolveTestMethodInfoForDiscovery(TestMethod testMethod, TestClassInfo testClassInfo) + { + MethodInfo methodInfo = GetMethodInfoForTestMethod(testMethod, testClassInfo); + + // Let's build a fake options type as it won't be used. + return new TestMethodInfo(methodInfo, testClassInfo, new(TimeoutInfo.FromTimeout(-1), null, null, false, null)); + } + /// /// Provides the Test Method Extension Attribute of the TestClass. /// diff --git a/src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs b/src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs index 9c6a8093b0..c576d3b6bd 100644 --- a/src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs +++ b/src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs @@ -278,11 +278,22 @@ internal static TestIdGenerationStrategy GetTestIdGenerationStrategy(Assembly as /// Gets TestDataSourceDiscovery assembly level attribute. /// /// The test assembly. + [Obsolete] internal static TestDataSourceDiscoveryOption? GetTestDataSourceDiscoveryOption(Assembly assembly) => PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(assembly, typeof(TestDataSourceDiscoveryAttribute)) .OfType() .FirstOrDefault()?.DiscoveryOption; + /// + /// Gets TestDataSourceOptions assembly level attribute. + /// + /// The test assembly. + /// The TestDataSourceOptionsAttribute if set. Null otherwise. + internal static TestDataSourceOptionsAttribute? GetTestDataSourceOptions(Assembly assembly) + => PlatformServiceProvider.Instance.ReflectionOperations.GetCustomAttributes(assembly, typeof(TestDataSourceOptionsAttribute)) + .OfType() + .FirstOrDefault(); + /// /// Get the parallelization behavior for a test method. /// diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs index 661f2fd551..270452d4db 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs @@ -11,7 +11,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// Attribute to define in-line data for a test method. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public class DataRowAttribute : Attribute, ITestDataSource +public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability { /// /// Initializes a new instance of the class. @@ -55,6 +55,9 @@ public DataRowAttribute(string?[]? stringArrayData) /// public string? DisplayName { get; set; } + /// + public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; + /// public IEnumerable GetData(MethodInfo methodInfo) => [Data]; diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs index 151685f3b8..2abb53a755 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs @@ -33,7 +33,7 @@ public enum DynamicDataSourceType /// Attribute to define dynamic data for a test method. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo +public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability { private readonly string _dynamicDataSourceName; private readonly DynamicDataSourceType _dynamicDataSourceType; @@ -84,6 +84,9 @@ public DynamicDataAttribute(string dynamicDataSourceName, Type dynamicDataDeclar /// public Type? DynamicDataDisplayNameDeclaringType { get; set; } + /// + public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; + /// public IEnumerable GetData(MethodInfo methodInfo) => DynamicDataProvider.Instance.GetData(_dynamicDataDeclaringType, _dynamicDataSourceType, _dynamicDataSourceName, methodInfo); diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryAttribute.cs index 9d7bd1f1db..660735c263 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryAttribute.cs @@ -7,6 +7,11 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// Specifies how to discover tests. /// [AttributeUsage(AttributeTargets.Assembly)] +#if NET6_0_OR_GREATER +[Obsolete("Attribute is obsolete and will be removed in v4, instead use 'TestDataSourceOptionsAttribute'.", DiagnosticId = "MSTESTOBS")] +#else +[Obsolete("Attribute is obsolete and will be removed in v4, instead use 'TestDataSourceOptionsAttribute'.")] +#endif public class TestDataSourceDiscoveryAttribute : Attribute { /// @@ -15,7 +20,8 @@ public class TestDataSourceDiscoveryAttribute : Attribute /// /// The to use when discovering tests. /// - public TestDataSourceDiscoveryAttribute(TestDataSourceDiscoveryOption discoveryOption) => DiscoveryOption = discoveryOption; + public TestDataSourceDiscoveryAttribute(TestDataSourceDiscoveryOption discoveryOption) + => DiscoveryOption = discoveryOption; /// /// Gets the discovery option. diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryOption.cs b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryOption.cs index 46c3be787b..25b9c9b32e 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryOption.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceDiscoveryOption.cs @@ -6,6 +6,11 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// /// The supported discovery modes for tests. /// +#if NET6_0_OR_GREATER +[Obsolete("Type is obsolete and will be removed in v4, instead use 'TestDataSourceUnfoldingStrategy'.", DiagnosticId = "MSTESTOBS")] +#else +[Obsolete("Type is obsolete and will be removed in v4, instead use 'TestDataSourceUnfoldingStrategy'.")] +#endif public enum TestDataSourceDiscoveryOption { /// diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs new file mode 100644 index 0000000000..6550f28093 --- /dev/null +++ b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Specifies options for all of the current assembly. +/// +/// +/// These options can be override by individual attribute. +[AttributeUsage(AttributeTargets.Assembly, Inherited = false)] +public sealed class TestDataSourceOptionsAttribute : Attribute +{ + /// + /// Initializes a new instance of the class. + /// + /// + /// The to use when executing parameterized tests. + /// + public TestDataSourceOptionsAttribute(TestDataSourceUnfoldingStrategy unfoldingStrategy) + => UnfoldingStrategy = unfoldingStrategy; + + /// + /// Gets the test unfolding strategy. + /// + public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; } +} diff --git a/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs new file mode 100644 index 0000000000..2f5ced8910 --- /dev/null +++ b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Specifies the capability of a test data source to define how parameterized tests should be executed, either as +/// individual test cases for each data row or as a single test case. This affects the test results and the UI +/// representation of the tests. +/// +public interface ITestDataSourceUnfoldingCapability +{ + TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; } +} + +/// +/// Specifies how parameterized tests should be executed, either as individual test cases for each data row or as a +/// single test case. This affects the test results and the UI representation of the tests. +/// +public enum TestDataSourceUnfoldingStrategy : byte +{ + /// + /// MSTest will decide whether to unfold the parameterized test based on value from the assembly level attribute + /// . If no assembly level attribute is specified, then the default + /// configuration is to unfold. + /// + Auto, + + /// + /// Each data row is treated as a separate test case. + /// + Unfold, + + /// + /// The parameterized test is not unfolded; all data rows are treated as a single test case. + /// + Fold, +} diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index ab058de62d..d3d4ae8281 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,14 @@ #nullable enable +Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.UnfoldingStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.UnfoldingStrategy.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.UnfoldingStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.UnfoldingStrategy.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceUnfoldingCapability +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceUnfoldingCapability.UnfoldingStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.TestDataSourceOptionsAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy unfoldingStrategy) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.UnfoldingStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.Auto = 0 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.Fold = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.Unfold = 1 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs index e36a2f78ef..636ac3737e 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Collections.Immutable; + using Microsoft.MSTestV2.CLIAutomation; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; namespace MSTest.IntegrationTests; @@ -21,24 +24,36 @@ public void CustomTestDataSourceTests() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomTestDataSourceTestMethod1"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomTestDataSourceTestMethod1"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.ContainsTestsPassed(testResults, "CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)"); } + public void CustomEmptyTestDataSourceTests() + { + // Arrange + string assemblyPath = GetAssetFullPath(TestAssetName); + + // Act + ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomEmptyTestDataSourceTestMethod"); + ImmutableArray testResults = RunTests(testCases); + + // Assert + VerifyE2E.ContainsTestsFailed(testResults, new string[] { null }); + } + public void AssertExtensibilityTests() { // Arrange string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FxExtensibilityTestProject.AssertExTest"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FxExtensibilityTestProject.AssertExTest"); + ImmutableArray testResults = RunTests(testCases); // Assert - VerifyE2E.ContainsTestsPassed(testResults, "BasicAssertExtensionTest", "ChainedAssertExtensionTest"); VerifyE2E.ContainsTestsFailed(testResults, "BasicFailingAssertExtensionTest", "ChainedFailingAssertExtensionTest"); } @@ -48,8 +63,8 @@ public void ExecuteCustomTestExtensibilityTests() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "(Name~CustomTestMethod1)|(Name~CustomTestClass1)"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "(Name~CustomTestMethod1)|(Name~CustomTestClass1)"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.ContainsTestsPassed( @@ -75,8 +90,8 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "Name~CustomTestMethod2"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "Name~CustomTestMethod2"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -94,4 +109,24 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests() "CustomTestMethod2 (\"C\")", "CustomTestMethod2 (\"C\")"); } + + public void WhenUsingCustomITestDataSourceWithExpansionDisabled_RespectSetting() + { + // Arrange + string assemblyPath = GetAssetFullPath(TestAssetName); + + // Act + ImmutableArray testCases = DiscoverTests(assemblyPath, "CustomDisableExpansionTestDataSourceTestMethod1"); + ImmutableArray testResults = RunTests(testCases); + + // Assert + Verify(testCases.Length == 1); + + VerifyE2E.TestsPassed( + testResults, + "CustomDisableExpansionTestDataSourceTestMethod1 (1,2,3)", + "CustomDisableExpansionTestDataSourceTestMethod1 (4,5,6)"); + + VerifyE2E.TestsFailed(testResults); + } } diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs index 952119a8bb..a370027ea9 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Collections.Immutable; + using Microsoft.MSTestV2.CLIAutomation; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; namespace MSTest.IntegrationTests; @@ -15,8 +18,8 @@ public void ExecuteDynamicDataTests() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DynamicDataTests"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -62,8 +65,8 @@ public void ExecuteDynamicDataTestsWithCategoryFilter() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DynamicDataWithCategory"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DynamicDataWithCategory"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.ContainsTestsPassed( @@ -73,4 +76,37 @@ public void ExecuteDynamicDataTestsWithCategoryFilter() VerifyE2E.FailedTestCount(testResults, 0); } + + public void ExecuteNonExpandableDynamicDataTests() + { + // Arrange + string assemblyPath = GetAssetFullPath(TestAssetName); + + // Act + ImmutableArray testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DisableExpansionTests"); + ImmutableArray testResults = RunTests(testCases); + + // Assert + Verify(testCases.Length == 6); + + VerifyE2E.TestsPassed( + testResults, + "TestPropertySourceOnCurrentType (1,a)", + "TestPropertySourceOnCurrentType (2,b)", + "TestPropertySourceOnDifferentType (3,c)", + "TestPropertySourceOnDifferentType (4,d)", + "TestPropertyWithTwoSourcesAndSecondDisablesExpansion (1,a)", + "TestPropertyWithTwoSourcesAndSecondDisablesExpansion (2,b)", + "TestPropertyWithTwoSourcesAndSecondDisablesExpansion (3,c)", + "TestPropertyWithTwoSourcesAndSecondDisablesExpansion (4,d)", + "TestMethodSourceOnDifferentType (3,c)", + "TestMethodSourceOnDifferentType (4,d)", + "TestPropertyWithTwoSourcesAndFirstDisablesExpansion (1,a)", + "TestPropertyWithTwoSourcesAndFirstDisablesExpansion (2,b)", + "TestPropertyWithTwoSourcesAndFirstDisablesExpansion (3,c)", + "TestPropertyWithTwoSourcesAndFirstDisablesExpansion (4,d)", + "TestMethodSourceOnCurrentType (1,a)", + "TestMethodSourceOnCurrentType (2,b)"); + VerifyE2E.FailedTestCount(testResults, 0); + } } diff --git a/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs b/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs index 14ba3ffaf2..44f953da41 100644 --- a/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs +++ b/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Parameterized tests/DataExtensibilityTests.cs @@ -13,7 +13,7 @@ public void ExecuteTestDataSourceExtensibilityTests() { InvokeVsTestForExecution([TestAssetName]); ValidatePassedTestsContain("CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)"); - ValidateFailedTestsContain(false, "FxExtensibilityTestProject.TestDataSourceExTests.CustomTestDataSourceTestMethod1"); + ValidateFailedTestsContain(false, "FxExtensibilityTestProject.TestDataSourceExTests.CustomEmptyTestDataSourceTestMethod"); } public void ExecuteDynamicDataExtensibilityTests() diff --git a/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Parameterized tests/DynamicDataTests.cs b/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Parameterized tests/DynamicDataTests.cs index 8cebe74926..a9e134080f 100644 --- a/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Parameterized tests/DynamicDataTests.cs +++ b/test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/Parameterized tests/DynamicDataTests.cs @@ -14,7 +14,7 @@ public void ExecuteDynamicDataTests() // Arrange & Act InvokeVsTestForExecution( [TestAssetName], - testCaseFilter: "DynamicDataTest"); + testCaseFilter: "ClassName=DataSourceTestProject.DynamicDataTests"); // Assert ValidatePassedTests( diff --git a/test/IntegrationTests/TestAssets/DynamicDataTestProject/DisableExpansionTests.cs b/test/IntegrationTests/TestAssets/DynamicDataTestProject/DisableExpansionTests.cs new file mode 100644 index 0000000000..c22a047e3f --- /dev/null +++ b/test/IntegrationTests/TestAssets/DynamicDataTestProject/DisableExpansionTests.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DynamicDataTestProject; + +[TestClass] +public sealed class DisableExpansionTests +{ + [TestMethod] + [DynamicData(nameof(PropertySource), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + public void TestPropertySourceOnCurrentType(int a, string s) + { + } + + [TestMethod] + [DynamicData(nameof(MethodSource), DynamicDataSourceType.Method, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + public void TestMethodSourceOnCurrentType(int a, string s) + { + } + + [TestMethod] + [DynamicData(nameof(PropertySource), typeof(DataSourceHelper), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + public void TestPropertySourceOnDifferentType(int a, string s) + { + } + + [TestMethod] + [DynamicData(nameof(MethodSource), typeof(DataSourceHelper), DynamicDataSourceType.Method, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + public void TestMethodSourceOnDifferentType(int a, string s) + { + } + + [TestMethod] + [DynamicData(nameof(PropertySource), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + [DynamicData(nameof(PropertySource), typeof(DataSourceHelper))] + public void TestPropertyWithTwoSourcesAndFirstDisablesExpansion(int a, string s) + { + } + + [TestMethod] + [DynamicData(nameof(PropertySource))] + [DynamicData(nameof(PropertySource), typeof(DataSourceHelper), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + public void TestPropertyWithTwoSourcesAndSecondDisablesExpansion(int a, string s) + { + } + + private static IEnumerable PropertySource => MethodSource(); + + private static IEnumerable MethodSource() + { + yield return new object[] { 1, "a" }; + yield return new object[] { 2, "b" }; + } +} + +public class DataSourceHelper +{ + public static IEnumerable PropertySource => MethodSource(); + + public static IEnumerable MethodSource() + { + yield return new object[] { 3, "c" }; + yield return new object[] { 4, "d" }; + } +} diff --git a/test/IntegrationTests/TestAssets/FxExtensibilityTestProject/TestDataSourceExTests.cs b/test/IntegrationTests/TestAssets/FxExtensibilityTestProject/TestDataSourceExTests.cs index b9a171c4b1..a040957911 100644 --- a/test/IntegrationTests/TestAssets/FxExtensibilityTestProject/TestDataSourceExTests.cs +++ b/test/IntegrationTests/TestAssets/FxExtensibilityTestProject/TestDataSourceExTests.cs @@ -13,13 +13,27 @@ public class TestDataSourceExTests { [TestMethod] [CustomTestDataSource] - [CustomEmptyTestDataSource] public void CustomTestDataSourceTestMethod1(int a, int b, int c) { Assert.AreEqual(1, a % 3); Assert.AreEqual(2, b % 3); Assert.AreEqual(0, c % 3); } + + [TestMethod] + [CustomDisableExpansionTestDataSource] + public void CustomDisableExpansionTestDataSourceTestMethod1(int a, int b, int c) + { + } + + [TestMethod] + [CustomEmptyTestDataSource] + public void CustomEmptyTestDataSourceTestMethod(int a, int b, int c) + { + Assert.AreEqual(1, a % 3); + Assert.AreEqual(2, b % 3); + Assert.AreEqual(0, c % 3); + } } [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] @@ -37,3 +51,13 @@ public class CustomEmptyTestDataSourceAttribute : Attribute, ITestDataSource public string GetDisplayName(MethodInfo methodInfo, object[] data) => data != null ? string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)) : null; } + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] +public class CustomDisableExpansionTestDataSourceAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability +{ + public TestDataSourceUnfoldingStrategy UnfoldingStrategy => TestDataSourceUnfoldingStrategy.Fold; + + public IEnumerable GetData(MethodInfo methodInfo) => [[1, 2, 3], [4, 5, 6]]; + + public string GetDisplayName(MethodInfo methodInfo, object[] data) => data != null ? string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)) : null; +} diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Discovery/AssemblyEnumeratorTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Discovery/AssemblyEnumeratorTests.cs index d09607fabc..b98e465b0c 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Discovery/AssemblyEnumeratorTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Discovery/AssemblyEnumeratorTests.cs @@ -396,7 +396,15 @@ private static Mock CreateMockTestableAssembly() mockAssembly .Setup(a => a.GetCustomAttributes( + typeof(TestDataSourceOptionsAttribute), + true)) + .Returns(Array.Empty()); + + mockAssembly + .Setup(a => a.GetCustomAttributes( +#pragma warning disable CS0618 // Type or member is obsolete typeof(TestDataSourceDiscoveryAttribute), +#pragma warning restore CS0618 // Type or member is obsolete true)) .Returns(Array.Empty()); @@ -429,14 +437,12 @@ internal TestableAssemblyEnumerator() reflectHelper.Object, typeValidator.Object, testMethodValidator.Object, - TestDataSourceDiscoveryOption.DuringExecution, TestIdGenerationStrategy.FullyQualified); } internal Mock MockTypeEnumerator { get; set; } - internal override TypeEnumerator GetTypeEnumerator(Type type, string assemblyFileName, bool discoverInternals, - TestDataSourceDiscoveryOption discoveryOption, TestIdGenerationStrategy testIdGenerationStrategy) + internal override TypeEnumerator GetTypeEnumerator(Type type, string assemblyFileName, bool discoverInternals, TestIdGenerationStrategy testIdGenerationStrategy) => MockTypeEnumerator.Object; } diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs index 3e2e123c22..12ef295dc1 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs @@ -575,16 +575,13 @@ private void SetupTestClassAndTestMethods(bool isValidTestClass, bool isValidTes rh => rh.IsMethodDeclaredInSameAssemblyAsType(It.IsAny(), It.IsAny())).Returns(isMethodFromSameAssembly); } - private TypeEnumerator GetTypeEnumeratorInstance(Type type, string assemblyName, - TestDataSourceDiscoveryOption discoveryOption = TestDataSourceDiscoveryOption.DuringExecution, - TestIdGenerationStrategy idGenerationStrategy = TestIdGenerationStrategy.FullyQualified) + private TypeEnumerator GetTypeEnumeratorInstance(Type type, string assemblyName, TestIdGenerationStrategy idGenerationStrategy = TestIdGenerationStrategy.FullyQualified) => new( type, assemblyName, _mockReflectHelper.Object, _mockTypeValidator.Object, _mockTestMethodValidator.Object, - discoveryOption, idGenerationStrategy); #endregion