public static TfsTestFilter CreateTestFilter(ITestCaseFilterExpression filterExpression)
{
var context = Substitute.For <IRunContext>();
context.GetTestCaseFilter(null, null).ReturnsForAnyArgs(filterExpression);
return(new TfsTestFilter(context));
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
// more on filtering
// https://github.com/nunit/nunit3-vs-adapter/blob/master/src/NUnitTestAdapter/VsTestFilter.cs
List <string> supportedProperties = new List <string>();
supportedProperties.Add("FullyQualifiedName");
ITestCaseFilterExpression fe = runContext.GetTestCaseFilter(supportedProperties, PropertyProvider);
log.Debug("Run settings:\n" + runContext.RunSettings.SettingsXml);
log.Debug("RunTests from Test Cases");
foreach (TestCase tc in tests)
{
if (fe == null || fe.MatchTestCase(tc, p => PropertyValueProvider(tc, p)))
{
log.Debug("Run test case: " + tc.FullyQualifiedName + " / " + tc.Id);
frameworkHandle.RecordStart(tc);
DateTime startTime = DateTime.Now;
TestResult tr = runner.RunTest(tc, runContext);
DateTime endTime = DateTime.Now;
tr.Duration = endTime - startTime;
frameworkHandle.RecordEnd(tc, tr.Outcome);
frameworkHandle.RecordResult(tr);
}
else
{
log.Debug("Test case filtered out: " + tc.FullyQualifiedName + " / " + tc.Id);
}
}
}
bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression filter)
{
filter = null;
try
{
// In Microsoft.VisualStudio.TestPlatform.ObjectModel V11 IRunContext provides a TestCaseFilter property
// GetTestCaseFilter only exists in V12+
#if PLATFORM_DOTNET
var getTestCaseFilterMethod = runContext.GetType().GetRuntimeMethod("GetTestCaseFilter", new[] { typeof(IEnumerable<string>), typeof(Func<string, TestProperty>) });
#else
var getTestCaseFilterMethod = runContext.GetType().GetMethod("GetTestCaseFilter");
#endif
if (getTestCaseFilterMethod != null)
filter = (ITestCaseFilterExpression)getTestCaseFilterMethod.Invoke(runContext, new object[] { supportedPropertyNames, null });
return true;
}
catch (TargetInvocationException e)
{
var innerExceptionType = e.InnerException.GetType();
if (innerExceptionType.FullName.EndsWith("TestPlatformFormatException", StringComparison.OrdinalIgnoreCase))
{
logger.LogError("{0}: Exception discovering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.InnerException.Message);
return false;
}
throw;
}
}
public InternalDiscoveryContext(string runSettings, string testCaseFilter)
{
this.runSettings = new InternalRunSettings(runSettings);
if (testCaseFilter != null)
{
filter = TestCaseFilterFactory.ParseTestFilter(testCaseFilter);
}
}
public static VsTestFilter CreateTestFilter(ITestCaseFilterExpression filterExpression)
{
var context = Substitute.For <IRunContext>();
context.GetTestCaseFilter(null, null).ReturnsForAnyArgs(filterExpression);
var settings = Substitute.For <IAdapterSettings>();
settings.DiscoveryMethod.Returns(DiscoveryMethod.Legacy);
return(VsTestFilterFactory.CreateVsTestFilter(settings, context));
}
private static bool MatchTestFilter(ITestCaseFilterExpression filterExpression, TestCase test, TestMethodFilter testMethodFilter)
{
if (filterExpression != null && filterExpression.MatchTestCase(test, p => testMethodFilter.PropertyValueProvider(test, p)) == false)
{
// Skip test if not fitting filter criteria.
return(false);
}
return(true);
}
private ITestCaseFilterExpression GetFilterExpression()
{
ITestCaseFilterExpression filterExpression = RunContext.GetTestCaseFilter(AllProperties, PropertyProvider);
string message = filterExpression == null
? "No test case filter provided"
: $"Test case filter: {filterExpression.TestCaseFilterValue}";
TestEnvironment.DebugInfo(message);
return(filterExpression);
}
private bool Matches(TestCase testCase, ITestCaseFilterExpression filterExpression)
{
bool matches =
filterExpression.MatchTestCase(testCase, propertyName => PropertyValueProvider(testCase, propertyName));
string message = matches
? String.Format(Resources.Matches, testCase.DisplayName, filterExpression.TestCaseFilterValue)
: String.Format(Resources.DoesntMatch, testCase.DisplayName, filterExpression.TestCaseFilterValue);
_logger.DebugInfo(message);
return(matches);
}
private bool Matches(TestCase testCase, ITestCaseFilterExpression filterExpression)
{
bool matches =
filterExpression.MatchTestCase(testCase, propertyName => PropertyValueProvider(testCase, propertyName));
string message = matches
? $"{testCase.DisplayName} matches {filterExpression.TestCaseFilterValue}"
: $"{testCase.DisplayName} does not match {filterExpression.TestCaseFilterValue}";
_testEnvironment.DebugInfo(message);
return(matches);
}
private ITestCaseFilterExpression GetFilterExpression()
{
try
{
ITestCaseFilterExpression filterExpression = _runContext.GetTestCaseFilter(_allPropertyNames, PropertyProvider);
string message = filterExpression == null
? "No test case filter provided"
: $"Test case filter: {filterExpression.TestCaseFilterValue}";
_logger.DebugInfo(message);
return(filterExpression);
}
catch (TestPlatformFormatException e)
{
_logger.LogError($"Test case filter is invalid: {e.Message}");
return(null);
}
}
private ITestCaseFilterExpression GetFilterExpression()
{
try
{
ITestCaseFilterExpression filterExpression = _runContext.GetTestCaseFilter(_allPropertyNames, PropertyProvider);
string message = filterExpression == null
? Resources.NoTestCaseFilter
: String.Format(Resources.TestCaseFilter, filterExpression.TestCaseFilterValue);
_logger.DebugInfo(message);
return(filterExpression);
}
catch (TestPlatformFormatException e)
{
_logger.LogError(String.Format(Resources.FilterInvalid, e.Message));
return(null);
}
}
private ITestCaseFilterExpression GetFilterExpression()
{
try
{
ITestCaseFilterExpression filterExpression = _runContext.GetTestCaseFilter(AllProperties, PropertyProvider);
string message = filterExpression == null
? "No test case filter provided"
: $"Test case filter: {filterExpression.TestCaseFilterValue}";
_testEnvironment.DebugInfo(message);
return(filterExpression);
}
catch (TestPlatformFormatException e)
{
_testEnvironment.LogWarning(e.Message);
return(null);
}
}
public IEnumerable <TestCase> GetFilteredTestList(IEnumerable <TestCase> testCases, IRunContext runContext, LoggerHelper logger, string assemblyFileName)
{
ITestCaseFilterExpression filter = null;
if (GetTestCaseFilterExpression(runContext, logger, assemblyFileName, out filter))
{
if (filter != null)
{
return(testCases.Where(testCase => filter.MatchTestCase(testCase, (p) => PropertyProvider(testCase, p))));
}
}
else
{
// Error while filtering, ensure discovered test list is empty
return(Enumerable.Empty <TestCase>());
}
// No filter is specified return the original list
return(testCases);
}
/// <summary>
/// Returns ITestCaseFilterExpression for TestProperties supported by adapter.
/// </summary>
/// <param name="context">The current context of the run.</param>
/// <param name="logger">Handler to report test messages/start/end and results.</param>
/// <param name="filterHasError">Indicates that the filter is unsupported/has an error.</param>
/// <returns>A filter expression.</returns>
internal ITestCaseFilterExpression GetFilterExpression(IDiscoveryContext context, IMessageLogger logger, out bool filterHasError)
{
filterHasError = false;
ITestCaseFilterExpression filter = null;
if (context != null)
{
try
{
filter = (context is IRunContext) ? this.GetTestCaseFilterFromRunContext(context as IRunContext) : this.GetTestCaseFilterFromDiscoveryContext(context);
}
catch (TestPlatformFormatException ex)
{
filterHasError = true;
logger.SendMessage(TestMessageLevel.Error, ex.Message);
}
}
return(filter);
}
/// <summary>
/// Returns ITestCaseFilterExpression for TestProperties supported by adapter.
/// </summary>
/// <param name="runContext">The current context of the run.</param>
/// <param name="testExecutionRecorder">Handler to report test messages/start/end and results.</param>
/// <param name="filterHasError">Indicates that the filter is unsupported/has an error.</param>
/// <returns>A filter expression.</returns>
internal ITestCaseFilterExpression GetFilterExpression(IRunContext runContext, IMessageLogger testExecutionRecorder, out bool filterHasError)
{
filterHasError = false;
ITestCaseFilterExpression filter = null;
if (runContext != null)
{
try
{
filter = runContext.GetTestCaseFilter(this.supportedProperties.Keys, this.PropertyProvider);
}
catch (TestPlatformFormatException ex)
{
filterHasError = true;
testExecutionRecorder.SendMessage(TestMessageLevel.Error, ex.Message);
}
}
return(filter);
}
bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression filter)
{
filter = null;
try
{
// In Microsoft.VisualStudio.TestPlatform.ObjectModel V11 IRunContext provides a TestCaseFilter property
// GetTestCaseFilter only exists in V12+
#if PLATFORM_DOTNET
var getTestCaseFilterMethod = runContext.GetType().GetRuntimeMethod("GetTestCaseFilter", new[] { typeof(IEnumerable <string>), typeof(Func <string, TestProperty>) });
#else
var getTestCaseFilterMethod = runContext.GetType().GetMethod("GetTestCaseFilter");
#endif
if (getTestCaseFilterMethod != null)
{
filter = (ITestCaseFilterExpression)getTestCaseFilterMethod.Invoke(runContext, new object[] { supportedPropertyNames, null });
}
return(true);
}
catch (TargetInvocationException e)
{
var innerExceptionType = e.InnerException.GetType();
if (innerExceptionType.FullName.EndsWith("TestPlatformFormatException", StringComparison.OrdinalIgnoreCase))
{
logger.LogError("{0}: Exception discovering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.InnerException.Message);
return(false);
}
throw;
}
}
public bool Matches(TestCase testCase)
{
ITestCaseFilterExpression filterExpression = GetFilterExpression();
return(filterExpression == null || Matches(testCase, filterExpression));
}
bool GetTestCaseFilterExpressionFromDiscoveryContext(IDiscoveryContext discoveryContext, LoggerHelper logger, out ITestCaseFilterExpression filter)
{
filter = null;
if (discoveryContext is IRunContext runContext)
{
try
{
filter = runContext.GetTestCaseFilter(supportedPropertyNames, null);
return(true);
}
catch (TestPlatformException e)
{
logger.LogWarning("Exception filtering tests: {0}", e.Message);
return(false);
}
}
else
{
try
{
#if WINDOWS_UAP
// on UWP .Net Native Tool Chain we are not able to run methods via invoke, act like no filter was specified for UWP
#else
// GetTestCaseFilter is present on DiscoveryContext but not in IDiscoveryContext interface
var method = discoveryContext.GetType().GetRuntimeMethod("GetTestCaseFilter", new[] { typeof(IEnumerable <string>), typeof(Func <string, TestProperty>) });
filter = (ITestCaseFilterExpression)method?.Invoke(discoveryContext, new object[] { supportedPropertyNames, null });
#endif
return(true);
}
catch (TargetInvocationException e)
{
if (e?.InnerException is TestPlatformException ex)
{
logger.LogWarning("Exception filtering tests: {0}", ex.InnerException?.Message);
return(false);
}
throw e.InnerException;
}
}
}
public IEnumerable <TestCase> Filter(IEnumerable <TestCase> testCases)
{
ITestCaseFilterExpression filterExpression = GetFilterExpression();
return(filterExpression == null ? testCases : testCases.Where(testCase => Matches(testCase, filterExpression)));
}
private bool Matches(TestCase testCase, ITestCaseFilterExpression filterExpression)
{
bool matches =
filterExpression.MatchTestCase(testCase, propertyName => PropertyValueProvider(testCase, propertyName));
string message = matches
? $"{testCase.DisplayName} matches {filterExpression.TestCaseFilterValue}"
: $"{testCase.DisplayName} does not match {filterExpression.TestCaseFilterValue}";
_testEnvironment.DebugInfo(message);
return matches;
}
bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression filter)
{
filter = null;
try
{
filter = runContext.GetTestCaseFilter(supportedPropertyNames, null);
return(true);
}
catch (TestPlatformFormatException e)
{
logger.LogWarning("{0}: Exception filtering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.Message);
return(false);
}
}
internal void SendTestCases(string source, IEnumerable <UnitTestElement> testElements, ITestCaseDiscoverySink discoverySink, IDiscoveryContext discoveryContext, IMessageLogger logger)
{
var shouldCollectSourceInformation = MSTestSettings.RunConfigurationSettings.CollectSourceInformation;
var navigationSessions = new Dictionary <string, object>();
try
{
if (shouldCollectSourceInformation)
{
navigationSessions.Add(source, PlatformServiceProvider.Instance.FileOperations.CreateNavigationSession(source));
}
// Get filter expression and skip discovery in case filter expression has parsing error.
ITestCaseFilterExpression filterExpression = this.TestMethodFilter.GetFilterExpression(discoveryContext, logger, out var filterHasError);
if (filterHasError)
{
return;
}
foreach (var testElement in testElements)
{
var testCase = testElement.ToTestCase();
// Filter tests based on test case filters
if (filterExpression != null && filterExpression.MatchTestCase(testCase, (p) => this.TestMethodFilter.PropertyValueProvider(testCase, p)) == false)
{
continue;
}
if (shouldCollectSourceInformation)
{
string testSource = testElement.TestMethod.DeclaringAssemblyName ?? source;
if (!navigationSessions.TryGetValue(testSource, out var testNavigationSession))
{
testNavigationSession = PlatformServiceProvider.Instance.FileOperations.CreateNavigationSession(testSource);
navigationSessions.Add(testSource, testNavigationSession);
}
if (testNavigationSession != null)
{
var className = testElement.TestMethod.DeclaringClassFullName
?? testElement.TestMethod.FullClassName;
var methodName = testElement.TestMethod.Name;
// If it is async test method use compiler generated type and method name for navigation data.
if (!string.IsNullOrEmpty(testElement.AsyncTypeName))
{
className = testElement.AsyncTypeName;
// compiler generated method name is "MoveNext".
methodName = "MoveNext";
}
PlatformServiceProvider.Instance.FileOperations.GetNavigationData(
testNavigationSession,
className,
methodName,
out var minLineNumber,
out var fileName);
if (!string.IsNullOrEmpty(fileName))
{
testCase.LineNumber = minLineNumber;
testCase.CodeFilePath = fileName;
}
}
}
discoverySink.SendTestCase(testCase);
}
}
finally
{
foreach (object navigationSession in navigationSessions.Values)
{
PlatformServiceProvider.Instance.FileOperations.DisposeNavigationSession(navigationSession);
}
}
}
public static void AssertExpectedResult(ITestCaseFilterExpression filterExpression, IReadOnlyCollection <TestCase> testCases, IReadOnlyCollection <string> expectedMatchingTestNames)
{
var matchingTestCases = CreateTestFilter(filterExpression).CheckFilter(testCases);
Assert.That(matchingTestCases.Select(t => t.FullyQualifiedName), Is.EquivalentTo(expectedMatchingTestNames));
}
