void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (RunSettingsHelper.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
try
{
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList()
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
// Apply any filtering
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
// Force unique names if there is more than 1 testcase with the same name
foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1))
{
foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames)
{
discoveredTestCaseWithDuplicateName.ForceUniqueName();
}
}
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
// We are in Run Specific tests scenario, the `TestCase` objects are available.
// Query the `TestCase` objects to find XunitTestCase objects.
foreach (var vstestCase in runInfo.TestCases)
{
var xunitTestCase = Deserialize(logger, controller, vstestCase);
if (xunitTestCase != null)
{
testCasesMap.Add(xunitTestCase.UniqueID, vstestCase);
testCases.Add(xunitTestCase);
}
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (RunSettingsHelper.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
void DiscoverTests <TVisitor>(IEnumerable <string> sources,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null)
where TVisitor : IVsDiscoverySink, IDisposable
{
try
{
RemotingUtility.CleanUpRegisteredChannels();
using (AssemblyHelper.SubscribeResolve())
{
foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources)
{
var assemblyFileName = GetAssemblyFileName(assemblyFileNameCanBeWithoutAbsolutePath);
var configuration = LoadConfiguration(assemblyFileName);
var fileName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = configuration.ShadowCopyOrDefault;
var diagnosticSink = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault);
using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
if (!DiscoverTestsInSource(framework, logger, testPlatformContext, visitorFactory, visitComplete, assemblyFileName, shadowCopy, configuration))
{
break;
}
}
}
}
catch (Exception e)
{
logger.LogWarning("Exception discovering tests: {0}", e.Unwrap());
}
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assemblyDisplayName = "(unknown assembly)";
try
{
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (RunSettingsHelper.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
// Apply any filtering
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
try
{
var serializations = runInfo.TestCases
.Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
.ToList();
var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);
if (deserializedTestCasesByUniqueId == null)
{
logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
}
else
{
logger.Log("Received {0} results from {1} requests", deserializedTestCasesByUniqueId.Count, serializations.Count);
for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
{
try
{
var kvp = deserializedTestCasesByUniqueId[idx];
var vsTestCase = runInfo.TestCases[idx];
if (kvp.Value == null)
{
logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
}
else
{
testCasesMap.Add(kvp.Key, vsTestCase);
testCases.Add(kvp.Value);
}
}
catch (Exception ex)
{
logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
}
}
}
}
catch (Exception ex)
{
logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (RunSettingsHelper.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (RunSettingsHelper.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
try
{
#if PLATFORM_DOTNET
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
{
var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) })
.Where(tc => tc.xunit != null)
.ToDictionary(tc => tc.xunit, tc => tc.vs);
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (RunSettingsHelper.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(xunitTestCases.Keys.ToList(), resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
void DiscoverTests <TVisitor>(IRunSettings runSettings,
IEnumerable <string> sources,
LoggerHelper logger,
Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null)
where TVisitor : IVsDiscoverySink, IDisposable
{
try
{
RemotingUtility.CleanUpRegisteredChannels();
using (AssemblyHelper.SubscribeResolve())
{
#if NET35 || NETCOREAPP1_0
// Reads settings like disabling appdomains, parallel etc.
// Do this first before invoking any thing else to ensure correct settings for the run
RunSettingsHelper.ReadRunSettings(runSettings?.SettingsXml);
#endif
var reporterMessageHandler = GetRunnerReporter(sources).CreateMessageHandler(new VisualStudioRunnerLogger(logger));
foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources)
{
var assemblyFileName = assemblyFileNameCanBeWithoutAbsolutePath;
#if !PLATFORM_DOTNET
assemblyFileName = Path.GetFullPath(assemblyFileNameCanBeWithoutAbsolutePath);
#endif
var assembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName
};
var configuration = LoadConfiguration(assemblyFileName);
var fileName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = configuration.ShadowCopyOrDefault;
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
}
}
else
{
var diagnosticSink = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault);
using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
{
var targetFramework = framework.TargetFramework;
if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
}
}
else
{
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
{
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
framework.Find(includeSourceInformation: true, discoveryMessageSink: visitor, discoveryOptions: discoveryOptions);
var totalTests = visitor.Finish();
visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
}
}
}
}
}
catch (Exception e)
{
var ex = e.Unwrap();
var fileNotFound = ex as FileNotFoundException;
#if !PLATFORM_DOTNET
var fileLoad = ex as FileLoadException;
#endif
if (ex is InvalidOperationException)
{
logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
}
else if (fileNotFound != null)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
}
#if !PLATFORM_DOTNET
else if (fileLoad != null)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
}
#endif
else
{
logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
}
}
}
}
}
catch (Exception e)
{
logger.LogWarning("Exception discovering tests: {0}", e.Unwrap());
}
}
void RunTests(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
RunSettings runSettings,
Func <List <AssemblyRunInfo> > getRunInfos)
{
Guard.ArgumentNotNull("runContext", runContext);
Guard.ArgumentNotNull("frameworkHandle", frameworkHandle);
try
{
RemotingUtility.CleanUpRegisteredChannels();
cancelled = false;
var runInfos = getRunInfos();
var parallelizeAssemblies = !runSettings.DisableParallelization && runInfos.All(runInfo => runInfo.Configuration.ParallelizeAssemblyOrDefault);
var reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(GetRunnerReporter(logger, runSettings, runInfos.Select(ari => ari.AssemblyFileName))
.CreateMessageHandler(new VisualStudioRunnerLogger(logger)));
var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(logger, runSettings.InternalDiagnostics);
using (AssemblyHelper.SubscribeResolveForAssembly(typeof(VsTestRunner), MessageSinkAdapter.Wrap(internalDiagnosticsMessageSink)))
{
if (parallelizeAssemblies)
{
runInfos
.Select(runInfo => RunTestsInAssemblyAsync(runContext, frameworkHandle, logger, testPlatformContext, runSettings, reporterMessageHandler, runInfo))
.ToList()
.ForEach(@event => @event.WaitOne());
}
else
{
runInfos
.ForEach(runInfo => RunTestsInAssembly(runContext, frameworkHandle, logger, testPlatformContext, runSettings, reporterMessageHandler, runInfo));
}
}
}
catch (Exception ex)
{
logger.LogError("Catastrophic failure: {0}", ex);
}
}
AssemblyRunner(AppDomainSupport appDomainSupport,
string assemblyFileName,
string configFileName = null,
bool shadowCopy = true,
string shadowCopyFolder = null)
{
controller = new XunitFrontController(appDomainSupport, assemblyFileName, configFileName, shadowCopy, shadowCopyFolder, diagnosticMessageSink: MessageSinkAdapter.Wrap(this));
configuration = ConfigReader.Load(assemblyFileName, configFileName);
}