ManualResetEvent RunTestsInAssemblyAsync(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { var @event = new ManualResetEvent(initialState: false); Action handler = () => { try { RunTestsInAssembly(runContext, frameworkHandle, logger, testPlatformContext, reporterMessageHandler, runInfo); } finally { @event.Set(); } }; #if WINDOWS_UAP var fireAndForget = Windows.System.Threading.ThreadPool.RunAsync(_ => handler()); #else ThreadPool.QueueUserWorkItem(_ => handler()); #endif return(@event); }
void DiscoverTests <TVisitor>(IEnumerable <string> sources, LoggerHelper logger, TestPlatformContext testPlatformContext, Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory, Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null) where TVisitor : IVsDiscoverySink, IDisposable { try { RemotingUtility.CleanUpRegisteredChannels(); using (AssemblyHelper.SubscribeResolve()) { foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources) { var assemblyFileName = GetAssemblyFileName(assemblyFileNameCanBeWithoutAbsolutePath); var configuration = LoadConfiguration(assemblyFileName); var fileName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = configuration.ShadowCopyOrDefault; var diagnosticSink = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault); using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink))) if (!DiscoverTestsInSource(framework, logger, testPlatformContext, visitorFactory, visitComplete, assemblyFileName, shadowCopy, configuration)) { break; } } } } catch (Exception e) { logger.LogWarning("Exception discovering tests: {0}", e.Unwrap()); } }
void ITestExecutor.RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { Guard.ArgumentNotNull("tests", tests); Guard.ArgumentValid("tests", "AppX not supported in this overload", !ContainsAppX(tests.Select(t => t.Source))); var stopwatch = Stopwatch.StartNew(); var logger = new LoggerHelper(frameworkHandle, stopwatch); #if NET452 || NETCOREAPP1_0 RunSettingsHelper.ReadRunSettings(runContext?.RunSettings?.SettingsXml); #endif // In the context of Run Specific tests, commandline runner doesn't require source information or // serialized xunit test case property var testPlatformContext = new TestPlatformContext { RequireSourceInformation = RunSettingsHelper.CollectSourceInformation, RequireXunitTestProperty = RunSettingsHelper.DesignMode }; RunTests( runContext, frameworkHandle, logger, testPlatformContext, () => tests.GroupBy(testCase => testCase.Source) .Select(group => new AssemblyRunInfo { AssemblyFileName = group.Key, Configuration = LoadConfiguration(group.Key), TestCases = group.ToList() }) .ToList() ); }
void ITestDiscoverer.DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { Guard.ArgumentNotNull("sources", sources); Guard.ArgumentNotNull("logger", logger); Guard.ArgumentNotNull("discoverySink", discoverySink); Guard.ArgumentValid("sources", "AppX not supported for discovery", !ContainsAppX(sources)); var stopwatch = Stopwatch.StartNew(); var loggerHelper = new LoggerHelper(logger, stopwatch); #if NET452 || NETCOREAPP1_0 // Reads settings like disabling appdomains, parallel etc. // Do this first before invoking any thing else to ensure correct settings for the run RunSettingsHelper.ReadRunSettings(discoveryContext?.RunSettings?.SettingsXml); #endif var testPlatformContext = new TestPlatformContext { // Discovery from command line (non designmode) never requires source information // since there is no session or command line runner doesn't send back VSTestCase objects // back to adapter. RequireSourceInformation = RunSettingsHelper.DesignMode, // Command line runner could request for Discovery in case of running specific tests. We need // the XunitTestCase serialized in this scenario. RequireXunitTestProperty = true }; DiscoverTests(sources, loggerHelper, testPlatformContext, (source, discoverer, discoveryOptions) => new VsDiscoverySink(source, discoverer, loggerHelper, discoverySink, discoveryOptions, testPlatformContext, () => cancelled) ); }
void ITestExecutor.RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { Guard.ArgumentNotNull("sources", sources); var stopwatch = Stopwatch.StartNew(); var logger = new LoggerHelper(frameworkHandle, stopwatch); #if NET452 || NETCOREAPP1_0 RunSettingsHelper.ReadRunSettings(runContext?.RunSettings?.SettingsXml); if (!ValidateRuntimeFramework()) { return; } #endif // In the context of Run All tests, commandline runner doesn't require source information or // serialized xunit test case property var testPlatformContext = new TestPlatformContext { RequireSourceInformation = RunSettingsHelper.CollectSourceInformation, RequireXunitTestProperty = RunSettingsHelper.DesignMode }; // In this case, we need to go thru the files manually if (ContainsAppX(sources)) { #if WINDOWS_UAP var sourcePath = Windows.ApplicationModel.Package.Current.InstalledLocation.Path; #elif NETCOREAPP1_0 var sourcePath = Directory.GetCurrentDirectory(); #else var sourcePath = Environment.CurrentDirectory; #endif sources = Directory.GetFiles(sourcePath, "*.dll") .Where(file => !platformAssemblies.Contains(Path.GetFileName(file))) .ToList(); ((List <string>)sources).AddRange(Directory.GetFiles(sourcePath, "*.exe") .Where(file => !platformAssemblies.Contains(Path.GetFileName(file)))); } RunTests(runContext, frameworkHandle, logger, testPlatformContext, () => sources.Select(source => { var assemblyFileName = GetAssemblyFileName(source); return(new AssemblyRunInfo { AssemblyFileName = assemblyFileName, Configuration = LoadConfiguration(assemblyFileName), TestCases = null // PERF: delay the discovery until we actually require it in RunTestsInAssembly }); }).ToList()); }
public VsDiscoverySink(string source, ITestFrameworkDiscoverer discoverer, LoggerHelper logger, ITestCaseDiscoverySink discoverySink, ITestFrameworkDiscoveryOptions discoveryOptions, TestPlatformContext testPlatformContext, Func <bool> cancelThunk) { this.source = source; this.discoverer = discoverer; this.logger = logger; this.discoverySink = discoverySink; this.discoveryOptions = discoveryOptions; this.testPlatformContext = testPlatformContext; this.cancelThunk = cancelThunk; discoveryEventSink.TestCaseDiscoveryMessageEvent += HandleTestCaseDiscoveryMessage; discoveryEventSink.DiscoveryCompleteMessageEvent += HandleDiscoveryCompleteMessage; }
void RunTests(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, Func <List <AssemblyRunInfo> > getRunInfos) { Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("frameworkHandle", frameworkHandle); try { RemotingUtility.CleanUpRegisteredChannels(); cancelled = false; var runInfos = getRunInfos(); var parallelizeAssemblies = !RunSettingsHelper.DisableParallelization && runInfos.All(runInfo => runInfo.Configuration.ParallelizeAssemblyOrDefault); var reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(GetRunnerReporter(runInfos.Select(ari => ari.AssemblyFileName)) .CreateMessageHandler(new VisualStudioRunnerLogger(logger))); using (AssemblyHelper.SubscribeResolve()) if (parallelizeAssemblies) { runInfos .Select(runInfo => RunTestsInAssemblyAsync(runContext, frameworkHandle, logger, testPlatformContext, reporterMessageHandler, runInfo)) .ToList() .ForEach(@event => @event.WaitOne()); } else { runInfos .ForEach(runInfo => RunTestsInAssembly(runContext, frameworkHandle, logger, testPlatformContext, reporterMessageHandler, runInfo)); } } catch (Exception ex) { logger.LogError("Catastrophic failure: {0}", ex); } }
public DiscoveredTestCase(string source, ITestFrameworkDiscoverer discoverer, ITestCase testCase, LoggerHelper logger, TestPlatformContext testPlatformContext) { var testMethod = testCase.TestMethod; var testClassName = testMethod.TestClass.Class.Name; var testMethodName = testMethod.Method.Name; TraitNames = testCase.Traits.Keys; VSTestCase = VsDiscoverySink.CreateVsTestCase(source, discoverer, testCase, false, logger, testPlatformContext); Name = $"{testClassName}.{testMethodName}"; TestCase = testCase; UniqueID = testCase.UniqueID; VSTestCase = VsDiscoverySink.CreateVsTestCase(source, discoverer, testCase, false, logger, testPlatformContext, testClassName, testMethodName, UniqueID); TraitNames = VSTestCase.Traits.Select(x => x.Name); }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList() }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); // Apply any filtering var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); // Force unique names if there is more than 1 testcase with the same name foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1)) { foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames) { discoveredTestCaseWithDuplicateName.ForceUniqueName(); } } foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { // We are in Run Specific tests scenario, the `TestCase` objects are available. // Query the `TestCase` objects to find XunitTestCase objects. foreach (var vstestCase in runInfo.TestCases) { var xunitTestCase = Deserialize(logger, controller, vstestCase); if (xunitTestCase != null) { testCasesMap.Add(xunitTestCase.UniqueID, vstestCase); testCases.Add(xunitTestCase); } } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
private bool DiscoverTestsInSource <TVisitor>(XunitFrontController framework, LoggerHelper logger, TestPlatformContext testPlatformContext, Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory, Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete, string assemblyFileName, bool shadowCopy, TestAssemblyConfiguration configuration) where TVisitor : IVsDiscoverySink, IDisposable { var reporterMessageHandler = GetRunnerReporter(new[] { assemblyFileName }).CreateMessageHandler(new VisualStudioRunnerLogger(logger)); var assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName }; var fileName = Path.GetFileNameWithoutExtension(assemblyFileName); try { if (cancelled) { return(false); } if (!IsXunitTestAssembly(assemblyFileName)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName); } } else { var targetFramework = framework.TargetFramework; if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework); } } else { var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions)) { var usingAppDomains = framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, usingAppDomains, shadowCopy, discoveryOptions)); framework.Find(testPlatformContext.RequireSourceInformation, visitor, discoveryOptions); var totalTests = visitor.Finish(); visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor); reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests)); } } } } catch (Exception e) { var ex = e.Unwrap(); if (ex is InvalidOperationException) { logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message); } else if (ex is FileNotFoundException fileNotFound) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName)); } #if !WINDOWS_UAP else if (ex is FileLoadException fileLoad) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName)); } #endif else { logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex); } } return(true); }
public static TestCase CreateVsTestCase(string source, ITestFrameworkDiscoverer discoverer, ITestCase xunitTestCase, bool forceUniqueName, LoggerHelper logger, TestPlatformContext testPlatformContext, string testClassName = null, string testMethodName = null, string uniqueID = null) { try { if (string.IsNullOrEmpty(testClassName)) { testClassName = xunitTestCase.TestMethod.TestClass.Class.Name; } if (string.IsNullOrEmpty(testMethodName)) { testMethodName = xunitTestCase.TestMethod.Method.Name; } if (string.IsNullOrEmpty(uniqueID)) { uniqueID = xunitTestCase.UniqueID; } var fqTestMethodName = $"{testClassName}.{testMethodName}"; var result = new TestCase(fqTestMethodName, uri, source) { DisplayName = Escape(xunitTestCase.DisplayName) }; if (testPlatformContext.RequireXunitTestProperty) { result.SetPropertyValue(VsTestRunner.SerializedTestCaseProperty, discoverer.Serialize(xunitTestCase)); } result.Id = GuidFromString(uri + uniqueID); if (forceUniqueName) { ForceUniqueName(result, uniqueID); } if (addTraitThunk != null) { var traits = xunitTestCase.Traits; foreach (var key in traits.Keys) { foreach (var value in traits[key]) { addTraitThunk(result, key, value); } } } if (testPlatformContext.RequireSourceInformation) { result.CodeFilePath = xunitTestCase.SourceInformation.FileName; result.LineNumber = xunitTestCase.SourceInformation.LineNumber.GetValueOrDefault(); } return(result); } catch (Exception ex) { logger.LogError(xunitTestCase, "Error creating Visual Studio test case for {0}: {1}", xunitTestCase.DisplayName, ex); return(null); } }