Esempio n. 1
0
        protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
        {
            if (cancel)
                return null;

            var assemblyElement = NeedsXml ? new XElement("assembly") : null;

            try
            {
                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories = false;
                assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;

                if (appDomains.HasValue)
                    assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;

                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                if (parallelizeTestCollections.HasValue)
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections);

                var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
                var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
                var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

                using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                using (var discoverySink = new TestDiscoverySink(() => cancel))
                {
                    // Discover & filter the tests
                    reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));

                    controller.Find(false, discoverySink, discoveryOptions);
                    discoverySink.Finished.WaitOne();

                    var testCasesDiscovered = discoverySink.TestCases.Count;
                    var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList();
                    var testCasesToRun = filteredTestCases.Count;

                    reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                    // Run the filtered tests
                    if (testCasesToRun == 0)
                        completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                    else
                    {
                        if (SerializeTestCases)
                            filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();

                        IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
                        if (assemblyElement != null)
                            resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                        if (longRunningSeconds > 0)
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
                        if (FailSkips)
                            resultsSink = new DelegatingFailSkipSink(resultsSink);

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                        controller.RunTests(filteredTestCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));

                        if (resultsSink.ExecutionSummary.Failed != 0)
                            ExitCode = 1;
                    }
                }
            }
            catch (Exception ex)
            {
                var e = ex;

                while (e != null)
                {
                    Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);

                    foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
                        Log.LogError(stackLine);

                    e = e.InnerException;
                }

                ExitCode = -1;
            }

            return assemblyElement;
        }
Esempio n. 2
0
        XElement ExecuteAssembly(object consoleLock,
                                 XunitProjectAssembly assembly,
                                 bool needsXml,
                                 bool?parallelizeTestCollections,
                                 int?maxThreadCount,
                                 bool diagnosticMessages,
                                 bool noColor,
                                 bool failSkips,
                                 bool stopOnFail,
                                 XunitFilters filters,
                                 bool internalDiagnosticMessages)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                if (!ValidateFileExists(consoleLock, assembly.ConfigFilename))
                {
                    return(null);
                }

                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories        = false;
                assembly.Configuration.DiagnosticMessages         |= diagnosticMessages;
                assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;

                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                executionOptions.SetStopOnTestFail(stopOnFail);
                if (maxThreadCount.HasValue)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
                }

                var assemblyDisplayName            = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageSink          = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, diagnosticMessages, noColor);
                var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, internalDiagnosticMessages, noColor);
                var longRunningSeconds             = assembly.Configuration.LongRunningTestSecondsOrDefault;

                using var testFramework = new XunitTestFramework(diagnosticMessageSink, assembly.ConfigFilename);
                var entryAssembly = Assembly.GetEntryAssembly();
                var assemblyInfo  = new ReflectionAssemblyInfo(entryAssembly);
                var discoverySink = new TestDiscoverySink(() => cancel);

                using (var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo))
                {
                    // Discover & filter the tests
                    reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, appDomain: false, shadowCopy: false, discoveryOptions));

                    testDiscoverer.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
                    discoverySink.Finished.WaitOne();
                }

                var testCasesDiscovered = discoverySink.TestCases.Count;
                var filteredTestCases   = discoverySink.TestCases.Where(filters.Filter).ToList();
                var testCasesToRun      = filteredTestCases.Count;

                reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                // Run the filtered tests
                if (testCasesToRun == 0)
                {
                    executionSummary = new ExecutionSummary();
                }
                else
                {
                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
                    if (assemblyElement != null)
                    {
                        resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                    }
                    if (longRunningSeconds > 0)
                    {
                        resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink));
                    }
                    if (failSkips)
                    {
                        resultsSink = new DelegatingFailSkipSink(resultsSink);
                    }

                    using var executor = testFramework.GetExecutor(entryAssembly.GetName());
                    executor.RunTests(filteredTestCases, resultsSink, executionOptions);
                    resultsSink.Finished.WaitOne();

                    executionSummary = resultsSink.ExecutionSummary;

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
                    {
                        Console.WriteLine("Canceling due to test failure...");
                        cancel = true;
                    }
                }
            }
            catch (Exception ex)
            {
                failed = true;

                var e = ex;
                while (e != null)
                {
                    Console.WriteLine($"{e.GetType().FullName}: {e.Message}");

                    if (internalDiagnosticMessages)
                    {
                        Console.WriteLine(e.StackTrace);
                    }

                    e = e.InnerException;
                }
            }

            return(assemblyElement);
        }
Esempio n. 3
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                TestPlatformContext testPlatformContext,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assembly = new XunitProjectAssembly {
                AssemblyFilename = runInfo.AssemblyFileName
            };
            var assemblyFileName    = runInfo.AssemblyFileName;
            var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
            var configuration       = runInfo.Configuration;
            var shadowCopy          = configuration.ShadowCopyOrDefault;

            var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
            var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

            if (RunSettingsHelper.DisableAppDomain)
            {
                appDomain = AppDomainSupport.Denied;
            }

            try
            {
#if WINDOWS_UAP
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink        = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                {
                    var testCasesMap = new Dictionary <string, TestCase>();
                    var testCases    = new List <ITestCase>();
                    if (runInfo.TestCases == null || !runInfo.TestCases.Any())
                    {
                        // Discover tests
                        AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
                        DiscoverTestsInSource(controller, logger, testPlatformContext,
                                              (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
                                              (source, discoverer, discoveryOptions, visitor) =>
                        {
                            if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
                            {
                                foreach (var testCase in visitor.TestCases)
                                {
                                    logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
                                               testCase.DisplayName, testCase.UniqueID);
                                }
                            }

                            assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
                            {
                                AssemblyFileName    = source,
                                DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList()
                            };
                        },
                                              assemblyFileName,
                                              shadowCopy,
                                              configuration
                                              );

                        if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
                        {
                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
                            }

                            return;
                        }

                        // Filter tests
                        var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));

                        // Apply any filtering
                        var filter            = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
                        var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();

                        // Force unique names if there is more than 1 testcase with the same name
                        foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1))
                        {
                            foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames)
                            {
                                discoveredTestCaseWithDuplicateName.ForceUniqueName();
                            }
                        }

                        foreach (var filteredTestCase in filteredTestCases)
                        {
                            var uniqueID = filteredTestCase.UniqueID;
                            if (testCasesMap.ContainsKey(uniqueID))
                            {
                                logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
                            }
                            else
                            {
                                testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
                                testCases.Add(filteredTestCase.TestCase);
                            }
                        }
                    }
                    else
                    {
                        // We are in Run Specific tests scenario, the `TestCase` objects are available.
                        // Query the `TestCase` objects to find XunitTestCase objects.
                        foreach (var vstestCase in runInfo.TestCases)
                        {
                            var xunitTestCase = Deserialize(logger, controller, vstestCase);
                            if (xunitTestCase != null)
                            {
                                testCasesMap.Add(xunitTestCase.UniqueID, vstestCase);
                                testCases.Add(xunitTestCase);
                            }
                        }
                    }

                    // Execute tests
                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (RunSettingsHelper.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(testCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }
Esempio n. 4
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                TestPlatformContext testPlatformContext,
                                RunSettings runSettings,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assemblyDisplayName = "(unknown assembly)";

            try
            {
                var assembly = new XunitProjectAssembly {
                    AssemblyFilename = runInfo.AssemblyFileName
                };
                var assemblyFileName = runInfo.AssemblyFileName;
                assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
                var configuration = runInfo.Configuration;
                var shadowCopy    = configuration.ShadowCopyOrDefault;

                var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

                if (runSettings.DisableAppDomain)
                {
                    appDomain = AppDomainSupport.Denied;
                }

#if WINDOWS_UAP
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink        = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                {
                    var testCasesMap = new Dictionary <string, TestCase>();
                    var testCases    = new List <ITestCase>();
                    if (runInfo.TestCases == null || !runInfo.TestCases.Any())
                    {
                        // Discover tests
                        AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
                        DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings,
                                              (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
                                              (source, discoverer, discoveryOptions, visitor) =>
                        {
                            if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
                            {
                                foreach (var testCase in visitor.TestCases)
                                {
                                    logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
                                               testCase.DisplayName, testCase.UniqueID);
                                }
                            }

                            assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
                            {
                                AssemblyFileName    = source,
                                DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
                            };
                        },
                                              assemblyFileName,
                                              shadowCopy,
                                              configuration
                                              );

                        if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
                        {
                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
                            }

                            return;
                        }

                        // Filter tests
                        var traitNames        = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
                        var filter            = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
                        var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();

                        foreach (var filteredTestCase in filteredTestCases)
                        {
                            var uniqueID = filteredTestCase.UniqueID;
                            if (testCasesMap.ContainsKey(uniqueID))
                            {
                                logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
                            }
                            else
                            {
                                testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
                                testCases.Add(filteredTestCase.TestCase);
                            }
                        }
                    }
                    else
                    {
                        try
                        {
                            var serializations = runInfo.TestCases
                                                 .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
                                                 .ToList();

                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}",
                                                     serializations.Count,
                                                     Environment.NewLine,
                                                     string.Join(Environment.NewLine, serializations.Select(x => $"  {x}")));
                            }

                            var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);

                            if (deserializedTestCasesByUniqueId == null)
                            {
                                logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
                            }
                            else
                            {
                                for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
                                {
                                    try
                                    {
                                        var kvp        = deserializedTestCasesByUniqueId[idx];
                                        var vsTestCase = runInfo.TestCases[idx];

                                        if (kvp.Value == null)
                                        {
                                            logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
                                        }
                                        else
                                        {
                                            testCasesMap.Add(kvp.Key, vsTestCase);
                                            testCases.Add(kvp.Value);
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
                        }
                    }

                    // Execute tests
                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (runSettings.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(testCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }
Esempio n. 5
0
        protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = NeedsXml ? new XElement("assembly") : null;

            try
            {
                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories        = false;
                assembly.Configuration.DiagnosticMessages         |= DiagnosticMessages;
                assembly.Configuration.InternalDiagnosticMessages |= InternalDiagnosticMessages;

                if (appDomains.HasValue)
                {
                    assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.IfAvailable : AppDomainSupport.Denied;
                }

                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections);
                }
                if (stopOnFail.HasValue)
                {
                    executionOptions.SetStopOnTestFail(stopOnFail);
                }

                var assemblyDisplayName   = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
                var appDomainSupport      = assembly.Configuration.AppDomainOrDefault;
                var shadowCopy            = assembly.Configuration.ShadowCopyOrDefault;
                var longRunningSeconds    = assembly.Configuration.LongRunningTestSecondsOrDefault;

                using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                    using (var discoverySink = new TestDiscoverySink(() => cancel))
                    {
                        // Discover & filter the tests
                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));

                        controller.Find(false, discoverySink, discoveryOptions);
                        discoverySink.Finished.WaitOne();

                        var testCasesDiscovered = discoverySink.TestCases.Count;
                        var filteredTestCases   = discoverySink.TestCases.Where(Filters.Filter).ToList();
                        var testCasesToRun      = filteredTestCases.Count;

                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                        // Run the filtered tests
                        if (testCasesToRun == 0)
                        {
                            completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                        }
                        else
                        {
                            if (SerializeTestCases)
                            {
                                filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
                            }

                            IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
                            if (assemblyElement != null)
                            {
                                resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                            }
                            if (longRunningSeconds > 0)
                            {
                                resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
                            }
                            if (FailSkips)
                            {
                                resultsSink = new DelegatingFailSkipSink(resultsSink);
                            }

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                            controller.RunTests(filteredTestCases, resultsSink, executionOptions);
                            resultsSink.Finished.WaitOne();

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));

                            if (resultsSink.ExecutionSummary.Failed != 0)
                            {
                                ExitCode = 1;
                                if (stopOnFail == true)
                                {
                                    Log.LogMessage(MessageImportance.High, "Canceling due to test failure...");
                                    Cancel();
                                }
                            }
                        }
                    }
            }
            catch (Exception ex)
            {
                var e = ex;

                while (e != null)
                {
                    Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);

                    foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
                    {
                        Log.LogError(stackLine);
                    }

                    e = e.InnerException;
                }

                ExitCode = -1;
            }

            return(assemblyElement);
        }
Esempio n. 6
0
        XElement ExecuteAssembly(object consoleLock,
                                 XunitProjectAssembly assembly,
                                 bool serialize,
                                 bool needsXml,
                                 bool?parallelizeTestCollections,
                                 int?maxThreadCount,
                                 bool diagnosticMessages,
                                 bool noColor,
                                 bool noAppDomain,
                                 bool failSkips,
                                 bool stopOnFail,
                                 XunitFilters filters,
                                 bool internalDiagnosticMessages)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
                {
                    return(null);
                }

                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories        = false;
                assembly.Configuration.DiagnosticMessages         |= diagnosticMessages;
                assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;

                if (noAppDomain)
                {
                    assembly.Configuration.AppDomain = AppDomainSupport.Denied;
                }

                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                executionOptions.SetStopOnTestFail(stopOnFail);
                if (maxThreadCount.HasValue)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
                }

                var assemblyDisplayName            = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageSink          = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
                var internalDiagnosticsMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor, ConsoleColor.DarkGray);
                var appDomainSupport   = assembly.Configuration.AppDomainOrDefault;
                var shadowCopy         = assembly.Configuration.ShadowCopyOrDefault;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

#if NETCOREAPP1_0 || NETCOREAPP2_0
                using (new NetCoreAssemblyDependencyResolver(assembly.AssemblyFilename, internalDiagnosticsMessageSink))
#endif
                using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                    using (var discoverySink = new TestDiscoverySink(() => cancel))
                    {
                        // Discover & filter the tests
                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));

                        controller.Find(false, discoverySink, discoveryOptions);
                        discoverySink.Finished.WaitOne();

                        var testCasesDiscovered = discoverySink.TestCases.Count;
                        var filteredTestCases   = discoverySink.TestCases.Where(filters.Filter).ToList();
                        var testCasesToRun      = filteredTestCases.Count;

                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                        // Run the filtered tests
                        if (testCasesToRun == 0)
                        {
                            completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                        }
                        else
                        {
                            if (serialize)
                            {
                                filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
                            }

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                            IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
                            if (assemblyElement != null)
                            {
                                resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                            }
                            if (longRunningSeconds > 0)
                            {
                                resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
                            }
                            if (failSkips)
                            {
                                resultsSink = new DelegatingFailSkipSink(resultsSink);
                            }

                            controller.RunTests(filteredTestCases, resultsSink, executionOptions);
                            resultsSink.Finished.WaitOne();

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                            if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
                            {
                                Console.WriteLine("Canceling due to test failure...");
                                cancel = true;
                            }
                        }
                    }
            }
            catch (Exception ex)
            {
                failed = true;

                var e = ex;
                while (e != null)
                {
                    Console.WriteLine($"{e.GetType().FullName}: {e.Message}");

                    if (internalDiagnosticMessages)
                    {
                        Console.WriteLine(e.StackTrace);
                    }

                    e = e.InnerException;
                }
            }

            return(assemblyElement);
        }
Esempio n. 7
0
        static XElement ExecuteAssembly(object consoleLock,
                                        XunitProjectAssembly assembly,
                                        bool serialize,
                                        bool needsXml,
                                        bool? parallelizeTestCollections,
                                        int? maxThreadCount,
                                        bool diagnosticMessages,
                                        bool noColor,
                                        bool noAppDomain,
                                        bool failSkips,
                                        XunitFilters filters)
        {
            if (cancel)
                return null;

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
                    return null;

                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories = false;
                assembly.Configuration.DiagnosticMessages |= diagnosticMessages;

                if (noAppDomain)
                    assembly.Configuration.AppDomain = AppDomainSupport.Denied;

                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                if (maxThreadCount.HasValue)
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                if (parallelizeTestCollections.HasValue)
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());

                var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
                var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
                var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

                using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                using (var discoverySink = new TestDiscoverySink(() => cancel))
                {
                    // Discover & filter the tests
                    reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));

                    controller.Find(false, discoverySink, discoveryOptions);
                    discoverySink.Finished.WaitOne();

                    var testCasesDiscovered = discoverySink.TestCases.Count;
                    var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
                    var testCasesToRun = filteredTestCases.Count;

                    reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                    // Run the filtered tests
                    if (testCasesToRun == 0)
                        completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                    else
                    {
                        if (serialize)
                            filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                        IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
                        if (assemblyElement != null)
                            resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                        if (longRunningSeconds > 0)
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
                        if (failSkips)
                            resultsSink = new DelegatingFailSkipSink(resultsSink);

                        controller.RunTests(filteredTestCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                failed = true;

                var e = ex;
                while (e != null)
                {
                    Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
                    e = e.InnerException;
                }
            }

            return assemblyElement;
        }
Esempio n. 8
0
        async ValueTask <XElement?> ExecuteAssembly(
            XunitProjectAssembly assembly,
            bool needsXml,
            _IMessageSink reporterMessageHandler)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);

                // The normal default is true here, but we want it to be false for us by default
                if (!assembly.Configuration.PreEnumerateTheories.HasValue)
                {
                    discoveryOptions.SetPreEnumerateTheories(false);
                }

                var assemblyDisplayName            = assembly.AssemblyDisplayName;
                var noColor                        = assembly.Project.Configuration.NoColorOrDefault;
                var diagnosticMessageSink          = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
                var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
                var longRunningSeconds             = assembly.Configuration.LongRunningTestSecondsOrDefault;

                var assemblyInfo = new ReflectionAssemblyInfo(testAssembly);

                await using var disposalTracker = new DisposalTracker();
                var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo);
                disposalTracker.Add(testFramework);

                var discoverySink = new TestDiscoverySink(() => cancel);

                // Discover & filter the tests
                var testDiscoverer    = testFramework.GetDiscoverer(assemblyInfo);
                var discoveryStarting = new TestAssemblyDiscoveryStarting
                {
                    AppDomain        = AppDomainOption.NotAvailable,
                    Assembly         = assembly,
                    DiscoveryOptions = discoveryOptions,
                    ShadowCopy       = false
                };
                reporterMessageHandler.OnMessage(discoveryStarting);

                testDiscoverer.Find(discoverySink, discoveryOptions);
                discoverySink.Finished.WaitOne();

                var testCasesDiscovered = discoverySink.TestCases.Count;
                var filteredTestCases   = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList();
                var testCasesToRun      = filteredTestCases.Count;

                var discoveryFinished = new TestAssemblyDiscoveryFinished
                {
                    Assembly            = assembly,
                    DiscoveryOptions    = discoveryOptions,
                    TestCasesDiscovered = testCasesDiscovered,
                    TestCasesToRun      = testCasesToRun
                };
                reporterMessageHandler.OnMessage(discoveryFinished);

                // Run the filtered tests
                if (testCasesToRun == 0)
                {
                    testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary());
                }
                else
                {
                    var executionStarting = new TestAssemblyExecutionStarting
                    {
                        Assembly         = assembly,
                        ExecutionOptions = executionOptions
                    };
                    reporterMessageHandler.OnMessage(executionStarting);

                    IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
                    if (assemblyElement != null)
                    {
                        resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                    }
                    if (longRunningSeconds > 0)
                    {
                        resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
                    }
                    if (assembly.Configuration.FailSkipsOrDefault)
                    {
                        resultsSink = new DelegatingFailSkipSink(resultsSink);
                    }

                    using (resultsSink)
                    {
                        var executor = testFramework.GetExecutor(assemblyInfo);

                        executor.RunTests(filteredTestCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary);

                        var executionFinished = new TestAssemblyExecutionFinished
                        {
                            Assembly         = assembly,
                            ExecutionOptions = executionOptions,
                            ExecutionSummary = resultsSink.ExecutionSummary
                        };
                        reporterMessageHandler.OnMessage(executionFinished);

                        if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
                        {
                            Console.WriteLine("Canceling due to test failure...");
                            cancel = true;
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                failed = true;

                var e = ex;
                while (e != null)
                {
                    Console.WriteLine($"{e.GetType().FullName}: {e.Message}");

                    if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
                    {
                        Console.WriteLine(e.StackTrace);
                    }

                    e = e.InnerException;
                }
            }

            return(assemblyElement);
        }
Esempio n. 9
0
        private void ExecuteTests(
            [NotNull] XunitProjectAssembly assembly,
            [NotNull] TestRunOptions options,
            [NotNull] XunitFilters filters,
            [NotNull] XunitFrontController controller,
            [NotNull] TestDiscoverySink discoverySink,
            [NotNull] ITestFrameworkExecutionOptions executionOptions,
            [CanBeNull] XElement assemblyElement,
            [NotNull] DiagnosticMessageSink diagnosticMessageSink)
        {
            var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
            var shadowCopy       = assembly.Configuration.ShadowCopyOrDefault;
            var serialize        = options.Serialize;
            var failSkips        = options.FailSkips;
            var stopOnFail       = options.StopOnFail;

            var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

            var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);

            // Discover & filter the tests
            _reporterMessageHandler.OnMessage(
                new TestAssemblyDiscoveryStarting(
                    assembly,
                    controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
                    shadowCopy,
                    discoveryOptions));

            controller.Find(false, discoverySink, discoveryOptions);
            discoverySink.Finished.WaitOne();

            var testCasesDiscovered = discoverySink.TestCases.Count;
            var filteredTestCases   = discoverySink.TestCases.Where(filters.Filter).ToList();
            var testCasesToRun      = filteredTestCases.Count;

            _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

            // Run the filtered tests
            if (testCasesToRun == 0)
            {
                _completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary());
            }
            else
            {
                if (serialize)
                {
                    filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
                }

                _reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                IExecutionSink resultsSink = new DelegatingExecutionSummarySink(
                    _reporterMessageHandler,
                    () => _cancel,
                    (path, summary) => _completionMessages.TryAdd(path, summary));
                if (assemblyElement != null)
                {
                    resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                }

                if (longRunningSeconds > 0)
                {
                    resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
                }

                if (failSkips)
                {
                    resultsSink = new DelegatingFailSkipSink(resultsSink);
                }

                controller.RunTests(filteredTestCases, resultsSink, executionOptions);
                resultsSink.Finished.WaitOne();

                _reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));

                if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0)
                {
                    return;
                }

                Console.WriteLine("Canceling due to test failure...");
                _cancel = true;
            }
        }
Esempio n. 10
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assembly = new XunitProjectAssembly {
                AssemblyFilename = runInfo.AssemblyFileName
            };
            var assemblyFileName    = runInfo.AssemblyFileName;
            var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
            var shadowCopy          = assembly.Configuration.ShadowCopyOrDefault;

            var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
            var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

            if (RunSettingsHelper.DisableAppDomain)
            {
                appDomain = AppDomainSupport.Denied;
            }

            try
            {
#if PLATFORM_DOTNET
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
                {
                    var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) })
                                         .Where(tc => tc.xunit != null)
                                         .ToDictionary(tc => tc.xunit, tc => tc.vs);

                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (RunSettingsHelper.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));



                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(xunitTestCases.Keys.ToList(), resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }