Example #1
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                TestPlatformContext testPlatformContext,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assemblyDisplayName = "(unknown assembly)";

            try
            {
                var assembly = new XunitProjectAssembly {
                    AssemblyFilename = runInfo.AssemblyFileName
                };
                var assemblyFileName = runInfo.AssemblyFileName;
                assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
                var configuration = runInfo.Configuration;
                var shadowCopy    = configuration.ShadowCopyOrDefault;

                var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

                if (RunSettingsHelper.DisableAppDomain)
                {
                    appDomain = AppDomainSupport.Denied;
                }

#if WINDOWS_UAP
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink        = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                {
                    var testCasesMap = new Dictionary <string, TestCase>();
                    var testCases    = new List <ITestCase>();
                    if (runInfo.TestCases == null || !runInfo.TestCases.Any())
                    {
                        // Discover tests
                        AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
                        DiscoverTestsInSource(controller, logger, testPlatformContext,
                                              (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
                                              (source, discoverer, discoveryOptions, visitor) =>
                        {
                            if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
                            {
                                foreach (var testCase in visitor.TestCases)
                                {
                                    logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
                                               testCase.DisplayName, testCase.UniqueID);
                                }
                            }

                            assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
                            {
                                AssemblyFileName    = source,
                                DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
                            };
                        },
                                              assemblyFileName,
                                              shadowCopy,
                                              configuration
                                              );

                        if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
                        {
                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
                            }

                            return;
                        }

                        // Filter tests
                        var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));

                        // Apply any filtering
                        var filter            = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
                        var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();

                        foreach (var filteredTestCase in filteredTestCases)
                        {
                            var uniqueID = filteredTestCase.UniqueID;
                            if (testCasesMap.ContainsKey(uniqueID))
                            {
                                logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
                            }
                            else
                            {
                                testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
                                testCases.Add(filteredTestCase.TestCase);
                            }
                        }
                    }
                    else
                    {
                        // We are in Run Specific tests scenario, the `TestCase` objects are available.
                        // Query the `TestCase` objects to find XunitTestCase objects.
                        foreach (var vstestCase in runInfo.TestCases)
                        {
                            var xunitTestCase = Deserialize(logger, controller, vstestCase);
                            if (xunitTestCase != null)
                            {
                                testCasesMap.Add(xunitTestCase.UniqueID, vstestCase);
                                testCases.Add(xunitTestCase);
                            }
                        }
                    }

                    // Execute tests
                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (RunSettingsHelper.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(testCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }
Example #2
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                TestPlatformContext testPlatformContext,
                                RunSettings runSettings,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assemblyDisplayName = "(unknown assembly)";

            try
            {
                var assembly = new XunitProjectAssembly {
                    AssemblyFilename = runInfo.AssemblyFileName
                };
                var assemblyFileName = runInfo.AssemblyFileName;
                assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
                var configuration = runInfo.Configuration;
                var shadowCopy    = configuration.ShadowCopyOrDefault;

                var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

                if (runSettings.DisableAppDomain)
                {
                    appDomain = AppDomainSupport.Denied;
                }

#if WINDOWS_UAP
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink        = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                {
                    var testCasesMap = new Dictionary <string, TestCase>();
                    var testCases    = new List <ITestCase>();
                    if (runInfo.TestCases == null || !runInfo.TestCases.Any())
                    {
                        // Discover tests
                        AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
                        DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings,
                                              (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
                                              (source, discoverer, discoveryOptions, visitor) =>
                        {
                            if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
                            {
                                foreach (var testCase in visitor.TestCases)
                                {
                                    logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
                                               testCase.DisplayName, testCase.UniqueID);
                                }
                            }

                            assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
                            {
                                AssemblyFileName    = source,
                                DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
                            };
                        },
                                              assemblyFileName,
                                              shadowCopy,
                                              configuration
                                              );

                        if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
                        {
                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
                            }

                            return;
                        }

                        // Filter tests
                        var traitNames        = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
                        var filter            = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
                        var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();

                        foreach (var filteredTestCase in filteredTestCases)
                        {
                            var uniqueID = filteredTestCase.UniqueID;
                            if (testCasesMap.ContainsKey(uniqueID))
                            {
                                logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
                            }
                            else
                            {
                                testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
                                testCases.Add(filteredTestCase.TestCase);
                            }
                        }
                    }
                    else
                    {
                        try
                        {
                            var serializations = runInfo.TestCases
                                                 .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
                                                 .ToList();

                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}",
                                                     serializations.Count,
                                                     Environment.NewLine,
                                                     string.Join(Environment.NewLine, serializations.Select(x => $"  {x}")));
                            }

                            var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);

                            if (deserializedTestCasesByUniqueId == null)
                            {
                                logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
                            }
                            else
                            {
                                for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
                                {
                                    try
                                    {
                                        var kvp        = deserializedTestCasesByUniqueId[idx];
                                        var vsTestCase = runInfo.TestCases[idx];

                                        if (kvp.Value == null)
                                        {
                                            logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
                                        }
                                        else
                                        {
                                            testCasesMap.Add(kvp.Key, vsTestCase);
                                            testCases.Add(kvp.Value);
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
                        }
                    }

                    // Execute tests
                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (runSettings.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(testCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }