Beispiel #1
0
        void ITestExecutor.RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            Guard.ArgumentNotNull("tests", tests);
            Guard.ArgumentValid("tests", "AppX not supported in this overload", !ContainsAppX(tests.Select(t => t.Source)));

            var stopwatch = Stopwatch.StartNew();
            var logger    = new LoggerHelper(frameworkHandle, stopwatch);

#if NET452 || NETCOREAPP1_0
            RunSettingsHelper.ReadRunSettings(runContext?.RunSettings?.SettingsXml);
#endif

            // In the context of Run Specific tests, commandline runner doesn't require source information or
            // serialized xunit test case property
            var testPlatformContext = new TestPlatformContext
            {
                RequireSourceInformation = RunSettingsHelper.CollectSourceInformation,
                RequireXunitTestProperty = RunSettingsHelper.DesignMode
            };

            RunTests(
                runContext, frameworkHandle, logger, testPlatformContext,
                () => tests.GroupBy(testCase => testCase.Source)
                .Select(group => new AssemblyRunInfo {
                AssemblyFileName = group.Key, Configuration = LoadConfiguration(group.Key), TestCases = group.ToList()
            })
                .ToList()
                );
        }
Beispiel #2
0
        ManualResetEvent RunTestsInAssemblyAsync(IRunContext runContext,
                                                 IFrameworkHandle frameworkHandle,
                                                 LoggerHelper logger,
                                                 TestPlatformContext testPlatformContext,
                                                 IMessageSinkWithTypes reporterMessageHandler,
                                                 AssemblyRunInfo runInfo)
        {
            var    @event  = new ManualResetEvent(initialState: false);
            Action handler = () =>
            {
                try
                {
                    RunTestsInAssembly(runContext, frameworkHandle, logger, testPlatformContext, reporterMessageHandler, runInfo);
                }
                finally
                {
                    @event.Set();
                }
            };

#if WINDOWS_UAP
            var fireAndForget = Windows.System.Threading.ThreadPool.RunAsync(_ => handler());
#else
            ThreadPool.QueueUserWorkItem(_ => handler());
#endif

            return(@event);
        }
Beispiel #3
0
        void ITestDiscoverer.DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
        {
            Guard.ArgumentNotNull("sources", sources);
            Guard.ArgumentNotNull("logger", logger);
            Guard.ArgumentNotNull("discoverySink", discoverySink);
            Guard.ArgumentValid("sources", "AppX not supported for discovery", !ContainsAppX(sources));

            var stopwatch    = Stopwatch.StartNew();
            var loggerHelper = new LoggerHelper(logger, stopwatch);

            RunSettingsHelper.ReadRunSettings(discoveryContext?.RunSettings?.SettingsXml);
            if (!ValidateRuntimeFramework())
            {
                return;
            }

            var testPlatformContext = new TestPlatformContext
            {
                // Discovery from command line (non designmode) never requires source information
                // since there is no session or command line runner doesn't send back VSTestCase objects
                // back to adapter.
                RequireSourceInformation = RunSettingsHelper.CollectSourceInformation,

                // Command line runner could request for Discovery in case of running specific tests. We need
                // the XunitTestCase serialized in this scenario.
                RequireXunitTestProperty = true
            };

            DiscoverTests(sources,
                          loggerHelper,
                          testPlatformContext,
                          (source, discoverer, discoveryOptions) => new VsDiscoverySink(source, discoverer, loggerHelper, discoverySink, discoveryOptions, testPlatformContext, () => cancelled)
                          );
        }
Beispiel #4
0
        void DiscoverTests <TVisitor>(IEnumerable <string> sources,
                                      LoggerHelper logger,
                                      TestPlatformContext testPlatformContext,
                                      Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
                                      Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null)
            where TVisitor : IVsDiscoverySink, IDisposable
        {
            try
            {
                RemotingUtility.CleanUpRegisteredChannels();

                using (AssemblyHelper.SubscribeResolve())
                {
                    foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources)
                    {
                        var assemblyFileName = GetAssemblyFileName(assemblyFileNameCanBeWithoutAbsolutePath);
                        var configuration    = LoadConfiguration(assemblyFileName);
                        var fileName         = Path.GetFileNameWithoutExtension(assemblyFileName);
                        var shadowCopy       = configuration.ShadowCopyOrDefault;
                        var diagnosticSink   = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault);

                        using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
                            if (!DiscoverTestsInSource(framework, logger, testPlatformContext, visitorFactory, visitComplete, assemblyFileName, shadowCopy, configuration))
                            {
                                break;
                            }
                    }
                }
            }
            catch (Exception e)
            {
                logger.LogWarning("Exception discovering tests: {0}", e.Unwrap());
            }
        }
Beispiel #5
0
        void ITestExecutor.RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            Guard.ArgumentNotNull("sources", sources);

            var stopwatch = Stopwatch.StartNew();
            var logger    = new LoggerHelper(frameworkHandle, stopwatch);

            PrintHeader(logger);

            var runSettings = RunSettings.Parse(runContext?.RunSettings?.SettingsXml);

            if (!runSettings.IsMatchingTargetFramework())
            {
                return;
            }

            // In the context of Run All tests, commandline runner doesn't require source information or
            // serialized xunit test case property
            var testPlatformContext = new TestPlatformContext
            {
                RequireSourceInformation = runSettings.CollectSourceInformation,
                RequireXunitTestProperty = runSettings.DesignMode
            };

            // In this case, we need to go thru the files manually
            if (ContainsAppX(sources))
            {
#if WINDOWS_UAP
                var sourcePath = Windows.ApplicationModel.Package.Current.InstalledLocation.Path;
#elif NETCOREAPP1_0
                var sourcePath = Directory.GetCurrentDirectory();
#else
                var sourcePath = Environment.CurrentDirectory;
#endif
                sources = Directory.GetFiles(sourcePath, "*.dll")
                          .Where(file => !platformAssemblies.Contains(Path.GetFileName(file)))
                          .ToList();

                ((List <string>)sources).AddRange(Directory.GetFiles(sourcePath, "*.exe")
                                                  .Where(file => !platformAssemblies.Contains(Path.GetFileName(file))));
            }

            RunTests(
                runContext, frameworkHandle, logger, testPlatformContext, runSettings,
                () => sources.Select(source =>
            {
                var assemblyFileName = GetAssemblyFileName(source);
                return(new AssemblyRunInfo
                {
                    AssemblyFileName = assemblyFileName,
                    Configuration = LoadConfiguration(assemblyFileName),
                    TestCases = null     // PERF: delay the discovery until we actually require it in RunTestsInAssembly
                });
            }).ToList()
                );
        }
Beispiel #6
0
        public static TestCase CreateVsTestCase(string source,
                                                TestCaseDescriptor descriptor,
                                                LoggerHelper logger,
                                                TestPlatformContext testPlatformContext)
        {
            try
            {
                var fqTestMethodName = $"{descriptor.ClassName}.{descriptor.MethodName} ({descriptor.UniqueID})";
                var result           = new TestCase(fqTestMethodName, uri, source)
                {
                    DisplayName = Escape(descriptor.DisplayName)
                };

                if (testPlatformContext.RequireXunitTestProperty)
                {
                    result.SetPropertyValue(VsTestRunner.SerializedTestCaseProperty, descriptor.Serialization);
                }

                result.Id = GuidFromString(uri + descriptor.UniqueID);

                if (addTraitThunk != null)
                {
                    var traits = descriptor.Traits;

                    foreach (var key in traits.Keys)
                    {
                        foreach (var value in traits[key])
                        {
                            addTraitThunk(result, key, value);
                        }
                    }
                }

                if (testPlatformContext.RequireSourceInformation)
                {
                    result.CodeFilePath = descriptor.SourceFileName;
                    result.LineNumber   = descriptor.SourceLineNumber.GetValueOrDefault();
                }

                return(result);
            }
            catch (Exception ex)
            {
                logger.LogErrorWithSource(source, "Error creating Visual Studio test case for {0}: {1}", descriptor.DisplayName, ex);
                return(null);
            }
        }
Beispiel #7
0
        void RunTests(IRunContext runContext,
                      IFrameworkHandle frameworkHandle,
                      LoggerHelper logger,
                      TestPlatformContext testPlatformContext,
                      Func <List <AssemblyRunInfo> > getRunInfos)
        {
            Guard.ArgumentNotNull("runContext", runContext);
            Guard.ArgumentNotNull("frameworkHandle", frameworkHandle);

            try
            {
                RemotingUtility.CleanUpRegisteredChannels();

                cancelled = false;

                var runInfos = getRunInfos();
                var parallelizeAssemblies  = !RunSettingsHelper.DisableParallelization && runInfos.All(runInfo => runInfo.Configuration.ParallelizeAssemblyOrDefault);
                var reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(GetRunnerReporter(runInfos.Select(ari => ari.AssemblyFileName))
                                                                              .CreateMessageHandler(new VisualStudioRunnerLogger(logger)));

#if NET452
                using (AssemblyHelper.SubscribeResolve())
#endif
                {
                    if (parallelizeAssemblies)
                    {
                        runInfos
                        .Select(runInfo => RunTestsInAssemblyAsync(runContext, frameworkHandle, logger, testPlatformContext, reporterMessageHandler, runInfo))
                        .ToList()
                        .ForEach(@event => @event.WaitOne());
                    }
                    else
                    {
                        runInfos
                        .ForEach(runInfo => RunTestsInAssembly(runContext, frameworkHandle, logger, testPlatformContext, reporterMessageHandler, runInfo));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("Catastrophic failure: {0}", ex);
            }
        }
Beispiel #8
0
        public VsDiscoverySink(string source,
                               ITestFrameworkDiscoverer discoverer,
                               LoggerHelper logger,
                               ITestCaseDiscoverySink discoverySink,
                               ITestFrameworkDiscoveryOptions discoveryOptions,
                               TestPlatformContext testPlatformContext,
                               Func <bool> cancelThunk)
        {
            this.source              = source;
            this.discoverer          = discoverer;
            this.logger              = logger;
            this.discoverySink       = discoverySink;
            this.discoveryOptions    = discoveryOptions;
            this.testPlatformContext = testPlatformContext;
            this.cancelThunk         = cancelThunk;

            descriptorProvider = (discoverer as ITestCaseDescriptorProvider) ?? new DefaultTestCaseDescriptorProvider(discoverer);

            discoveryEventSink.TestCaseDiscoveryMessageEvent += HandleTestCaseDiscoveryMessage;
            discoveryEventSink.DiscoveryCompleteMessageEvent += HandleDiscoveryCompleteMessage;
        }
Beispiel #9
0
 public DiscoveredTestCase(string source, TestCaseDescriptor descriptor, ITestCase testCase, LoggerHelper logger, TestPlatformContext testPlatformContext)
 {
     Name       = $"{descriptor.ClassName}.{descriptor.MethodName} ({descriptor.UniqueID})";
     TestCase   = testCase;
     UniqueID   = descriptor.UniqueID;
     VSTestCase = VsDiscoverySink.CreateVsTestCase(source, descriptor, logger, testPlatformContext);
     TraitNames = descriptor.Traits.Keys;
 }
Beispiel #10
0
        IList <DiscoveredTestCase> GetVsTestCases(string source, ITestFrameworkDiscoverer discoverer, VsExecutionDiscoverySink visitor, LoggerHelper logger, TestPlatformContext testPlatformContext)
        {
            var descriptorProvider = (discoverer as ITestCaseDescriptorProvider) ?? new DefaultTestCaseDescriptorProvider(discoverer);
            var testCases          = visitor.TestCases;
            var descriptors        = descriptorProvider.GetTestCaseDescriptors(testCases, false);
            var results            = new List <DiscoveredTestCase>(descriptors.Count);

            for (int idx = 0; idx < descriptors.Count; ++idx)
            {
                results.Add(new DiscoveredTestCase(source, descriptors[idx], testCases[idx], logger, testPlatformContext));
            }

            return(results);
        }
Beispiel #11
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                TestPlatformContext testPlatformContext,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assemblyDisplayName = "(unknown assembly)";

            try
            {
                var assembly = new XunitProjectAssembly {
                    AssemblyFilename = runInfo.AssemblyFileName
                };
                var assemblyFileName = runInfo.AssemblyFileName;
                assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
                var configuration = runInfo.Configuration;
                var shadowCopy    = configuration.ShadowCopyOrDefault;

                var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

                if (RunSettingsHelper.DisableAppDomain)
                {
                    appDomain = AppDomainSupport.Denied;
                }

#if WINDOWS_UAP
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink        = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                {
                    var testCasesMap = new Dictionary <string, TestCase>();
                    var testCases    = new List <ITestCase>();
                    if (runInfo.TestCases == null || !runInfo.TestCases.Any())
                    {
                        // Discover tests
                        AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
                        DiscoverTestsInSource(controller, logger, testPlatformContext,
                                              (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
                                              (source, discoverer, discoveryOptions, visitor) =>
                        {
                            if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
                            {
                                foreach (var testCase in visitor.TestCases)
                                {
                                    logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
                                               testCase.DisplayName, testCase.UniqueID);
                                }
                            }

                            assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
                            {
                                AssemblyFileName    = source,
                                DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
                            };
                        },
                                              assemblyFileName,
                                              shadowCopy,
                                              configuration
                                              );

                        if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
                        {
                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
                            }

                            return;
                        }

                        // Filter tests
                        var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));

                        // Apply any filtering
                        var filter            = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
                        var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();

                        foreach (var filteredTestCase in filteredTestCases)
                        {
                            var uniqueID = filteredTestCase.UniqueID;
                            if (testCasesMap.ContainsKey(uniqueID))
                            {
                                logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
                            }
                            else
                            {
                                testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
                                testCases.Add(filteredTestCase.TestCase);
                            }
                        }
                    }
                    else
                    {
                        // We are in Run Specific tests scenario, the `TestCase` objects are available.
                        // Query the `TestCase` objects to find XunitTestCase objects.
                        foreach (var vstestCase in runInfo.TestCases)
                        {
                            var xunitTestCase = Deserialize(logger, controller, vstestCase);
                            if (xunitTestCase != null)
                            {
                                testCasesMap.Add(xunitTestCase.UniqueID, vstestCase);
                                testCases.Add(xunitTestCase);
                            }
                        }
                    }

                    // Execute tests
                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (RunSettingsHelper.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(testCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }
Beispiel #12
0
        private bool DiscoverTestsInSource <TVisitor>(XunitFrontController framework,
                                                      LoggerHelper logger,
                                                      TestPlatformContext testPlatformContext,
                                                      Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
                                                      Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete,
                                                      string assemblyFileName,
                                                      bool shadowCopy,
                                                      TestAssemblyConfiguration configuration)
            where TVisitor : IVsDiscoverySink, IDisposable
        {
            if (cancelled)
            {
                return(false);
            }

            var fileName = "(unknown assembly)";

            try
            {
                var reporterMessageHandler = GetRunnerReporter(new[] { assemblyFileName }).CreateMessageHandler(new VisualStudioRunnerLogger(logger));
                var assembly = new XunitProjectAssembly {
                    AssemblyFilename = assemblyFileName
                };
                fileName = Path.GetFileNameWithoutExtension(assemblyFileName);

                if (!IsXunitTestAssembly(assemblyFileName))
                {
                    if (configuration.DiagnosticMessagesOrDefault)
                    {
                        logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
                    }
                }
                else
                {
                    var targetFramework = framework.TargetFramework;
                    if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
                        targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
                        targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
                    {
                        if (configuration.DiagnosticMessagesOrDefault)
                        {
                            logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
                        }
                    }
                    else
                    {
                        var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);

                        using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
                        {
                            var usingAppDomains = framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied;
                            reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, usingAppDomains, shadowCopy, discoveryOptions));

                            framework.Find(testPlatformContext.RequireSourceInformation, visitor, discoveryOptions);

                            var totalTests = visitor.Finish();

                            visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);

                            reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
                        }
                    }
                }
            }
            catch (Exception e)
            {
                var ex = e.Unwrap();

                if (ex is InvalidOperationException)
                {
                    logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
                }
                else if (ex is FileNotFoundException fileNotFound)
                {
                    logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
                }
#if !WINDOWS_UAP
                else if (ex is FileLoadException fileLoad)
                {
                    logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
                }
#endif
                else
                {
                    logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
                }
            }

            return(true);
        }
Beispiel #13
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                TestPlatformContext testPlatformContext,
                                RunSettings runSettings,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assemblyDisplayName = "(unknown assembly)";

            try
            {
                var assembly = new XunitProjectAssembly {
                    AssemblyFilename = runInfo.AssemblyFileName
                };
                var assemblyFileName = runInfo.AssemblyFileName;
                assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
                var configuration = runInfo.Configuration;
                var shadowCopy    = configuration.ShadowCopyOrDefault;

                var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

                if (runSettings.DisableAppDomain)
                {
                    appDomain = AppDomainSupport.Denied;
                }

#if WINDOWS_UAP
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink        = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                {
                    var testCasesMap = new Dictionary <string, TestCase>();
                    var testCases    = new List <ITestCase>();
                    if (runInfo.TestCases == null || !runInfo.TestCases.Any())
                    {
                        // Discover tests
                        AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
                        DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings,
                                              (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
                                              (source, discoverer, discoveryOptions, visitor) =>
                        {
                            if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
                            {
                                foreach (var testCase in visitor.TestCases)
                                {
                                    logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
                                               testCase.DisplayName, testCase.UniqueID);
                                }
                            }

                            assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
                            {
                                AssemblyFileName    = source,
                                DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
                            };
                        },
                                              assemblyFileName,
                                              shadowCopy,
                                              configuration
                                              );

                        if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
                        {
                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
                            }

                            return;
                        }

                        // Filter tests
                        var traitNames        = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
                        var filter            = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
                        var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();

                        foreach (var filteredTestCase in filteredTestCases)
                        {
                            var uniqueID = filteredTestCase.UniqueID;
                            if (testCasesMap.ContainsKey(uniqueID))
                            {
                                logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
                            }
                            else
                            {
                                testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
                                testCases.Add(filteredTestCase.TestCase);
                            }
                        }
                    }
                    else
                    {
                        try
                        {
                            var serializations = runInfo.TestCases
                                                 .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
                                                 .ToList();

                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}",
                                                     serializations.Count,
                                                     Environment.NewLine,
                                                     string.Join(Environment.NewLine, serializations.Select(x => $"  {x}")));
                            }

                            var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);

                            if (deserializedTestCasesByUniqueId == null)
                            {
                                logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
                            }
                            else
                            {
                                for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
                                {
                                    try
                                    {
                                        var kvp        = deserializedTestCasesByUniqueId[idx];
                                        var vsTestCase = runInfo.TestCases[idx];

                                        if (kvp.Value == null)
                                        {
                                            logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
                                        }
                                        else
                                        {
                                            testCasesMap.Add(kvp.Key, vsTestCase);
                                            testCases.Add(kvp.Value);
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
                        }
                    }

                    // Execute tests
                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (runSettings.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(testCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }