예제 #1
0
        Assembly LoadAssembly(AssemblyLoadContext context, AssemblyName assemblyName)
        {
            if (lookupCache.TryGetValue(assemblyName.Name, out var result))
            {
                return(result);
            }

            if (internalDiagnosticsMessageSink != null)
            {
                internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Resolving '{assemblyName.Name}'"));
            }

            var path = Path.Combine(directory, assemblyName.Name);

            result = ResolveAndLoadAssembly(context, path, out var resolvedAssemblyPath);

            if (internalDiagnosticsMessageSink != null)
            {
                if (result == null)
                {
                    internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Resolution failed, passed down to next resolver"));
                }
                else
                {
                    internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Successful: '{resolvedAssemblyPath}'"));
                }
            }

            lookupCache[assemblyName.Name] = result;
            return(result);
        }
예제 #2
0
        public async Task InstallingSamePackageFromRemoteUpdatesLocal(string installCommandName, string uninstallCommandName)
        {
            _messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)} started."));
            var home = TestUtils.CreateTemporaryFolder("Home");

            using var packageManager = new PackageManager();
            string packageLocation = await packageManager.GetNuGetPackage(
                "Microsoft.DotNet.Common.ProjectTemplates.5.0",
                minimumVersion : new NuGet.Versioning.NuGetVersion(6, 0, 0),
                logger : new XunitNuGetLogger(_messageSink)).ConfigureAwait(false);

            _messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)}: Microsoft.DotNet.Common.ProjectTemplates.5.0 is downloaded to {packageLocation}.)"));

            new DotnetNewCommand(_log, installCommandName, packageLocation)
            .WithCustomHive(home)
            .WithWorkingDirectory(TestUtils.CreateTemporaryFolder())
            .Execute()
            .Should().ExitWith(0)
            .And.NotHaveStdErr()
            .And.HaveStdOutMatching($"Success: Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0::([\\d\\.a-z-])+ installed the following templates:")
            .And.HaveStdOutContaining("console")
            .And.HaveStdOutContaining("classlib");

            new DotnetNewCommand(_log, uninstallCommandName)
            .WithCustomHive(home)
            .WithWorkingDirectory(TestUtils.CreateTemporaryFolder())
            .Execute()
            .Should().ExitWith(0)
            .And.NotHaveStdErr()
            .And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0")
            .And.HaveStdOutContaining("Author: Microsoft")
            .And.HaveStdOutContaining("Version:")
            .And.NotHaveStdOutContaining("Version: 5.0.0");

            new DotnetNewCommand(_log, installCommandName, "Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0")
            .WithCustomHive(home)
            .WithWorkingDirectory(TestUtils.CreateTemporaryFolder())
            .Execute()
            .Should().ExitWith(0)
            .And.NotHaveStdErr()
            .And.HaveStdOutContaining("The following template packages will be installed:")
            .And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0")
            .And.HaveStdOutMatching("Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0 \\(version ([\\d\\.a-z-])+\\) is already installed, it will be replaced with version 5\\.0\\.0")
            .And.HaveStdOutMatching("Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0::([\\d\\.a-z-])+ was successfully uninstalled")
            .And.HaveStdOutContaining($"Success: Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0 installed the following templates:")
            .And.HaveStdOutContaining("console")
            .And.HaveStdOutContaining("classlib");

            new DotnetNewCommand(_log, uninstallCommandName)
            .WithCustomHive(home)
            .WithWorkingDirectory(TestUtils.CreateTemporaryFolder())
            .Execute()
            .Should().ExitWith(0)
            .And.NotHaveStdErr()
            .And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0")
            .And.HaveStdOutContaining("Author: Microsoft")
            .And.HaveStdOutContaining("Version: 5.0.0");

            _messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)} finished."));
        }
예제 #3
0
파일: Xunit1.cs 프로젝트: kerryjiang/xunit
        RunSummary RunTestClass(ITestCollection testCollection, string className, IList <Xunit1TestCase> testCases, IMessageSink messageSink)
        {
            var handler = new TestClassCallbackHandler(testCases, messageSink);
            var results = handler.TestClassResults;

            results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className));

            if (results.Continue)
            {
                try
                {
                    var methodNames = testCases.Select(tc => tc.Method.Name).ToList();
                    executor.RunTests(className, methodNames, handler);
                    handler.LastNodeArrived.WaitOne();
                }
                catch (Exception ex)
                {
                    var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex);

                    results.Continue = messageSink.OnMessage(new ErrorMessage(failureInformation.ExceptionTypes, failureInformation.Messages, failureInformation.StackTraces, failureInformation.ExceptionParentIndices)) && results.Continue;
                }
            }

            results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
            return(results);
        }
예제 #4
0
        bool OnStart(XmlNode xml)
        {
            var testCase = FindTestCase(xml.Attributes["type"].Value, xml.Attributes["method"].Value);

            SendTestCaseMessagesWhenAppropriate(testCase);
            return(messageSink.OnMessage(new TestStarting(testCase, xml.Attributes["name"].Value)) && TestClassResults.Continue);
        }
        bool OnClass(XmlNode xml)
        {
            SendTestCaseMessagesWhenAppropriate(null);

            var     @continue = true;
            XmlNode failureNode;

            if ((failureNode = xml.SelectSingleNode("failure")) != null)
            {
                var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(failureNode);

                var errorMessage = new ErrorMessage(testCases,
                                                    failureInformation.ExceptionTypes,
                                                    failureInformation.Messages,
                                                    failureInformation.StackTraces,
                                                    failureInformation.ExceptionParentIndices);
                @continue = messageSink.OnMessage(errorMessage);
            }

            TestClassResults.Time    = Decimal.Parse(xml.Attributes["time"].Value, CultureInfo.InvariantCulture);
            TestClassResults.Total   = Int32.Parse(xml.Attributes["total"].Value, CultureInfo.InvariantCulture);
            TestClassResults.Failed  = Int32.Parse(xml.Attributes["failed"].Value, CultureInfo.InvariantCulture);
            TestClassResults.Skipped = Int32.Parse(xml.Attributes["skipped"].Value, CultureInfo.InvariantCulture);
            return(@continue && TestClassResults.Continue);
        }
        public OutputController(IReportConfiguration configuration, IMessageSink diagnosticMessageSink)
        {
            _configuration         = configuration;
            _diagnosticMessageSink = diagnosticMessageSink;

            IReportWriter writer;

            if (!configuration.WriteOutput)
            {
                _diagnosticMessageSink.OnMessage(new DiagnosticMessage("Output is disabled"));
                writer = new NullWriter();
            }
            else
            {
                _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Creating report at {configuration.XmlReportFile}"));
                _fileStream = new FileStream(configuration.XmlReportFile, FileMode.Create, FileAccess.Write,
                                             FileShare.None, 4096, true);

                _sw = new StreamWriter(_fileStream);

                XmlWriterSettings xws = new XmlWriterSettings();
                xws.Async = true;
                _xw       = XmlWriter.Create(_sw, xws);

                writer = new ReportWriter(_xw);
            }
            Report = new ScenarioReport(configuration.AssemblyName, writer, _diagnosticMessageSink);
        }
예제 #7
0
        /// <inheritdoc/>
        public void Run(IEnumerable<ITestCase> testMethods, IMessageSink messageSink)
        {
            bool cancelled = false;
            var totalSummary = new RunSummary();

            string currentDirectory = Directory.GetCurrentDirectory();

            try
            {
                Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));

                if (messageSink.OnMessage(new TestAssemblyStarting()))
                {
                    var classGroups = testMethods.Cast<XunitTestCase>().GroupBy(tc => tc.Class).ToList();

                    if (classGroups.Count > 0)
                    {
                        var collectionSummary = new RunSummary();

                        if (messageSink.OnMessage(new TestCollectionStarting()))
                        {
                            foreach (var group in classGroups)
                            {
                                var classSummary = new RunSummary();

                                if (!messageSink.OnMessage(new TestClassStarting { ClassName = group.Key.Name }))
                                    cancelled = true;
                                else
                                {
                                    cancelled = RunTestClass(messageSink, group, classSummary);
                                    collectionSummary.Aggregate(classSummary);
                                }

                                if (!messageSink.OnMessage(new TestClassFinished { Assembly = assemblyInfo, ClassName = group.Key.Name, TestsRun = classSummary.Total }))
                                    cancelled = true;

                                if (cancelled)
                                    break;
                            }
                        }

                        messageSink.OnMessage(new TestCollectionFinished { Assembly = assemblyInfo, TestsRun = collectionSummary.Total });
                        totalSummary.Aggregate(collectionSummary);
                    }
                }

                messageSink.OnMessage(new TestAssemblyFinished
                {
                    Assembly = assemblyInfo,
                    TestsRun = totalSummary.Total,
                    TestsFailed = totalSummary.Failed,
                    TestsSkipped = totalSummary.Skipped,
                    ExecutionTime = totalSummary.Time
                });
            }
            finally
            {
                Directory.SetCurrentDirectory(currentDirectory);
            }
        }
        public Assembly LoadManagedDll([NotNull] string assemblyName, [NotNull] Func <string, Assembly> managedAssemblyLoader)
        {
            if (_managedAssemblyCache.TryGetValue(assemblyName, out var result))
            {
                return(result);
            }

            _internalDiagnosticsMessageSink?.OnMessage(new DiagnosticMessage($"[DependencyContextAssemblyCache.LoadManagedDll] Resolving '{assemblyName}'"));

            var tupleResult          = ResolveManagedAssembly(assemblyName, managedAssemblyLoader);
            var resolvedAssemblyPath = tupleResult.Item1;

            result = tupleResult.Item2;
            _managedAssemblyCache[assemblyName] = result;

            if (_internalDiagnosticsMessageSink == null)
            {
                return(result);
            }

            _internalDiagnosticsMessageSink.OnMessage(
                result == null
                                                          ? new DiagnosticMessage("[DependencyContextAssemblyCache.LoadManagedDll] Resolution failed, passed down to next resolver")
                                                          : new DiagnosticMessage($"[DependencyContextAssemblyCache.LoadManagedDll] Successful: '{resolvedAssemblyPath}'"));

            return(result);
        }
예제 #9
0
        /// <inheritdoc/>
        protected override IntPtr LoadUnmanagedDll(string unmanagedDllName)
        {
            var result = base.LoadUnmanagedDll(unmanagedDllName);

            if (result != default)
            {
                return(result);
            }

            if (!unmanagedAssemblyCache.TryGetValue(unmanagedDllName, out var resolvedAssemblyPath))
            {
                if (internalDiagnosticsMessageSink != null)
                {
                    internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Resolving '{unmanagedDllName}'"));
                }

                resolvedAssemblyPath = ResolveUnmanagedAssembly(unmanagedDllName);
                unmanagedAssemblyCache[unmanagedDllName] = resolvedAssemblyPath;

                if (internalDiagnosticsMessageSink != null)
                {
                    if (resolvedAssemblyPath == null)
                    {
                        internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Resolution failed, passed down to next resolver"));
                    }
                    else
                    {
                        internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Successful: '{resolvedAssemblyPath}'"));
                    }
                }
            }

            return(resolvedAssemblyPath != null?LoadUnmanagedDllFromPath(resolvedAssemblyPath) : default);
예제 #10
0
파일: Xunit1.cs 프로젝트: wradecki/xunit
        void Find(Predicate <ITestCaseDiscoveryMessage> filter, bool includeSourceInformation, IMessageSink messageSink)
        {
            try
            {
                XmlNode assemblyXml = null;

                var handler = new XmlNodeCallbackHandler(xml => { assemblyXml = xml; return(true); });
                executor.EnumerateTests(handler);

                foreach (XmlNode method in assemblyXml.SelectNodes("//method"))
                {
                    var testCase = method.ToTestCase(assemblyFileName, configFileName);
                    if (testCase != null)
                    {
                        if (includeSourceInformation)
                        {
                            testCase.SourceInformation = sourceInformationProvider.GetSourceInformation(testCase);
                        }

                        var message = new TestCaseDiscoveryMessage(testCase);
                        if (filter(message))
                        {
                            messageSink.OnMessage(message);
                        }
                    }
                }
            }
            finally
            {
                messageSink.OnMessage(new DiscoveryCompleteMessage());
            }
        }
예제 #11
0
            protected override async Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
            {
                var parameters = string.Empty;

                if (testCase.TestMethodArguments != null)
                {
                    parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null"));
                }

                var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})";

                _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"STARTED: {test}"));

                using var timer = new Timer(
                          _ => _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"WARNING: {test} has been running for more than 15 minutes")),
                          null,
                          TimeSpan.FromMinutes(15),
                          Timeout.InfiniteTimeSpan);

                try
                {
                    var result = await base.RunTestCaseAsync(testCase);

                    var status = result.Failed > 0 ? "FAILURE" : "SUCCESS";

                    _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"{status}: {test} ({result.Time}s)"));

                    return(result);
                }
                catch (Exception ex)
                {
                    _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"ERROR: {test} ({ex.Message})"));
                    throw;
                }
            }
예제 #12
0
        Assembly LoadAssembly(AssemblyName assemblyName)
        {
            if (lookupCache.TryGetValue(assemblyName.Name, out var result))
            {
                return(result);
            }

            var path = Path.Combine(directory, assemblyName.Name);

            result = ResolveAndLoadAssembly(path, out var resolvedAssemblyPath);

            if (internalDiagnosticsMessageSink != null)
            {
                if (result == null)
                {
                    internalDiagnosticsMessageSink.OnMessage(new _DiagnosticMessage($"[AssemblyHelper_Desktop.LoadAssembly] Resolution for '{assemblyName.Name}' failed, passed down to next resolver"));
                }
                else
                {
                    internalDiagnosticsMessageSink.OnMessage(new _DiagnosticMessage($"[AssemblyHelper_Desktop.LoadAssembly] Resolved '{assemblyName.Name}' to '{resolvedAssemblyPath}'"));
                }
            }

            lookupCache[assemblyName.Name] = result;
            return(result);
        }
예제 #13
0
파일: Xunit1.cs 프로젝트: wradecki/xunit
        /// <summary>
        /// Starts the process of running all the xUnit.net v1 tests.
        /// </summary>
        /// <param name="testCases">The test cases to run; if null, all tests in the assembly are run.</param>
        /// <param name="messageSink">The message sink to report results back to.</param>
        public void Run(IEnumerable <ITestCase> testCases, IMessageSink messageSink)
        {
            var results        = new Xunit1RunSummary();
            var environment    = String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version);
            var firstTestCase  = testCases.FirstOrDefault();
            var testCollection = firstTestCase == null ? null : firstTestCase.TestMethod.TestClass.TestCollection;

            if (testCollection != null)
            {
                try
                {
                    if (messageSink.OnMessage(new TestAssemblyStarting(testCases, testCollection.TestAssembly, DateTime.Now, environment, TestFrameworkDisplayName)))
                    {
                        results = RunTestCollection(testCollection, testCases, messageSink);
                    }
                }
                catch (Exception ex)
                {
                    var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex);

                    messageSink.OnMessage(new ErrorMessage(testCases, failureInformation.ExceptionTypes,
                                                           failureInformation.Messages, failureInformation.StackTraces,
                                                           failureInformation.ExceptionParentIndices));
                }
                finally
                {
                    messageSink.OnMessage(new TestAssemblyFinished(testCases, testCollection.TestAssembly, results.Time, results.Total, results.Failed, results.Skipped));
                }
            }
        }
예제 #14
0
        /// <summary>
        /// Gives an opportunity to override test case orderer. By default, this method gets the
        /// orderer from the collection definition. If this function returns <c>null</c>, the
        /// test case orderer passed into the constructor will be used.
        /// </summary>
        protected virtual ITestCaseOrderer GetTestCaseOrderer()
        {
            if (TestCollection.CollectionDefinition != null)
            {
                var ordererAttribute = TestCollection.CollectionDefinition.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
                if (ordererAttribute != null)
                {
                    try
                    {
                        var testCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(diagnosticMessageSink, ordererAttribute);
                        if (testCaseOrderer != null)
                        {
                            return(testCaseOrderer);
                        }

                        var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList();
                        diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Could not find type '{args[0]}' in {args[1]} for collection-level test case orderer on test collection '{TestCollection.DisplayName}'"));
                    }
                    catch (Exception ex)
                    {
                        var innerEx = ex.Unwrap();
                        var args    = ordererAttribute.GetConstructorArguments().Cast <string>().ToList();
                        diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Collection-level test case orderer '{args[0]}' for test collection '{TestCollection.DisplayName}' threw '{innerEx.GetType().FullName}' during construction: {innerEx.Message}{Environment.NewLine}{innerEx.StackTrace}"));
                    }
                }
            }

            return(null);
        }
예제 #15
0
파일: Xunit1.cs 프로젝트: donisetti/xunit
        Xunit1RunSummary RunTestCollection(ITestCollection testCollection, IEnumerable <ITestCase> testCases, IMessageSink messageSink)
        {
            var results = new Xunit1RunSummary();

            results.Continue = messageSink.OnMessage(new TestCollectionStarting(testCases, testCollection));

            try
            {
                if (results.Continue)
                {
                    foreach (var testClassGroup in testCases.GroupBy(tc => tc.TestMethod.TestClass, Comparer.Instance))
                    {
                        var classResults = RunTestClass(testClassGroup.Key, testClassGroup.ToList(), messageSink);
                        results.Aggregate(classResults);
                        if (!classResults.Continue)
                        {
                            break;
                        }
                    }
                }
            }
            finally
            {
                results.Continue = messageSink.OnMessage(new TestCollectionFinished(testCases, testCollection, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
            }

            return(results);
        }
예제 #16
0
        /// <inheritdoc/>
        protected void SetTestCaseOrderer(IReflectionTypeInfo Class)
        {
            var ordererAttribute = Class.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();

            if (ordererAttribute != null)
            {
                try
                {
                    var testCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(diagnosticMessageSink, ordererAttribute);
                    if (testCaseOrderer != null)
                    {
                        TestCaseOrderer = testCaseOrderer;
                    }
                    else
                    {
                        var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList();
                        diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Could not find type '{args[0]}' in {args[1]} for class-level test case orderer on test class '{Class.Name}'"));
                    }
                }
                catch (Exception ex)
                {
                    var innerEx = ex.InnerException ?? ex; //.Unwrap();
                    var args    = ordererAttribute.GetConstructorArguments().Cast <string>().ToList();
                    diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Class-level test case orderer '{args[0]}' for test class '{Class.Name}' threw '{innerEx.GetType().FullName}' " +
                                                                          $"during construction: {innerEx.Message}{Environment.NewLine}{innerEx.StackTrace}"));
                }
            }
        }
예제 #17
0
        /// <inheritdoc/>
        protected override bool RunTests(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator)
        {
            bool cancelled = false;

            if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = DisplayName }))
                cancelled = true;
            else
            {
                try
                {
                    lambda();

                    if (!messageSink.OnMessage(new TestPassed { TestCase = this, TestDisplayName = DisplayName }))
                        cancelled = true;
                }
                catch (Exception ex)
                {
                    if (!messageSink.OnMessage(new TestFailed(ex) { TestCase = this, TestDisplayName = DisplayName }))
                        cancelled = true;
                }
            }

            if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = DisplayName }))
                cancelled = true;

            return cancelled;
        }
        public static IDisposable SubscribeResolveForAssembly([NotNull] string assemblyFileName, [CanBeNull] IMessageSink internalDiagnosticsMessageSink)
        {
            var assemblyFolder = Path.GetDirectoryName(assemblyFileName).ThrowIfNull();
            var depsJsonFile   = Path.Combine(assemblyFolder, Path.GetFileNameWithoutExtension(assemblyFileName) + ".deps.json");

            if (!File.Exists(depsJsonFile))
            {
                internalDiagnosticsMessageSink?.OnMessage(
                    new DiagnosticMessage(
                        $"[DependencyContextAssemblyHelper.SubscribeResolveForAssembly] Skipping resolution for '{depsJsonFile}': File not found"));

                return(null);
            }

            using (var stream = File.OpenRead(depsJsonFile))
            {
                var context = JsonReader.Read(stream);
                if (context != null)
                {
                    return(new DependencyContextAssemblyHelper(assemblyFolder, context, internalDiagnosticsMessageSink));
                }

                internalDiagnosticsMessageSink?.OnMessage(
                    new DiagnosticMessage(
                        $"[DependencyContextAssemblyHelper.SubscribeResolveForAssembly] Skipping resolution for '{depsJsonFile}': File appears to be malformed"));

                return(null);
            }
        }
            protected override async Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
            {
                var parameters = string.Empty;

                if (testCase.TestMethodArguments != null)
                {
                    parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null"));
                }

                var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})";

                _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"STARTED: {test}"));

                try
                {
                    var result = await base.RunTestCaseAsync(testCase);

                    var status = result.Failed > 0 ? "FAILURE" : "SUCCESS";

                    _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"{status}: {test} ({result.Time}s)"));

                    return(result);
                }
                catch (Exception ex)
                {
                    _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"ERROR: {test} ({ex.Message})"));
                    throw;
                }
            }
예제 #20
0
        static Type GetTestFrameworkType(IAssemblyInfo testAssembly, IMessageSink diagnosticMessageSink)
        {
            try
            {
                var testFrameworkAttr = testAssembly.GetCustomAttributes(typeof(ITestFrameworkAttribute)).FirstOrDefault();
                if (testFrameworkAttr != null)
                {
                    var discovererAttr = testFrameworkAttr.GetCustomAttributes(typeof(TestFrameworkDiscovererAttribute)).FirstOrDefault();
                    if (discovererAttr != null)
                    {
                        var discoverer = ExtensibilityPointFactory.GetTestFrameworkTypeDiscoverer(diagnosticMessageSink, discovererAttr);
                        if (discoverer != null)
                        {
                            return(discoverer.GetTestFrameworkType(testFrameworkAttr));
                        }

                        var ctorArgs = discovererAttr.GetConstructorArguments().ToArray();
                        diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Unable to create custom test framework discoverer type '{ctorArgs[1]}, {ctorArgs[0]}'"));
                    }
                    else
                    {
                        diagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test framework attribute was not decorated with [TestFrameworkDiscoverer]"));
                    }
                }
            }
            catch (Exception ex)
            {
                diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework discoverer construction: {ex.Unwrap()}"));
            }

            return(typeof(XunitTestFramework));
        }
예제 #21
0
        protected override async Task <RunSummary> RunTestAsync()
        {
            var runSummary = new BenchmarkRunSummary
            {
                TestClassFullName       = TestCase.TestMethod.TestClass.Class.Name,
                TestClass               = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
                TestMethod              = TestCase.TestMethodName,
                Variation               = TestCase.Variation,
                ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
                RunStarted              = DateTime.UtcNow,
                MachineName             = _machineName,
                Framework               = _framework,
                Architecture            = IntPtr.Size > 4 ? "x64" : "x86",
                WarmupIterations        = TestCase.WarmupIterations,
                Iterations              = TestCase.Iterations,
                CustomData              = BenchmarkConfig.Instance.CustomData
            };

            for (var i = 0; i < TestCase.WarmupIterations; i++)
            {
                var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
                runSummary.Aggregate(await runner.RunAsync());
            }

            for (var i = 0; i < TestCase.Iterations; i++)
            {
                TestCase.MetricCollector.Reset();
                var runner           = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
                var iterationSummary = new BenchmarkIterationSummary();
                iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
                runSummary.Aggregate(iterationSummary);
            }

            if (runSummary.Failed != 0)
            {
                _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
            }
            else
            {
                runSummary.PopulateMetrics();
                _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));

                foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
                {
                    try
                    {
                        new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
                    }
                    catch (Exception ex)
                    {
                        _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}"));
                        throw;
                    }
                }
            }

            return(runSummary);
        }
        /// <summary>
        ///     Initializes a new instance of the <see cref="DependencyContextAssemblyCache" /> class.
        /// </summary>
        /// <param name="assemblyFolder">The assembly folder</param>
        /// <param name="dependencyContext">The dependency context</param>
        /// <param name="internalDiagnosticsMessageSink">The message sink</param>
        public DependencyContextAssemblyCache(
            [NotNull] string assemblyFolder,
            [NotNull] DependencyContext dependencyContext,
            [CanBeNull] IMessageSink internalDiagnosticsMessageSink)
        {
            _assemblyFolder = assemblyFolder;
            _internalDiagnosticsMessageSink = internalDiagnosticsMessageSink;

            _assemblyResolver = new XunitPackageCompilationAssemblyResolver(internalDiagnosticsMessageSink);

            internalDiagnosticsMessageSink?.OnMessage(
                new DiagnosticMessage(
                    $"[DependencyContextAssemblyCache..ctor] Runtime graph: [{string.Join(",", dependencyContext.RuntimeGraph.Select(x => $"'{x.Runtime}'"))}]"));

            var currentRuntime = RuntimeEnvironment.GetRuntimeIdentifier();
            var fallbacks      = dependencyContext.RuntimeGraph.FirstOrDefault(x => string.Equals(x.Runtime, currentRuntime, StringComparison.OrdinalIgnoreCase));
            HashSet <string> compatibleRuntimes = fallbacks != null
                                                     ? new HashSet <string>(fallbacks.Fallbacks, StringComparer.OrdinalIgnoreCase)
                                                     : new HashSet <string>(StringComparer.OrdinalIgnoreCase);

            compatibleRuntimes.Add(currentRuntime);
            compatibleRuntimes.Add(string.Empty);

            internalDiagnosticsMessageSink?.OnMessage(
                new DiagnosticMessage(
                    $"[DependencyContextAssemblyCache..ctor] Compatible runtimes: [{string.Join(",", compatibleRuntimes.Select(x => $"'{x}'"))}]"));

            _managedAssemblyCache = new Dictionary <string, Assembly>(StringComparer.OrdinalIgnoreCase);
            _managedAssemblyMap   =
                dependencyContext.RuntimeLibraries
                .Where(lib => lib.RuntimeAssemblyGroups?.Count > 0)
                .Select(lib => Tuple.Create(lib, lib.RuntimeAssemblyGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime))))
                .Where(tuple => tuple.Item2?.AssetPaths != null)
                .SelectMany(
                    tuple => tuple.Item2.AssetPaths.Where(x => x != null)
                    .Select(path => Tuple.Create(Path.GetFileNameWithoutExtension(path), Tuple.Create(tuple.Item1, tuple.Item2))))
                .ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase);

            internalDiagnosticsMessageSink?.OnMessage(
                new DiagnosticMessage(
                    $"[DependencyContextAssemblyCache..ctor] Managed assembly map includes: {string.Join(",", _managedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}"));

#if NETCOREAPP1_0
            unmanagedAssemblyCache = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
            unmanagedAssemblyMap   =
                dependencyContext.RuntimeLibraries
                .Where(lib => lib.NativeLibraryGroups?.Count > 0)
                .Select(lib => Tuple.Create(lib, lib.NativeLibraryGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime))))
                .Where(tuple => tuple.Item2?.AssetPaths != null)
                .SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileName(path), Tuple.Create(tuple.Item1, tuple.Item2))))
                .ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase);

            if (internalDiagnosticsMessageSink != null)
            {
                internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[DependencyContextAssemblyCache..ctor] Unmanaged assembly map includes: {string.Join(",", unmanagedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}"));
            }
#endif
        }
예제 #23
0
        public DriverFixture(IMessageSink messageSink)
        {
            try
            {
                Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);

                var testWebBrowser = Environment.GetEnvironmentVariable("TestWebBrowser") ?? "Chrome";
                messageSink.OnMessage(new DiagnosticMessage($"TestWebBrowser = {testWebBrowser}"));

                if (testWebBrowser == "Firefox")
                {
                    var profiles = new FirefoxProfileManager();
                    var profile  = profiles.GetProfile("selenium");

                    messageSink.OnMessage(new DiagnosticMessage("Profiles:"));
                    foreach (var existingProfile in profiles.ExistingProfiles)
                    {
                        messageSink.OnMessage(new DiagnosticMessage($"Profile = {existingProfile}"));
                    }

                    var options = new FirefoxOptions {
                        Profile = profile
                    };
                    options.SetLoggingPreference(LogType.Browser, LogLevel.All);

                    Driver = new FirefoxDriver(FirefoxDriverService.CreateDefaultService(Environment.CurrentDirectory),
                                               new FirefoxOptions {
                        Profile = profile
                    },
                                               TimeSpan.FromSeconds(30));
                }
                else if (testWebBrowser == "IE")
                {
                    Driver = new InternetExplorerDriver(Environment.CurrentDirectory);
                }
                else if (testWebBrowser == "Chrome")
                {
                    Driver = new ChromeDriver(Environment.CurrentDirectory);
                }

                Driver.Manage().Window.Position = new Point(0, 0);
                Driver.Manage().Window.Size = new Size(1920, 1080);

                Driver.Url = Environment.GetEnvironmentVariable("WebUrl") ?? "http://localhost:5000/";
                messageSink.OnMessage(new DiagnosticMessage($"WebUrl = {Driver?.Url ?? "NULL"}"));

                Driver.Manage().Timeouts().ImplicitWait           = TimeSpan.FromSeconds(10);
                Driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(10);
                Driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(10);

                Driver.FindElement(By.TagName("app-root"));
            }
            catch (Exception ex)
            {
                messageSink.OnMessage(new DiagnosticMessage($"Exception while initializing driver: {ex.Message}"));
                Driver?.Dispose();
            }
        }
예제 #24
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                IMessageSink reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assembly = new XunitProjectAssembly {
                AssemblyFilename = runInfo.AssemblyFileName
            };
            var assemblyFileName    = runInfo.AssemblyFileName;
            var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
            var shadowCopy          = assembly.Configuration.ShadowCopyOrDefault;
            var appDomain           = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;

            if (DisableAppDomainRequestedInRunContext(runContext.RunSettings.SettingsXml))
            {
                appDomain = AppDomainSupport.Denied;
            }

            try
            {
#if PLATFORM_DOTNET
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
                {
                    var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) })
                                         .Where(tc => tc.xunit != null)
                                         .ToDictionary(tc => tc.xunit, tc => tc.vs);

                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    executionOptions.SetSynchronousMessageReporting(true);

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var executionSink = new VsExecutionSink(frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled))
                    {
                        controller.RunTests(xunitTestCases.Keys.ToList(), executionSink, executionOptions);
                        executionSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, executionSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }
        /// <summary>
        /// Initializes a new instance of the <see cref="NetCoreAssemblyDependencyResolver"/> class.
        /// </summary>
        /// <param name="assemblyFilePath">The path to the assembly</param>
        /// <param name="internalDiagnosticsMessageSink">An optional message sink for use with internal diagnostics messages;
        /// may pass <c>null</c> for no internal diagnostics messages</param>
        public NetCoreAssemblyDependencyResolver(string assemblyFilePath, IMessageSink internalDiagnosticsMessageSink)
        {
            this.internalDiagnosticsMessageSink = internalDiagnosticsMessageSink;

            var assembly = LoadFromAssemblyPath(assemblyFilePath);

            assemblyFolder    = Path.GetDirectoryName(assemblyFilePath);
            assemblyResolver  = new XunitPackageCompilationAssemblyResolver();
            dependencyContext = DependencyContext.Load(assembly);

            var compatibleRuntimes = default(HashSet <string>);
            var currentRuntime     = RuntimeEnvironment.GetRuntimeIdentifier();
            var fallbacks          = dependencyContext.RuntimeGraph.FirstOrDefault(x => string.Equals(x.Runtime, currentRuntime, StringComparison.OrdinalIgnoreCase));

            if (fallbacks != null)
            {
                compatibleRuntimes = new HashSet <string>(fallbacks.Fallbacks, StringComparer.OrdinalIgnoreCase);
            }
            else
            {
                compatibleRuntimes = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
            }

            compatibleRuntimes.Add(currentRuntime);
            compatibleRuntimes.Add(string.Empty);

            managedAssemblyCache = new Dictionary <string, Assembly>(StringComparer.OrdinalIgnoreCase);
            managedAssemblyMap   =
                dependencyContext.RuntimeLibraries
                .Where(lib => lib.RuntimeAssemblyGroups?.Count > 0)
                .Select(lib => Tuple.Create(lib, lib.RuntimeAssemblyGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime))))
                .Where(tuple => tuple.Item2?.AssetPaths != null)
                .SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileNameWithoutExtension(path), Tuple.Create(tuple.Item1, tuple.Item2))))
                .ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase);

            if (internalDiagnosticsMessageSink != null)
            {
                internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver..ctor] Managed assembly map includes: {string.Join(",", managedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}"));
            }

            unmanagedAssemblyCache = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
            unmanagedAssemblyMap   =
                dependencyContext.RuntimeLibraries
                .Where(lib => lib.NativeLibraryGroups?.Count > 0)
                .Select(lib => Tuple.Create(lib, lib.NativeLibraryGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime))))
                .Where(tuple => tuple.Item2?.AssetPaths != null)
                .SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileName(path), Tuple.Create(tuple.Item1, tuple.Item2))))
                .ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase);

            if (internalDiagnosticsMessageSink != null)
            {
                internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver..ctor] Unmanaged assembly map includes: {string.Join(",", unmanagedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}"));
            }

            Default.Resolving += OnResolving;
        }
예제 #26
0
        /// <summary>
        /// Runs a retryable test case, handling any wait and retry logic between test runs, reporting statuses out to xunit etc...
        /// </summary>
        /// <param name="testCase">The test case to be retried</param>
        /// <param name="diagnosticMessageSink">The diagnostic message sink to write messages to about retries, waits etc...</param>
        /// <param name="messageBus">The message bus xunit is listening for statuses to report on</param>
        /// <param name="cancellationTokenSource">The cancellation token source from xunit</param>
        /// <param name="fnRunSingle">(async) Lambda to run this test case once (without retries) - takes the blocking message bus and returns the test run result</param>
        /// <returns>Resulting run summary</returns>
        public static async Task <RunSummary> RunAsync(
            IRetryableTestCase testCase,
            IMessageSink diagnosticMessageSink,
            IMessageBus messageBus,
            CancellationTokenSource cancellationTokenSource,
            Func <IMessageBus, Task <RunSummary> > fnRunSingle)
        {
            for (var i = 1; ; i++)
            {
                // Prevent messages from the test run from being passed through, as we don't want
                //  a message to mark the test as failed when we're going to retry it
                using BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus);
                diagnosticMessageSink.OnMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})",
                                                                      testCase.DisplayName, i, testCase.MaxRetries));

                blockingMessageBus.QueueMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})",
                                                                      testCase.DisplayName, i, testCase.MaxRetries));
                RunSummary summary = await fnRunSingle(blockingMessageBus);

                // If we succeeded, or we've reached the max retries return the result
                if (summary.Failed == 0 || i == testCase.MaxRetries)
                {
                    // If we have failed (after all retries, log that)
                    if (summary.Failed != 0)
                    {
                        diagnosticMessageSink.OnMessage(new DiagnosticMessage(
                                                            "Test \"{0}\" has failed and been retried the maximum number of times ({1})",
                                                            testCase.DisplayName, testCase.MaxRetries));
                    }

                    blockingMessageBus.Flush();
                    return(summary);
                }
                // Otherwise log that we've had a failed run and will retry
                diagnosticMessageSink.OnMessage(new DiagnosticMessage(
                                                    "Test \"{0}\" failed but is set to retry ({1}/{2}) . . .", testCase.DisplayName, i,
                                                    testCase.MaxRetries));

                // If there is a delay between test attempts, apply it now
                if (testCase.DelayBetweenRetriesMs > 0)
                {
                    diagnosticMessageSink.OnMessage(new DiagnosticMessage(
                                                        "Test \"{0}\" attempt ({1}/{2}) delayed by {3}ms. Waiting . . .", testCase.DisplayName, i,
                                                        testCase.MaxRetries, testCase.DelayBetweenRetriesMs));

                    // Don't await to prevent thread hopping.
                    //  If all of a users test cases in a collection/class are synchronous and expecting to not thread-hop
                    //  (because they're making use of thread static/thread local/managed thread ID to share data between tests rather than
                    //  a more modern async-friendly mechanism) then if a thread-hop were to happen here we'd get flickering tests.
                    //  SpecFlow relies on this as they use the managed thread ID to separate instances of some of their internal classes, which caused
                    //  a this problem for xRetry.SpecFlow: https://github.com/JoshKeegan/xRetry/issues/18
                    Task.Delay(testCase.DelayBetweenRetriesMs, cancellationTokenSource.Token).Wait();
                }
            }
        }
예제 #27
0
파일: xunitTests.cs 프로젝트: redoz/xunit
        private void ReturnDiscoveryMessages(IMessageSink sink)
        {
            foreach (var testCase in DiscoveryTestCases)
            {
                sink.OnMessage(new TestCaseDiscoveryMessage {
                    TestCase = testCase
                });
            }

            sink.OnMessage(new DiscoveryCompleteMessage());
        }
 public WinSqlite3TestFramework(IMessageSink messageSink)
     : base(messageSink)
 {
     messageSink.OnMessage(new DiagnosticMessage {
         Message = "Using custom test framework"
     });
     if (Environment.GetEnvironmentVariable("TEST_WINSQLITE3") != null)
     {
         messageSink.OnMessage(new DiagnosticMessage {
             Message = "Using winsqlite3"
         });
         SqliteEngine.UseWinSqlite3();
     }
 }
예제 #29
0
        /// <inheritdoc/>
        public override bool OnMessage(IMessageSinkMessage message)
        {
            var result = base.OnMessage(message);

            result = innerMessageSink.OnMessage(message) || result;
            return(result);
        }
        public ServiceProviderFixture(IMessageSink diagnosticMessageSink)
        {
            this.TestId        = Guid.NewGuid();
            this.Configuration = new ConfigurationBuilder().
                                 AddJsonFile("appsettings.test.json").
                                 AddJsonFile("appsettings.test.Development.json", true).
                                 // https://www.jerriepelser.com/blog/aspnet-core-no-more-worries-about-checking-in-secrets/
                                 AddUserSecrets <ServiceProviderFixture>().
                                 AddEnvironmentVariables("CTP_").
                                 Build();

            var containerType = Enum.Parse <ContainerType>(Configuration.GetValue("Container", "BuiltIn"));

            diagnosticMessageSink.OnMessage(new DiagnosticMessage("Use container {0}", containerType.ToString()));
            switch (containerType)
            {
            case ContainerType.BuiltIn:
                var services = new ServiceCollection();
                services.UseCommercetools(Configuration, "Client", TokenFlow.ClientCredentials);
                services.AddLogging(c => c.AddProvider(new InMemoryLoggerProvider()));
                this.ServiceProvider = services.BuildServiceProvider();
                break;

            case ContainerType.SimpleInjector:
                var container = new Container();
                container.UseCommercetools(Configuration, "Client", TokenFlow.ClientCredentials);
                container.Verify();
                this.ServiceProvider = container;
                break;
            }
        }
예제 #31
0
        // This method is called by the xUnit test framework classes to run the test case. We will do the
        // loop here, forwarding on to the implementation in XunitTestCase to do the heavy lifting. We will
        // continue to re-run the test until the aggregator has an error (meaning that some internal error
        // condition happened), or the test runs without failure, or we've hit the maximum number of tries.
        public override async Task <RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
                                                         IMessageBus messageBus,
                                                         object[] constructorArguments,
                                                         ExceptionAggregator aggregator,
                                                         CancellationTokenSource cancellationTokenSource)
        {
            var runCount = 0;

            while (true)
            {
                // This is really the only tricky bit: we need to capture and delay messages (since those will
                // contain run status) until we know we've decided to accept the final result;
                var delayedMessageBus = new DelayedMessageBus(messageBus);

                var summary = await base.RunAsync(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource);

                if (aggregator.HasExceptions || summary.Failed == 0 || ++runCount >= maxRetries)
                {
                    delayedMessageBus.Dispose();  // Sends all the delayed messages
                    return(summary);
                }

                diagnosticMessageSink.OnMessage(new DiagnosticMessage("Execution of '{0}' failed (attempt #{1}), retrying...", DisplayName, runCount));

                var delayInMs = Math.Min((int)Math.Pow(2, runCount - 1) * this.exponentialBackoffMs, MaxDelayInMs);
                await Task.Delay(delayInMs);
            }
        }
예제 #32
0
        /// <inheritdoc/>
        protected override void Initialize()
        {
            base.Initialize();

            var factAttribute   = TestMethod.Method.GetCustomAttributes(typeof(FactAttribute)).Single();
            var baseDisplayName = factAttribute.GetNamedArgument <string>("DisplayName") ?? BaseDisplayName;

            DisplayName = TypeUtility.GetDisplayNameWithArguments(TestMethod.Method, baseDisplayName, TestMethodArguments, MethodGenericTypes);
            SkipReason  = factAttribute.GetNamedArgument <string>("Skip");

            foreach (var traitAttribute in TestMethod.Method.GetCustomAttributes(typeof(ITraitAttribute))
                     .Concat(TestMethod.TestClass.Class.GetCustomAttributes(typeof(ITraitAttribute))))
            {
                var discovererAttribute = traitAttribute.GetCustomAttributes(typeof(TraitDiscovererAttribute)).FirstOrDefault();
                if (discovererAttribute != null)
                {
                    var discoverer = ExtensibilityPointFactory.GetTraitDiscoverer(diagnosticMessageSink, discovererAttribute);
                    if (discoverer != null)
                    {
                        foreach (var keyValuePair in discoverer.GetTraits(traitAttribute))
                        {
                            Traits.Add(keyValuePair.Key, keyValuePair.Value);
                        }
                    }
                }
                else
                {
                    diagnosticMessageSink.OnMessage(new DiagnosticMessage("Trait attribute on '{0}' did not have [TraitDiscoverer]", DisplayName));
                }
            }
        }
예제 #33
0
        /// <summary>
        /// Gets the test collection definitions for the given assembly.
        /// </summary>
        /// <param name="assemblyInfo">The assembly.</param>
        /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param>
        /// <returns>A list of mappings from test collection name to test collection definitions (as <see cref="ITypeInfo"/></returns>
        public static Dictionary<string, ITypeInfo> GetTestCollectionDefinitions(IAssemblyInfo assemblyInfo, IMessageSink diagnosticMessageSink)
        {
            var attributeTypesByName =
                assemblyInfo.GetTypes(false)
                            .Select(type => new { Type = type, Attribute = type.GetCustomAttributes(typeof(CollectionDefinitionAttribute).AssemblyQualifiedName).FirstOrDefault() })
                            .Where(list => list.Attribute != null)
                            .GroupBy(list => (string)list.Attribute.GetConstructorArguments().Single(),
                                     list => list.Type,
                                     StringComparer.OrdinalIgnoreCase);

            var result = new Dictionary<string, ITypeInfo>();

            foreach (var grouping in attributeTypesByName)
            {
                var types = grouping.ToList();
                result[grouping.Key] = types[0];

                if (types.Count > 1)
                    diagnosticMessageSink.OnMessage(new DiagnosticMessage("Multiple test collections declared with name '{0}': {1}",
                                                                          grouping.Key,
                                                                          string.Join(", ", types.Select(type => type.Name))));
            }

            return result;
        }
 public ConditionalTestFramework(IMessageSink messageSink)
     : base(messageSink)
 {
     messageSink.OnMessage(new DiagnosticMessage
     {
         Message = "Using " + nameof(ConditionalTestFramework)
     });
 }
예제 #35
0
        /// <inheritdoc />
        protected override bool RunTestsOnMethod(IMessageSink messageSink,
                                                 Type classUnderTest,
                                                 object[] constructorArguments,
                                                 MethodInfo methodUnderTest,
                                                 List<BeforeAfterTestAttribute> beforeAfterAttributes,
                                                 ExceptionAggregator aggregator,
                                                 ref decimal executionTime)
        {
            try
            {
                var testMethod = Reflector.Wrap(methodUnderTest);

                var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute));
                foreach (var dataAttribute in dataAttributes)
                {
                    var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
                    var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList();
                    var discovererType = Reflector.GetType(args[0], args[1]);
                    IDataDiscoverer discoverer = (IDataDiscoverer)Activator.CreateInstance(discovererType);

                    foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod))
                        if (RunTestWithArguments(messageSink, classUnderTest, constructorArguments, methodUnderTest, dataRow, GetDisplayNameWithArguments(DisplayName, dataRow), beforeAfterAttributes, aggregator, ref executionTime))
                            return true;
                }

                return false;
            }
            catch (Exception ex)
            {
                var cancelled = false;

                if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = DisplayName }))
                    cancelled = true;
                else
                {
                    if (!messageSink.OnMessage(new TestFailed(ex.Unwrap()) { TestCase = this, TestDisplayName = DisplayName }))
                        cancelled = true;
                }

                if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = DisplayName }))
                    cancelled = true;

                return cancelled;
            }
        }
예제 #36
0
        /// <inheritdoc />
        protected override void RunTestsOnMethod(IMessageSink messageSink,
                                                 Type classUnderTest,
                                                 object[] constructorArguments,
                                                 MethodInfo methodUnderTest,
                                                 List<BeforeAfterTestAttribute> beforeAfterAttributes,
                                                 ExceptionAggregator aggregator,
                                                 CancellationTokenSource cancellationTokenSource,
                                                 ref decimal executionTime)
        {
            try
            {
                var testMethod = Reflector.Wrap(methodUnderTest);

                var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute));
                foreach (var dataAttribute in dataAttributes)
                {
                    var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
                    var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList();
                    var discovererType = Reflector.GetType(args[1], args[0]);
                    IDataDiscoverer discoverer = (IDataDiscoverer)Activator.CreateInstance(discovererType);

                    foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod))
                    {
                        RunTestWithArguments(messageSink, classUnderTest, constructorArguments, methodUnderTest, dataRow, GetDisplayNameWithArguments(DisplayName, dataRow), beforeAfterAttributes, aggregator, cancellationTokenSource, ref executionTime);
                        if (cancellationTokenSource.IsCancellationRequested)
                            return;
                    }
                }
            }
            catch (Exception ex)
            {
                if (!messageSink.OnMessage(new TestStarting(this, DisplayName)))
                    cancellationTokenSource.Cancel();
                else
                {
                    if (!messageSink.OnMessage(new TestFailed(this, DisplayName, executionTime, ex.Unwrap())))
                        cancellationTokenSource.Cancel();
                }

                if (!messageSink.OnMessage(new TestFinished(this, DisplayName, executionTime)))
                    cancellationTokenSource.Cancel();
            }
        }
예제 #37
0
        /// <inheritdoc/>
        public void Find(bool includeSourceInformation, IMessageSink messageSink)
        {
            Guard.ArgumentNotNull("messageSink", messageSink);

            ThreadPool.QueueUserWorkItem(_ =>
            {
                foreach (var type in assemblyInfo.GetTypes(includePrivateTypes: false))
                    if (!FindImpl(type, includeSourceInformation, messageSink))
                        break;

                messageSink.OnMessage(new DiscoveryCompleteMessage());
            });
        }
예제 #38
0
        /// <inheritdoc/>
        protected override void RunTests(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
        {
            if (!messageSink.OnMessage(new TestStarting(this, DisplayName)))
                cancellationTokenSource.Cancel();
            else
            {
                try
                {
                    lambda();

                    if (!messageSink.OnMessage(new TestPassed(this, DisplayName, 0)))
                        cancellationTokenSource.Cancel();
                }
                catch (Exception ex)
                {
                    if (!messageSink.OnMessage(new TestFailed(this, DisplayName, 0, ex)))
                        cancellationTokenSource.Cancel();
                }
            }

            if (!messageSink.OnMessage(new TestFinished(this, DisplayName, 0)))
                cancellationTokenSource.Cancel();
        }
예제 #39
0
        /// <inheritdoc/>
        public void Find(string typeName, bool includeSourceInformation, IMessageSink messageSink)
        {
            Guard.ArgumentNotNullOrEmpty("typeName", typeName);
            Guard.ArgumentNotNull("messageSink", messageSink);

            ThreadPool.QueueUserWorkItem(_ =>
            {
                ITypeInfo typeInfo = assemblyInfo.GetType(typeName);
                if (typeInfo != null)
                    FindImpl(typeInfo, includeSourceInformation, messageSink);

                messageSink.OnMessage(new DiscoveryCompleteMessage());
            });
        }
예제 #40
0
        /// <inheritdoc/>
        public async void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink)
        {
            var cancellationTokenSource = new CancellationTokenSource();
            var totalSummary = new RunSummary();

            string currentDirectory = Directory.GetCurrentDirectory();

            try
            {
                Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));

                if (messageSink.OnMessage(new TestAssemblyStarting(assemblyFileName, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, DateTime.Now,
                                                                   String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version),
                                                                   XunitTestFrameworkDiscoverer.DisplayName)))
                {
                    // TODO: Contract for Run() states that null "testCases" means "run everything".

                    var tasks =
                        testCases.Cast<XunitTestCase>()
                                 .GroupBy(tc => tc.TestCollection)
                                 .Select(collectionGroup => Task.Run(() => RunTestCollection(messageSink, collectionGroup.Key, collectionGroup, cancellationTokenSource)))
                                 .ToArray();

                    var summaries = await Task.WhenAll(tasks);
                    totalSummary.Time = summaries.Sum(s => s.Time);
                    totalSummary.Total = summaries.Sum(s => s.Total);
                    totalSummary.Failed = summaries.Sum(s => s.Failed);
                    totalSummary.Skipped = summaries.Sum(s => s.Skipped);
                }

                messageSink.OnMessage(new TestAssemblyFinished(assemblyInfo, totalSummary.Time, totalSummary.Total, totalSummary.Failed, totalSummary.Skipped));
            }
            finally
            {
                Directory.SetCurrentDirectory(currentDirectory);
            }
        }
예제 #41
0
        static ITestFramework CreateInnerTestFramework(Type testFrameworkType, IMessageSink diagnosticMessageSink)
        {
            try
            {
                var ctorWithSink = testFrameworkType.GetTypeInfo().DeclaredConstructors
                                                                  .FirstOrDefault(ctor =>
                                                                  {
                                                                      var paramInfos = ctor.GetParameters();
                                                                      return paramInfos.Length == 1 && paramInfos[0].ParameterType == typeof(IMessageSink);
                                                                  });
                if (ctorWithSink != null)
                    return (ITestFramework)ctorWithSink.Invoke(new object[] { diagnosticMessageSink });

                return (ITestFramework)Activator.CreateInstance(testFrameworkType);
            }
            catch (Exception ex)
            {
                diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework construction: {ex.Unwrap()}"));
                return new XunitTestFramework(diagnosticMessageSink);
            }
        }
예제 #42
0
        // This method is called by the xUnit test framework classes to run the test case. We will do the
        // loop here, forwarding on to the implementation in XunitTestCase to do the heavy lifting. We will
        // continue to re-run the test until the aggregator has an error (meaning that some internal error
        // condition happened), or the test runs without failure, or we've hit the maximum number of tries.
        public override async Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
                                                        IMessageBus messageBus,
                                                        object[] constructorArguments,
                                                        ExceptionAggregator aggregator,
                                                        CancellationTokenSource cancellationTokenSource)
        {
            var runCount = 0;

            while (true)
            {
                // This is really the only tricky bit: we need to capture and delay messages (since those will
                // contain run status) until we know we've decided to accept the final result;
                var delayedMessageBus = new DelayedMessageBus(messageBus);

                var summary = await base.RunAsync(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource);
                if (aggregator.HasExceptions || summary.Failed == 0 || ++runCount >= maxRetries)
                {
                    delayedMessageBus.Dispose();  // Sends all the delayed messages
                    return summary;
                }

                diagnosticMessageSink.OnMessage(new DiagnosticMessage("Execution of '{0}' failed (attempt #{1}), retrying...", DisplayName, runCount));
            }
        }
예제 #43
0
파일: Xunit1.cs 프로젝트: kamchung322/xunit
        RunSummary RunTestClass(ITestCollection testCollection, string className, IList<Xunit1TestCase> testCases, IMessageSink messageSink)
        {
            var handler = new TestClassCallbackHandler(testCases, messageSink);
            var results = handler.TestClassResults;
            results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className));

            if (results.Continue)
            {
                try
                {
                    var methodNames = testCases.Select(tc => tc.Method.Name).ToList();
                    executor.RunTests(className, methodNames, handler);
                    handler.LastNodeArrived.WaitOne();
                }
                catch (Exception ex)
                {
                    var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex);

                    results.Continue = messageSink.OnMessage(new ErrorMessage(failureInformation.ExceptionTypes, failureInformation.Messages, failureInformation.StackTraces, failureInformation.ExceptionParentIndices)) && results.Continue;
                }
            }

            results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
            return results;
        }
예제 #44
0
파일: xunit.cs 프로젝트: roczj/xunit
        public override bool Execute()
        {
            RemotingUtility.CleanUpRegisteredChannels();

            if (TeamCity)
            {
                Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead.");
                return false;
            }
            if (Verbose)
            {
                Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead.");
                return false;
            }

            XElement assembliesElement = null;
            var environment = string.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version);

            if (NeedsXml)
                assembliesElement = new XElement("assemblies");

            switch (MaxParallelThreads)
            {
                case null:
                case "default":
                    break;

                case "unlimited":
                    maxThreadCount = 0;
                    break;

                default:
                    int threadValue;
                    if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 0)
                    {
                        Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
                        return false;
                    }

                    maxThreadCount = threadValue;
                    break;
            }

            var originalWorkingFolder = Directory.GetCurrentDirectory();

            using (AssemblyHelper.SubscribeResolve())
            {
                var reporters = GetAvailableRunnerReporters();
                var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled);

                if (reporter == null && !string.IsNullOrWhiteSpace(Reporter))
                {
                    reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase));
                    if (reporter == null)
                    {
                        var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList();
                        if (switchableReporters.Count == 0)
                            Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter);
                        else
                            Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters));

                        return false;
                    }
                }

                if (reporter == null)
                    reporter = new DefaultRunnerReporter();

                logger = new MSBuildLogger(Log);
                reporterMessageHandler = reporter.CreateMessageHandler(logger);

                if (!NoLogo)
                    Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment);

                var project = new XunitProject();
                foreach (var assembly in Assemblies)
                {
                    var assemblyFileName = assembly.GetMetadata("FullPath");
                    var configFileName = assembly.GetMetadata("ConfigFile");
                    if (configFileName != null && configFileName.Length == 0)
                        configFileName = null;

                    project.Add(new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName, ShadowCopy = ShadowCopy });
                }

                if (WorkingFolder != null)
                    Directory.SetCurrentDirectory(WorkingFolder);

                var clockTime = Stopwatch.StartNew();

                if (!parallelizeAssemblies.HasValue)
                    parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);

                if (parallelizeAssemblies.GetValueOrDefault())
                {
                    var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly)));
                    var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
                    foreach (var assemblyElement in results.Where(result => result != null))
                        assembliesElement.Add(assemblyElement);
                }
                else
                {
                    foreach (var assembly in project.Assemblies)
                    {
                        var assemblyElement = ExecuteAssembly(assembly);
                        if (assemblyElement != null)
                            assembliesElement.Add(assemblyElement);
                    }
                }

                clockTime.Stop();

                if (completionMessages.Count > 0)
                    reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
            }

            Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);

            if (NeedsXml)
            {
                if (Xml != null)
                    assembliesElement.Save(Xml.GetMetadata("FullPath"));

                if (XmlV1 != null)
                    Transform("xUnit1.xslt", assembliesElement, XmlV1);

                if (Html != null)
                    Transform("HTML.xslt", assembliesElement, Html);

                if (NUnit != null)
                    Transform("NUnitXml.xslt", assembliesElement, NUnit);
            }

            return ExitCode == 0;
        }
예제 #45
0
        /// <inheritdoc/>
        public void Find(bool includeSourceInformation, IMessageSink messageSink)
        {
            Guard.ArgumentNotNull("messageSink", messageSink);

            ThreadPool.QueueUserWorkItem(_ =>
            {
                foreach (var type in assemblyInfo.GetTypes(includePrivateTypes: false))
                    if (!FindImpl(type, includeSourceInformation, messageSink))
                        break;

                var warnings = messageAggregator.GetAndClear<EnvironmentalWarning>().Select(w => w.Message).ToList();
                messageSink.OnMessage(new DiscoveryCompleteMessage(warnings));
            });
        }
예제 #46
0
        /// <summary>
        /// Runs a single test for a given test method.
        /// </summary>
        /// <param name="messageSink">The message sink to send results to.</param>
        /// <param name="classUnderTest">The class under test.</param>
        /// <param name="constructorArguments">The arguments to pass to the constructor.</param>
        /// <param name="methodUnderTest">The method under test.</param>
        /// <param name="testMethodArguments">The arguments to pass to the test method.</param>
        /// <param name="displayName">The display name for the test.</param>
        /// <param name="beforeAfterAttributes">The <see cref="BeforeAfterTestAttribute"/> instances attached to the test.</param>
        /// <param name="aggregator">The error aggregator to use for catching exception.</param>
        /// <param name="executionTime">The time spent executing the tests.</param>
        protected bool RunTestWithArguments(IMessageSink messageSink,
                                            Type classUnderTest,
                                            object[] constructorArguments,
                                            MethodInfo methodUnderTest,
                                            object[] testMethodArguments,
                                            string displayName,
                                            List<BeforeAfterTestAttribute> beforeAfterAttributes,
                                            ExceptionAggregator aggregator,
                                            ref decimal executionTime)
        {
            bool cancelled = false;

            if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = displayName }))
                cancelled = true;
            else
            {
                if (!String.IsNullOrEmpty(SkipReason))
                {
                    if (!messageSink.OnMessage(new TestSkipped { TestCase = this, TestDisplayName = DisplayName, Reason = SkipReason }))
                        cancelled = true;
                }
                else
                {
                    var beforeAttributesRun = new Stack<BeforeAfterTestAttribute>();
                    var stopwatch = Stopwatch.StartNew();

                    if (!aggregator.HasExceptions)
                    {
                        aggregator.Run(() =>
                        {
                            object testClass = null;

                            if (!methodUnderTest.IsStatic)
                            {
                                if (!messageSink.OnMessage(new TestClassConstructionStarting { TestCase = this, TestDisplayName = displayName }))
                                    cancelled = true;

                                try
                                {
                                    if (!cancelled)
                                        testClass = Activator.CreateInstance(classUnderTest, constructorArguments);
                                }
                                finally
                                {
                                    if (!messageSink.OnMessage(new TestClassConstructionFinished { TestCase = this, TestDisplayName = displayName }))
                                        cancelled = true;
                                }
                            }

                            if (!cancelled)
                            {
                                aggregator.Run(() =>
                                {
                                    foreach (var beforeAfterAttribute in beforeAfterAttributes)
                                    {
                                        if (!messageSink.OnMessage(new BeforeTestStarting { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name }))
                                            cancelled = true;
                                        else
                                        {
                                            try
                                            {
                                                beforeAfterAttribute.Before(methodUnderTest);
                                                beforeAttributesRun.Push(beforeAfterAttribute);
                                            }
                                            finally
                                            {
                                                if (!messageSink.OnMessage(new BeforeTestFinished { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name }))
                                                    cancelled = true;
                                            }
                                        }

                                        if (cancelled)
                                            return;
                                    }

                                    if (!cancelled)
                                    {
                                        var parameterTypes = methodUnderTest.GetParameters().Select(p => p.ParameterType).ToArray();
                                        aggregator.Run(() =>
                                        {
                                            var result = methodUnderTest.Invoke(testClass, ConvertArguments(testMethodArguments ?? EmptyArray, parameterTypes));
                                            var task = result as Task;
                                            if (task != null)
                                                task.GetAwaiter().GetResult();
                                        });
                                    }
                                });

                                foreach (var beforeAfterAttribute in beforeAttributesRun)
                                {
                                    if (!messageSink.OnMessage(new AfterTestStarting { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name }))
                                        cancelled = true;

                                    aggregator.Run(() => beforeAfterAttribute.After(methodUnderTest));

                                    if (!messageSink.OnMessage(new AfterTestFinished { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name }))
                                        cancelled = true;
                                }
                            }

                            aggregator.Run(() =>
                            {
                                IDisposable disposable = testClass as IDisposable;
                                if (disposable != null)
                                {
                                    if (!messageSink.OnMessage(new TestClassDisposeStarting { TestCase = this, TestDisplayName = displayName }))
                                        cancelled = true;

                                    try
                                    {
                                        disposable.Dispose();
                                    }
                                    finally
                                    {
                                        if (!messageSink.OnMessage(new TestClassDisposeFinished { TestCase = this, TestDisplayName = displayName }))
                                            cancelled = true;
                                    }
                                }
                            });
                        });
                    }

                    stopwatch.Stop();

                    if (!cancelled)
                    {
                        executionTime = (decimal)stopwatch.Elapsed.TotalSeconds;

                        var exception = aggregator.ToException();
                        var testResult = exception == null ? (TestResultMessage)new TestPassed() : new TestFailed(exception);
                        testResult.TestCase = this;
                        testResult.TestDisplayName = displayName;
                        testResult.ExecutionTime = executionTime;

                        if (!messageSink.OnMessage(testResult))
                            cancelled = true;
                    }
                }
            }

            if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = displayName, ExecutionTime = executionTime }))
                cancelled = true;

            return cancelled;
        }
예제 #47
0
        private static bool RunTestClass(IMessageSink messageSink, IGrouping<ITypeInfo, XunitTestCase> group, RunSummary classSummary)
        {
            bool cancelled = false;
            var aggregator = new ExceptionAggregator();

            Type testClassType = ((IReflectionTypeInfo)group.Key).Type;
            Dictionary<Type, object> fixtureMappings = new Dictionary<Type, object>();
            List<object> constructorArguments = new List<object>();

            // TODO: Read class fixtures from test collection
            foreach (var iface in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
            {
                Type fixtureType = iface.GetGenericArguments().Single();
                object fixture = null;
                aggregator.Run(() => fixture = Activator.CreateInstance(fixtureType));
                fixtureMappings.Add(fixtureType, fixture);
            }

            var ctors = testClassType.GetConstructors();
            if (ctors.Length != 1)
            {
                aggregator.Add(new TestClassException("A test class may only define a single public constructor."));
            }
            else
            {
                var ctor = ctors.Single();
                List<string> unusedArguments = new List<string>();

                foreach (var paramInfo in ctor.GetParameters())
                {
                    object fixture;

                    if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture))
                        constructorArguments.Add(fixture);
                    else
                        unusedArguments.Add(paramInfo.ParameterType.Name + " " + paramInfo.Name);
                }

                if (unusedArguments.Count > 0)
                    aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments)));
            }

            var methodGroups = group.GroupBy(tc => tc.Method);

            foreach (var method in methodGroups)
            {
                if (!messageSink.OnMessage(new TestMethodStarting { ClassName = group.Key.Name, MethodName = method.Key.Name }))
                    cancelled = true;
                else
                    cancelled = RunTestMethod(messageSink, constructorArguments.ToArray(), method, classSummary, aggregator);

                if (!messageSink.OnMessage(new TestMethodFinished { ClassName = group.Key.Name, MethodName = method.Key.Name }))
                    cancelled = true;

                if (cancelled)
                    break;
            }

            foreach (var fixture in fixtureMappings.Values.OfType<IDisposable>())
            {
                try
                {
                    fixture.Dispose();
                }
                catch (Exception ex)
                {
                    if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap())))
                        cancelled = true;
                }
            }

            return cancelled;
        }
예제 #48
0
        private void ReturnDiscoveryMessages(IMessageSink sink)
        {
            foreach (var testCase in DiscoveryTestCases)
                sink.OnMessage(new TestCaseDiscoveryMessage { TestCase = testCase });

            sink.OnMessage(new DiscoveryCompleteMessage());
        }
예제 #49
0
        /// <summary>
        /// Core implementation to discover unit tests in a given test class.
        /// </summary>
        /// <param name="type">The test class.</param>
        /// <param name="includeSourceInformation">Set to <c>true</c> to attempt to include source information.</param>
        /// <param name="messageSink">The message sink to send discovery messages to.</param>
        /// <returns>Returns <c>true</c> if discovery should continue; <c>false</c> otherwise.</returns>
        protected virtual bool FindImpl(ITypeInfo type, bool includeSourceInformation, IMessageSink messageSink)
        {
            string currentDirectory = Directory.GetCurrentDirectory();
            var testCollection = TestCollectionFactory.Get(type);

            try
            {
                if (!String.IsNullOrEmpty(assemblyInfo.AssemblyPath))
                    Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));

                foreach (IMethodInfo method in type.GetMethods(includePrivateMethods: true))
                {
                    IAttributeInfo factAttribute = method.GetCustomAttributes(typeof(FactAttribute)).FirstOrDefault();
                    if (factAttribute != null)
                    {
                        IAttributeInfo discovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitDiscovererAttribute)).FirstOrDefault();
                        if (discovererAttribute != null)
                        {
                            var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList();
                            var discovererType = Reflector.GetType(args[1], args[0]);
                            if (discovererType != null)
                            {
                                IXunitDiscoverer discoverer = (IXunitDiscoverer)Activator.CreateInstance(discovererType);

                                foreach (XunitTestCase testCase in discoverer.Discover(testCollection, assemblyInfo, type, method, factAttribute))
                                    if (!messageSink.OnMessage(new TestCaseDiscoveryMessage(UpdateTestCaseWithSourceInfo(testCase, includeSourceInformation))))
                                        return false;
                            }
                            // TODO: Figure out a way to report back an error when discovererType is not available
                            // TODO: What if the discovererType can't be created or cast to IXunitDiscoverer?
                            // TODO: Performance optimization: cache instances of the discoverer type
                        }
                    }
                }

                return true;
            }
            finally
            {
                Directory.SetCurrentDirectory(currentDirectory);
            }
        }
예제 #50
0
파일: Xunit1.cs 프로젝트: JayBazuzi/xunit
        void Find(Predicate<ITestCaseDiscoveryMessage> filter, bool includeSourceInformation, IMessageSink messageSink)
        {
            try
            {
                XmlNode assemblyXml = null;

                using (var handler = new XmlNodeCallbackHandler(xml => { assemblyXml = xml; return true; }))
                    executor.EnumerateTests(handler);

                foreach (XmlNode method in assemblyXml.SelectNodes("//method"))
                {
                    var testCase = method.ToTestCase(assemblyFileName);
                    if (testCase != null)
                    {
                        if (includeSourceInformation)
                            testCase.SourceInformation = sourceInformationProvider.GetSourceInformation(testCase);

                        testCase.TestCollection = testCollection;

                        var message = new TestCaseDiscoveryMessage(testCase);
                        if (filter(message))
                            messageSink.OnMessage(message);
                    }
                }
            }
            finally
            {
                messageSink.OnMessage(new DiscoveryCompleteMessage(new string[0]));
            }
        }
예제 #51
0
        private RunSummary RunTestCollection(IMessageSink messageSink, ITestCollection collection, IEnumerable<XunitTestCase> testCases, CancellationTokenSource cancellationTokenSource)
        {
            var collectionSummary = new RunSummary();
            var collectionFixtureMappings = new Dictionary<Type, object>();
            var aggregator = new ExceptionAggregator();

            if (collection.CollectionDefinition != null)
            {
                var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type;
                foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture<>)))
                    CreateFixture(interfaceType, aggregator, collectionFixtureMappings);
            }

            if (messageSink.OnMessage(new TestCollectionStarting(collection)))
            {
                foreach (var testCasesByClass in testCases.GroupBy(tc => tc.Class))
                {
                    var classSummary = new RunSummary();

                    if (!messageSink.OnMessage(new TestClassStarting(collection, testCasesByClass.Key.Name)))
                        cancellationTokenSource.Cancel();
                    else
                    {
                        RunTestClass(messageSink, collection, collectionFixtureMappings, (IReflectionTypeInfo)testCasesByClass.Key, testCasesByClass, classSummary, aggregator, cancellationTokenSource);
                        collectionSummary.Aggregate(classSummary);
                    }

                    if (!messageSink.OnMessage(new TestClassFinished(collection, testCasesByClass.Key.Name, classSummary.Time, classSummary.Total, classSummary.Failed, classSummary.Skipped)))
                        cancellationTokenSource.Cancel();

                    if (cancellationTokenSource.IsCancellationRequested)
                        break;
                }
            }

            foreach (var fixture in collectionFixtureMappings.Values.OfType<IDisposable>())
            {
                try
                {
                    fixture.Dispose();
                }
                catch (Exception ex)
                {
                    if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap())))
                        cancellationTokenSource.Cancel();
                }
            }

            messageSink.OnMessage(new TestCollectionFinished(collection, collectionSummary.Time, collectionSummary.Total, collectionSummary.Failed, collectionSummary.Skipped));
            return collectionSummary;
        }
예제 #52
0
 protected static bool OnMessage(IMessageSink messageSink, IMessageSinkMessage message)
 {
     var result = messageSink.OnMessage(message);
     RemotingServices.Disconnect((MarshalByRefObject)message);
     return result;
 }
예제 #53
0
        /// <summary>
        /// Runs a single test for a given test method.
        /// </summary>
        /// <param name="messageSink">The message sink to send results to.</param>
        /// <param name="classUnderTest">The class under test.</param>
        /// <param name="constructorArguments">The arguments to pass to the constructor.</param>
        /// <param name="methodUnderTest">The method under test.</param>
        /// <param name="testMethodArguments">The arguments to pass to the test method.</param>
        /// <param name="displayName">The display name for the test.</param>
        /// <param name="beforeAfterAttributes">The <see cref="BeforeAfterTestAttribute"/> instances attached to the test.</param>
        /// <param name="parentAggregator">The parent aggregator that contains the exceptions up to this point.</param>
        /// <param name="cancellationTokenSource">The cancellation token source that indicates whether cancellation has been requested.</param>
        /// <param name="executionTime">The time spent executing the tests.</param>
        protected void RunTestWithArguments(IMessageSink messageSink,
                                            Type classUnderTest,
                                            object[] constructorArguments,
                                            MethodInfo methodUnderTest,
                                            object[] testMethodArguments,
                                            string displayName,
                                            List<BeforeAfterTestAttribute> beforeAfterAttributes,
                                            ExceptionAggregator parentAggregator,
                                            CancellationTokenSource cancellationTokenSource,
                                            ref decimal executionTime)
        {
            var aggregator = new ExceptionAggregator(parentAggregator);

            if (!messageSink.OnMessage(new TestStarting(this, displayName)))
                cancellationTokenSource.Cancel();
            else
            {
                if (!String.IsNullOrEmpty(SkipReason))
                {
                    if (!messageSink.OnMessage(new TestSkipped(this, displayName, SkipReason)))
                        cancellationTokenSource.Cancel();
                }
                else
                {
                    var beforeAttributesRun = new Stack<BeforeAfterTestAttribute>();
                    var stopwatch = Stopwatch.StartNew();

                    if (!aggregator.HasExceptions)
                        aggregator.Run(() =>
                        {
                            object testClass = null;

                            if (!methodUnderTest.IsStatic)
                            {
                                if (!messageSink.OnMessage(new TestClassConstructionStarting(this, displayName)))
                                    cancellationTokenSource.Cancel();

                                try
                                {
                                    if (!cancellationTokenSource.IsCancellationRequested)
                                        testClass = Activator.CreateInstance(classUnderTest, constructorArguments);
                                }
                                finally
                                {
                                    if (!messageSink.OnMessage(new TestClassConstructionFinished(this, displayName)))
                                        cancellationTokenSource.Cancel();
                                }
                            }

                            if (!cancellationTokenSource.IsCancellationRequested)
                            {
                                aggregator.Run(() =>
                                {
                                    foreach (var beforeAfterAttribute in beforeAfterAttributes)
                                    {
                                        var attributeName = beforeAfterAttribute.GetType().Name;
                                        if (!messageSink.OnMessage(new BeforeTestStarting(this, displayName, attributeName)))
                                            cancellationTokenSource.Cancel();
                                        else
                                        {
                                            try
                                            {
                                                beforeAfterAttribute.Before(methodUnderTest);
                                                beforeAttributesRun.Push(beforeAfterAttribute);
                                            }
                                            finally
                                            {
                                                if (!messageSink.OnMessage(new BeforeTestFinished(this, displayName, attributeName)))
                                                    cancellationTokenSource.Cancel();
                                            }
                                        }

                                        if (cancellationTokenSource.IsCancellationRequested)
                                            return;
                                    }

                                    if (!cancellationTokenSource.IsCancellationRequested)
                                    {
                                        var parameterTypes = methodUnderTest.GetParameters().Select(p => p.ParameterType).ToArray();
                                        var oldSyncContext = SynchronizationContext.Current;

                                        try
                                        {
                                            var asyncSyncContext = new AsyncTestSyncContext();
                                            SetSynchronizationContext(asyncSyncContext);

                                            aggregator.Run(() =>
                                            {
                                                var result = methodUnderTest.Invoke(testClass, ConvertArguments(testMethodArguments ?? EmptyArray, parameterTypes));
                                                var task = result as Task;
                                                if (task != null)
                                                    task.GetAwaiter().GetResult();
                                                else
                                                {
                                                    var ex = asyncSyncContext.WaitForCompletion();
                                                    if (ex != null)
                                                        aggregator.Add(ex);
                                                }
                                            });
                                        }
                                        finally
                                        {
                                            SetSynchronizationContext(oldSyncContext);
                                        }
                                    }
                                });

                                foreach (var beforeAfterAttribute in beforeAttributesRun)
                                {
                                    var attributeName = beforeAfterAttribute.GetType().Name;
                                    if (!messageSink.OnMessage(new AfterTestStarting(this, displayName, attributeName)))
                                        cancellationTokenSource.Cancel();

                                    aggregator.Run(() => beforeAfterAttribute.After(methodUnderTest));

                                    if (!messageSink.OnMessage(new AfterTestFinished(this, displayName, attributeName)))
                                        cancellationTokenSource.Cancel();
                                }
                            }

                            aggregator.Run(() =>
                            {
                                IDisposable disposable = testClass as IDisposable;
                                if (disposable != null)
                                {
                                    if (!messageSink.OnMessage(new TestClassDisposeStarting(this, displayName)))
                                        cancellationTokenSource.Cancel();

                                    try
                                    {
                                        disposable.Dispose();
                                    }
                                    finally
                                    {
                                        if (!messageSink.OnMessage(new TestClassDisposeFinished(this, displayName)))
                                            cancellationTokenSource.Cancel();
                                    }
                                }
                            });
                        });

                    stopwatch.Stop();

                    if (!cancellationTokenSource.IsCancellationRequested)
                    {
                        executionTime = (decimal)stopwatch.Elapsed.TotalSeconds;

                        var exception = aggregator.ToException();
                        var testResult = exception == null ? (TestResultMessage)new TestPassed(this, displayName, executionTime) : new TestFailed(this, displayName, executionTime, exception);
                        if (!messageSink.OnMessage(testResult))
                            cancellationTokenSource.Cancel();
                    }
                }
            }

            if (!messageSink.OnMessage(new TestFinished(this, displayName, executionTime)))
                cancellationTokenSource.Cancel();
        }
예제 #54
0
        /// <summary>
        /// Executes the test case, returning 0 or more result messages through the message sink.
        /// </summary>
        /// <param name="messageSink">The message sink to report results to.</param>
        /// <param name="constructorArguments">The arguments to pass to the constructor.</param>
        /// <param name="aggregator">The error aggregator to use for catching exception.</param>
        /// <param name="cancellationTokenSource">The cancellation token source that indicates whether cancellation has been requested.</param>
        public virtual void Run(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
        {
            int totalFailed = 0;
            int totalRun = 0;
            int totalSkipped = 0;
            decimal executionTime = 0M;

            if (!messageSink.OnMessage(new TestCaseStarting(this)))
                cancellationTokenSource.Cancel();
            else
            {
                var delegatingSink = new DelegatingMessageSink(messageSink, msg =>
                {
                    if (msg is ITestResultMessage)
                    {
                        totalRun++;
                        executionTime += ((ITestResultMessage)msg).ExecutionTime;
                    }
                    if (msg is ITestFailed)
                        totalFailed++;
                    if (msg is ITestSkipped)
                        totalSkipped++;
                });

                RunTests(delegatingSink, constructorArguments, aggregator, cancellationTokenSource);
            }

            if (!messageSink.OnMessage(new TestCaseFinished(this, executionTime, totalRun, totalFailed, totalSkipped)))
                cancellationTokenSource.Cancel();
        }
예제 #55
0
        static Type GetTestFrameworkType(IAssemblyInfo testAssembly, IMessageSink diagnosticMessageSink)
        {
            try
            {
                var testFrameworkAttr = testAssembly.GetCustomAttributes(typeof(ITestFrameworkAttribute)).FirstOrDefault();
                if (testFrameworkAttr != null)
                {
                    var discovererAttr = testFrameworkAttr.GetCustomAttributes(typeof(TestFrameworkDiscovererAttribute)).FirstOrDefault();
                    if (discovererAttr != null)
                    {
                        var discoverer = ExtensibilityPointFactory.GetTestFrameworkTypeDiscoverer(diagnosticMessageSink, discovererAttr);
                        if (discoverer != null)
                            return discoverer.GetTestFrameworkType(testFrameworkAttr);

                        var ctorArgs = discovererAttr.GetConstructorArguments().ToArray();
                        diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Unable to create custom test framework discoverer type '{ctorArgs[1]}, {ctorArgs[0]}'"));
                    }
                    else
                    {
                        diagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test framework attribute was not decorated with [TestFrameworkDiscoverer]"));
                    }
                }
            }
            catch (Exception ex)
            {
                diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework discoverer construction: {ex.Unwrap()}"));
            }

            return typeof(XunitTestFramework);
        }
예제 #56
0
파일: Xunit1.cs 프로젝트: JayBazuzi/xunit
        RunSummary RunTestCollection(ITestCollection testCollection, IEnumerable<Xunit1TestCase> testCases, IMessageSink messageSink)
        {
            var results = new RunSummary();
            results.Continue = messageSink.OnMessage(new TestCollectionStarting(testCollection));

            if (results.Continue)
                foreach (var testClassGroup in testCases.GroupBy(tc => tc.Class.Name))
                {
                    var classResults = RunTestClass(testCollection, testClassGroup.Key, testClassGroup.ToList(), messageSink);
                    results.Aggregate(classResults);
                    if (!classResults.Continue)
                        break;
                }

            results.Continue = messageSink.OnMessage(new TestCollectionFinished(testCollection, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
            return results;
        }
예제 #57
0
파일: Xunit1.cs 프로젝트: JayBazuzi/xunit
        RunSummary RunTestClass(ITestCollection testCollection, string className, IList<Xunit1TestCase> testCases, IMessageSink messageSink)
        {
            var handler = new TestClassCallbackHandler(testCases, messageSink);
            var results = handler.TestClassResults;
            results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className));

            if (results.Continue)
            {
                try
                {
                    var methodNames = testCases.Select(tc => tc.Method.Name).ToList();
                    executor.RunTests(className, methodNames, handler);
                    handler.LastNodeArrived.WaitOne();
                }
                catch (Exception ex)
                {
                    var stackTrace = ex.StackTrace;
                    var rethrowIndex = stackTrace.IndexOf("$$RethrowMarker$$");
                    if (rethrowIndex > -1)
                        stackTrace = stackTrace.Substring(0, rethrowIndex);

                    results.Continue = messageSink.OnMessage(new ErrorMessage(ex.GetType().FullName, ex.Message, stackTrace)) && results.Continue;
                }
            }

            results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
            return results;
        }
예제 #58
0
        /// <inheritdoc/>
        public void Find(string typeName, bool includeSourceInformation, IMessageSink messageSink)
        {
            Guard.ArgumentNotNullOrEmpty("typeName", typeName);
            Guard.ArgumentNotNull("messageSink", messageSink);

            ThreadPool.QueueUserWorkItem(_ =>
            {
                ITypeInfo typeInfo = assemblyInfo.GetType(typeName);
                if (typeInfo != null)
                    FindImpl(typeInfo, includeSourceInformation, messageSink);

                var warnings = messageAggregator.GetAndClear<EnvironmentalWarning>().Select(w => w.Message).ToList();
                messageSink.OnMessage(new DiscoveryCompleteMessage(warnings));
            });
        }
예제 #59
0
파일: Xunit1.cs 프로젝트: JayBazuzi/xunit
        /// <inheritdoc/>
        public void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink)
        {
            var results = new RunSummary();
            var environment = String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version);

            // TODO: Contract for Run() states that null "testCases" means "run everything".

            if (messageSink.OnMessage(new TestAssemblyStarting(assemblyFileName, configFileName, DateTime.Now, environment, TestFrameworkDisplayName)))
                foreach (var testCollectionGroup in testCases.Cast<Xunit1TestCase>().GroupBy(tc => tc.TestCollection))
                {
                    var collectionResults = RunTestCollection(testCollectionGroup.Key, testCollectionGroup, messageSink);
                    results.Aggregate(collectionResults);
                    if (!collectionResults.Continue)
                        break;
                }

            messageSink.OnMessage(new TestAssemblyFinished(new Xunit1AssemblyInfo(assemblyFileName), results.Time, results.Total, results.Failed, results.Skipped));
        }
예제 #60
0
        private static void RunTestClass(IMessageSink messageSink,
                                         ITestCollection collection,
                                         Dictionary<Type, object> collectionFixtureMappings,
                                         IReflectionTypeInfo testClass,
                                         IEnumerable<XunitTestCase> testCases,
                                         RunSummary classSummary,
                                         ExceptionAggregator aggregator,
                                         CancellationTokenSource cancellationTokenSource)
        {
            var testClassType = testClass.Type;
            var fixtureMappings = new Dictionary<Type, object>();
            var constructorArguments = new List<object>();

            if (testClassType.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture<>)))
                aggregator.Add(new TestClassException("A test class may not be decorated with ICollectionFixture<> (decorate the test collection class instead)."));

            foreach (var interfaceType in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
                CreateFixture(interfaceType, aggregator, fixtureMappings);

            if (collection.CollectionDefinition != null)
            {
                var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type;
                foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
                    CreateFixture(interfaceType, aggregator, fixtureMappings);
            }

            var isStaticClass = testClassType.IsAbstract && testClassType.IsSealed;
            if (!isStaticClass)
            {
                var ctors = testClassType.GetConstructors();
                if (ctors.Length != 1)
                {
                    aggregator.Add(new TestClassException("A test class may only define a single public constructor."));
                }
                else
                {
                    var ctor = ctors.Single();
                    var unusedArguments = new List<string>();

                    foreach (var paramInfo in ctor.GetParameters())
                    {
                        object fixture;

                        if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture) || collectionFixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture))
                            constructorArguments.Add(fixture);
                        else
                            unusedArguments.Add(paramInfo.ParameterType.Name + " " + paramInfo.Name);
                    }

                    if (unusedArguments.Count > 0)
                        aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments)));
                }
            }

            var methodGroups = testCases.GroupBy(tc => tc.Method);

            foreach (var method in methodGroups)
            {
                if (!messageSink.OnMessage(new TestMethodStarting(collection, testClass.Name, method.Key.Name)))
                    cancellationTokenSource.Cancel();
                else
                    RunTestMethod(messageSink, constructorArguments.ToArray(), method, classSummary, aggregator, cancellationTokenSource);

                if (!messageSink.OnMessage(new TestMethodFinished(collection, testClass.Name, method.Key.Name)))
                    cancellationTokenSource.Cancel();

                if (cancellationTokenSource.IsCancellationRequested)
                    break;
            }

            foreach (var fixture in fixtureMappings.Values.OfType<IDisposable>())
            {
                try
                {
                    fixture.Dispose();
                }
                catch (Exception ex)
                {
                    if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap())))
                        cancellationTokenSource.Cancel();
                }
            }
        }