예제 #1
0
        void DiscoverTests <TVisitor>(IEnumerable <string> sources,
                                      LoggerHelper logger,
                                      Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
                                      Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null)
            where TVisitor : IVsDiscoverySink, IDisposable
        {
            try
            {
                RemotingUtility.CleanUpRegisteredChannels();

                using (AssemblyHelper.SubscribeResolve())
                {
                    var reporterMessageHandler = new DefaultRunnerReporterWithTypes().CreateMessageHandler(new VisualStudioRunnerLogger(logger));

                    foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources)
                    {
                        var assemblyFileName = assemblyFileNameCanBeWithoutAbsolutePath;
#if !PLATFORM_DOTNET
                        assemblyFileName = Path.GetFullPath(assemblyFileNameCanBeWithoutAbsolutePath);
#endif
                        var assembly = new XunitProjectAssembly {
                            AssemblyFilename = assemblyFileName
                        };
                        var configuration = LoadConfiguration(assemblyFileName);
                        var fileName      = Path.GetFileNameWithoutExtension(assemblyFileName);
                        var shadowCopy    = configuration.ShadowCopyOrDefault;

                        try
                        {
                            if (cancelled)
                            {
                                break;
                            }

                            if (!IsXunitTestAssembly(assemblyFileName))
                            {
                                if (configuration.DiagnosticMessagesOrDefault)
                                {
                                    logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
                                }
                            }
                            else
                            {
                                var diagnosticSink = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault);

                                using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
                                {
                                    var targetFramework = framework.TargetFramework;
                                    if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
                                        targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
                                        targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
                                    {
                                        if (configuration.DiagnosticMessagesOrDefault)
                                        {
                                            logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
                                        }
                                    }
                                    else
                                    {
                                        var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);

                                        using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
                                        {
                                            reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied, shadowCopy, discoveryOptions));

                                            framework.Find(includeSourceInformation: true, discoveryMessageSink: visitor, discoveryOptions: discoveryOptions);
                                            var totalTests = visitor.Finish();

                                            visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);

                                            reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
                                        }
                                    }
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            var ex           = e.Unwrap();
                            var fileNotFound = ex as FileNotFoundException;
#if !PLATFORM_DOTNET
                            var fileLoad = ex as FileLoadException;
#endif
                            if (ex is InvalidOperationException)
                            {
                                logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
                            }
                            else if (fileNotFound != null)
                            {
                                logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
                            }
#if !PLATFORM_DOTNET
                            else if (fileLoad != null)
                            {
                                logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
                            }
#endif
                            else
                            {
                                logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
                            }
                        }
                    }
                }
            }
            catch (Exception e)
            {
                logger.LogWarning("Exception discovering tests: {0}", e.Unwrap());
            }
        }
예제 #2
0
        public void TheoryWithInlineData()
        {
            string code = @"
module FSharpTests

open Xunit

[<Theory>]
[<InlineData>]
[<InlineData(42)>]
[<InlineData(42, 21.12)>]
let TestMethod (x:int) =
    Assert.True(true)
";

            using (var assembly = FSharpAcceptanceTestV2Assembly.Create(code))
                using (var controller = new TestableXunit2(assembly.FileName, null, true))
                {
                    var sink = new TestDiscoverySink();

                    controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
                    sink.Finished.WaitOne();

                    Assert.Collection(sink.TestCases.OrderBy(tc => tc.DisplayName),
                                      testCase => Assert.Equal("FSharpTests.TestMethod(x: ???)", testCase.DisplayName),
                                      testCase => Assert.Equal("FSharpTests.TestMethod(x: 42)", testCase.DisplayName),
                                      testCase => Assert.Equal("FSharpTests.TestMethod(x: 42, ???: 21.12)", testCase.DisplayName)
                                      );
                }
        }
예제 #3
0
        public void FactAcceptanceTest()
        {
            string code = @"
using System;
using Xunit;

namespace Namespace1
{
    public class Class1
    {
        [Fact]
        [Trait(""Name!"", ""Value!"")]
        public void Trait() { }

        [Fact(Skip=""Skipping"")]
        public void Skipped() { }

        [Fact(DisplayName=""Custom Test Name"")]
        public void CustomName() { }
    }
}

namespace Namespace2
{
    public class OuterClass
    {
        public class Class2
        {
            [Fact]
            public void TestMethod() { }
        }
    }
}";

            using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code))
                using (var controller = new TestableXunit2(assembly.FileName, null, true))
                {
                    var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();

                    controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());

                    sink.Finished.WaitOne();
                    ITestCase[] testCases = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(tcdm => tcdm.TestCase).ToArray();

                    Assert.Equal(4, testCases.Length);

                    ITestCase traitTest = Assert.Single(testCases, tc => tc.DisplayName == "Namespace1.Class1.Trait");
                    string    key       = Assert.Single(traitTest.Traits.Keys);
                    Assert.Equal("Name!", key);
                    string value = Assert.Single(traitTest.Traits[key]);
                    Assert.Equal("Value!", value);

                    ITestCase skipped = Assert.Single(testCases, tc => tc.DisplayName == "Namespace1.Class1.Skipped");
                    Assert.Equal("Skipping", skipped.SkipReason);

                    Assert.Single(testCases, tc => tc.DisplayName == "Custom Test Name");
                    Assert.Single(testCases, tc => tc.DisplayName == "Namespace2.OuterClass+Class2.TestMethod");
                }
        }
        public void RoundTrip()
        {
            string code = @"
using System;
using Xunit;

namespace Namespace1
{
    public class Class1
    {
        [Fact]
        public void FactMethod() { }

        [Theory]
        [InlineData(42)]
        public void TheoryMethod(int x) { }
    }
}";

            using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code))
            {
                var serializations   = default(List <string>);
                var testCollectionId = default(Guid);

                using (var serializationController = new TestableXunitFrontController(assembly.FileName))
                {
                    var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
                    serializationController.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
                    sink.Finished.WaitOne();

                    var testCases = sink.Messages.OfType <ITestCaseDiscoveryMessage>().OrderBy(tcdm => tcdm.TestCase.TestMethod.Method.Name).Select(tcdm => tcdm.TestCase).ToList();
                    testCollectionId = testCases[0].TestMethod.TestClass.TestCollection.UniqueID;
                    var descriptors = serializationController.GetTestCaseDescriptors(testCases, true);
                    serializations = descriptors.Select(d => d.Serialization).ToList();
                }

                Assert.Collection(serializations,
                                  s => Assert.Equal($":F:Namespace1.Class1:FactMethod:1:0:{testCollectionId.ToString("N")}", s),
                                  s => Assert.StartsWith("Xunit.Sdk.XunitTestCase, xunit.execution.{Platform}:", s)
                                  );

                using (var deserializationController = new TestableXunitFrontController(assembly.FileName))
                {
                    var deserializations = deserializationController.BulkDeserialize(serializations);

                    Assert.Collection(deserializations.Select(kvp => kvp.Value),
                                      testCase => Assert.Equal("Namespace1.Class1.FactMethod", testCase.DisplayName),
                                      testCase => Assert.Equal("Namespace1.Class1.TheoryMethod(x: 42)", testCase.DisplayName)
                                      );
                }
            }
        }
예제 #5
0
        public void TheoryWithInlineData()
        {
            string code = @"
using System;
using Xunit;

public class TestClass
{
    [Theory]
    [InlineData]
    [InlineData(42)]
    [InlineData(42, 21.12)]
    public void TestMethod(int x) { }
}";

            using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code))
                using (var controller = new TestableXunit2(assembly.FileName, null, true))
                {
                    var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();

                    controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());

                    sink.Finished.WaitOne();
                    string[] testCaseNames = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(tcdm => tcdm.TestCase.DisplayName).ToArray();

                    Assert.Equal(3, testCaseNames.Length);

                    Assert.Contains("TestClass.TestMethod(x: ???)", testCaseNames);
                    Assert.Contains("TestClass.TestMethod(x: 42)", testCaseNames);
                    Assert.Contains($"TestClass.TestMethod(x: 42, ???: {21.12})", testCaseNames);
                }
        }
예제 #6
0
        XElement ExecuteAssembly(object consoleLock,
                                 XunitProjectAssembly assembly,
                                 bool needsXml,
                                 bool?parallelizeTestCollections,
                                 int?maxThreadCount,
                                 bool diagnosticMessages,
                                 bool noColor,
                                 XunitFilters filters,
                                 bool designTime,
                                 bool listTestCases,
                                 IReadOnlyList <string> designTimeFullyQualifiedNames)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                // Turn off pre-enumeration of theories when we're not running in Visual Studio
                if (!designTime)
                {
                    assembly.Configuration.PreEnumerateTheories = false;
                }

                if (diagnosticMessages)
                {
                    assembly.Configuration.DiagnosticMessages = true;
                }

                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                if (maxThreadCount.HasValue)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
                }

                var assemblyDisplayName       = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageVisitor  = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
                var sourceInformationProvider = new SourceInformationProviderAdapater(services);

                using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, false, diagnosticMessageSink: diagnosticMessageVisitor, sourceInformationProvider: sourceInformationProvider))
                    using (var discoveryVisitor = new TestDiscoveryVisitor())
                    {
                        var includeSourceInformation = designTime && listTestCases;

                        // Discover & filter the tests
                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));

                        controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
                        discoveryVisitor.Finished.WaitOne();

                        IDictionary <ITestCase, VsTestCase> vsTestCases = null;
                        if (designTime)
                        {
                            vsTestCases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases);
                        }

                        if (listTestCases)
                        {
                            lock (consoleLock)
                            {
                                if (designTime)
                                {
                                    var sink = (ITestDiscoverySink)services.GetService(typeof(ITestDiscoverySink));

                                    foreach (var testcase in vsTestCases.Values)
                                    {
                                        if (sink != null)
                                        {
                                            sink.SendTest(testcase);
                                        }

                                        Console.WriteLine(testcase.FullyQualifiedName);
                                    }
                                }
                                else
                                {
                                    foreach (var testcase in discoveryVisitor.TestCases)
                                    {
                                        Console.WriteLine(testcase.DisplayName);
                                    }
                                }
                            }

                            return(assemblyElement);
                        }

                        IExecutionVisitor resultsVisitor;

                        if (designTime)
                        {
                            var sink = (ITestExecutionSink)services.GetService(typeof(ITestExecutionSink));
                            resultsVisitor = new DesignTimeExecutionVisitor(sink, vsTestCases, reporterMessageHandler);
                        }
                        else
                        {
                            resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
                        }

                        IList <ITestCase> filteredTestCases;
                        var testCasesDiscovered = discoveryVisitor.TestCases.Count;
                        if (!designTime || designTimeFullyQualifiedNames.Count == 0)
                        {
                            filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
                        }
                        else
                        {
                            filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList();
                        }
                        var testCasesToRun = filteredTestCases.Count;

                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                        if (filteredTestCases.Count == 0)
                        {
                            completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                        }
                        else
                        {
                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                            controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
                            resultsVisitor.Finished.WaitOne();

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
                        }
                    }
            }
            catch (Exception ex)
            {
                failed = true;

                var e = ex;
                while (e != null)
                {
                    Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
                    e = e.InnerException;
                }
            }

            return(assemblyElement);
        }
예제 #7
0
        XElement ExecuteAssembly(object consoleLock,
                                 XunitProjectAssembly assembly,
                                 bool serialize,
                                 bool needsXml,
                                 bool?parallelizeTestCollections,
                                 int?maxThreadCount,
                                 bool diagnosticMessages,
                                 bool noColor,
                                 bool noAppDomain,
                                 bool failSkips,
                                 bool stopOnFail,
                                 XunitFilters filters,
                                 bool internalDiagnosticMessages)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
                {
                    return(null);
                }

                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories        = false;
                assembly.Configuration.DiagnosticMessages         |= diagnosticMessages;
                assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;

                if (noAppDomain)
                {
                    assembly.Configuration.AppDomain = AppDomainSupport.Denied;
                }

                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                executionOptions.SetStopOnTestFail(stopOnFail);
                if (maxThreadCount.HasValue)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
                }

                var assemblyDisplayName            = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageSink          = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
                var internalDiagnosticsMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor, ConsoleColor.DarkGray);
                var appDomainSupport   = assembly.Configuration.AppDomainOrDefault;
                var shadowCopy         = assembly.Configuration.ShadowCopyOrDefault;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

#if NETCOREAPP1_0 || NETCOREAPP2_0
                using (new NetCoreAssemblyDependencyResolver(assembly.AssemblyFilename, internalDiagnosticsMessageSink))
#endif
                using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                    using (var discoverySink = new TestDiscoverySink(() => cancel))
                    {
                        // Discover & filter the tests
                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));

                        controller.Find(false, discoverySink, discoveryOptions);
                        discoverySink.Finished.WaitOne();

                        var testCasesDiscovered = discoverySink.TestCases.Count;
                        var filteredTestCases   = discoverySink.TestCases.Where(filters.Filter).ToList();
                        var testCasesToRun      = filteredTestCases.Count;

                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                        // Run the filtered tests
                        if (testCasesToRun == 0)
                        {
                            completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                        }
                        else
                        {
                            if (serialize)
                            {
                                filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
                            }

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                            IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
                            if (assemblyElement != null)
                            {
                                resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                            }
                            if (longRunningSeconds > 0)
                            {
                                resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
                            }
                            if (failSkips)
                            {
                                resultsSink = new DelegatingFailSkipSink(resultsSink);
                            }

                            controller.RunTests(filteredTestCases, resultsSink, executionOptions);
                            resultsSink.Finished.WaitOne();

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                            if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
                            {
                                Console.WriteLine("Canceling due to test failure...");
                                cancel = true;
                            }
                        }
                    }
            }
            catch (Exception ex)
            {
                failed = true;

                var e = ex;
                while (e != null)
                {
                    Console.WriteLine($"{e.GetType().FullName}: {e.Message}");

                    if (internalDiagnosticMessages)
                    {
                        Console.WriteLine(e.StackTrace);
                    }

                    e = e.InnerException;
                }
            }

            return(assemblyElement);
        }
예제 #8
0
        /// <summary>
        /// MultiNodeTestRunner takes the following <see cref="args"/>:
        ///
        /// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}]
        ///
        /// <list type="number">
        /// <listheader>
        ///     <term>Argument</term>
        ///     <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description>
        /// </listheader>
        /// <item>
        ///     <term>AssemblyName</term>
        ///     <description>
        ///         The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory.
        ///
        ///         i.e. "Akka.Cluster.Tests.MultiNode.dll"
        ///              "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll"
        ///     </description>
        /// </item>
        /// <item>
        ///     <term>-Dmultinode.enable-filesink</term>
        ///     <description>Having this flag set means that the contents of this test run will be saved in the
        ///                 current working directory as a .JSON file.
        ///     </description>
        /// </item>
        /// <item>
        ///     <term>-Dmultinode.multinode.output-directory</term>
        ///     <description>Setting this flag means that any persistent multi-node test runner output files
        ///                  will be written to this directory instead of the default, which is the same folder
        ///                  as the test binary.
        ///     </description>
        /// </item>
        /// </list>
        /// </summary>
        static void Main(string[] args)
        {
            TestRunSystem   = ActorSystem.Create("TestRunnerLogging");
            SinkCoordinator = TestRunSystem.ActorOf(Props.Create <SinkCoordinator>(), "sinkCoordinator");

            var assemblyName = Path.GetFullPath(args[0]);

            EnableAllSinks(assemblyName);
            PublishRunnerMessage(String.Format("Running MultiNodeTests for {0}", assemblyName));

            using (var controller = new XunitFrontController(assemblyName))
            {
                using (var discovery = new Discovery())
                {
                    controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
                    discovery.Finished.WaitOne();

                    foreach (var test in discovery.Tests.Reverse())
                    {
                        if (!string.IsNullOrEmpty(test.Value.First().SkipReason))
                        {
                            PublishRunnerMessage(string.Format("Skipping test {0}. Reason - {1}", test.Value.First().MethodName, test.Value.First().SkipReason));
                            continue;
                        }

                        PublishRunnerMessage(string.Format("Starting test {0}", test.Value.First().MethodName));

                        var processes = new List <Process>();

                        StartNewSpec(test.Value);

                        foreach (var nodeTest in test.Value)
                        {
                            //Loop through each test, work out number of nodes to run on and kick off process
                            var process = new Process();
                            processes.Add(process);
                            process.StartInfo.UseShellExecute        = false;
                            process.StartInfo.RedirectStandardOutput = true;
                            process.StartInfo.FileName  = "Akka.NodeTestRunner.exe";
                            process.StartInfo.Arguments = String.Format(@"-Dmultinode.test-assembly=""{0}"" -Dmultinode.test-class=""{1}"" -Dmultinode.test-method=""{2}"" -Dmultinode.max-nodes={3} -Dmultinode.server-host=""{4}"" -Dmultinode.host=""{5}"" -Dmultinode.index={6}",
                                                                        assemblyName, nodeTest.TypeName, nodeTest.MethodName, test.Value.Count, "localhost", "localhost", nodeTest.Node - 1);
                            var nodeIndex = nodeTest.Node;
                            process.OutputDataReceived +=
                                (sender, line) =>
                            {
                                //ignore any trailing whitespace
                                if (string.IsNullOrEmpty(line.Data) || string.IsNullOrWhiteSpace(line.Data))
                                {
                                    return;
                                }
                                string message = line.Data;
                                if (!message.StartsWith("[NODE", true, CultureInfo.InvariantCulture))
                                {
                                    message = "[NODE" + nodeIndex + "]" + message;
                                }
                                PublishToAllSinks(message);
                            };

                            var closureTest = nodeTest;
                            process.Exited += (sender, eventArgs) =>
                            {
                                if (process.ExitCode == 0)
                                {
                                    ReportSpecPassFromExitCode(nodeIndex, closureTest.TestName);
                                }
                            };
                            process.Start();

                            process.BeginOutputReadLine();
                            PublishRunnerMessage(string.Format("Started node {0} on pid {1}", nodeTest.Node, process.Id));
                        }

                        foreach (var process in processes)
                        {
                            process.WaitForExit();
                            var exitCode = process.ExitCode;
                            process.Close();
                        }

                        PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected.");
                        Thread.Sleep(TimeSpan.FromSeconds(3));
                        FinishSpec();
                    }
                }
            }
            Console.WriteLine("Complete");
            PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected.");
            Thread.Sleep(TimeSpan.FromSeconds(5));
            CloseAllSinks();

            //Block until all Sinks have been terminated.
            TestRunSystem.AwaitTermination(TimeSpan.FromMinutes(1));

            //Return the proper exit code
            Environment.Exit(ExitCodeContainer.ExitCode);
        }
예제 #9
0
        static XElement ExecuteAssembly(object consoleLock,
                                        XunitProjectAssembly assembly,
                                        bool serialize,
                                        bool needsXml,
                                        bool?parallelizeTestCollections,
                                        int?maxThreadCount,
                                        bool diagnosticMessages,
                                        bool noColor,
                                        bool?noAppDomain,
                                        XunitFilters filters)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
                {
                    return(null);
                }

                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories = false;
                assembly.Configuration.DiagnosticMessages  |= diagnosticMessages;

                if (noAppDomain.HasValue)
                {
                    assembly.Configuration.UseAppDomain = !noAppDomain.GetValueOrDefault();
                }

                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                if (maxThreadCount.HasValue)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
                }

                var assemblyDisplayName      = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
                var useAppDomain             = assembly.Configuration.UseAppDomainOrDefault;

                using (var controller = new XunitFrontController(useAppDomain, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
                    using (var discoveryVisitor = new TestDiscoveryVisitor())
                    {
                        // Discover & filter the tests
                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, useAppDomain, discoveryOptions));

                        controller.Find(false, discoveryVisitor, discoveryOptions);
                        discoveryVisitor.Finished.WaitOne();

                        var testCasesDiscovered = discoveryVisitor.TestCases.Count;
                        var filteredTestCases   = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
                        var testCasesToRun      = filteredTestCases.Count;

                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                        // Run the filtered tests
                        if (testCasesToRun == 0)
                        {
                            completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                        }
                        else
                        {
                            if (serialize)
                            {
                                filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
                            }

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                            var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
                            controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
                            resultsVisitor.Finished.WaitOne();

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
                        }
                    }
            }
            catch (Exception ex)
            {
                failed = true;

                var e = ex;
                while (e != null)
                {
                    Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
                    e = e.InnerException;
                }
            }

            return(assemblyElement);
        }
예제 #10
0
파일: Program.cs 프로젝트: wfurt/buildtools
        static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool?parallelizeTestCollections, int?maxThreadCount, ExtendedXunitFilters filters)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
                {
                    return(null);
                }

                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories = false;

                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                if (maxThreadCount.HasValue)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
                }

                lock (consoleLock)
                {
                    if (assembly.Configuration.DiagnosticMessages ?? false)
                    {
                        Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
                                          Path.GetFileNameWithoutExtension(assembly.AssemblyFilename),
                                          discoveryOptions.GetMethodDisplay(),
                                          !executionOptions.GetDisableParallelization(),
                                          executionOptions.GetMaxParallelThreads());
                    }
                    else
                    {
                        Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
                    }
                }

                using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
                    using (var discoveryVisitor = new TestDiscoveryVisitor())
                    {
                        controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
                        discoveryVisitor.Finished.WaitOne();

                        lock (consoleLock)
                        {
                            Console.WriteLine("Discovered:  {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
                        }

                        var resultsVisitor    = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress);
                        var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
                        if (filteredTestCases.Count == 0)
                        {
                            lock (consoleLock)
                            {
                                Console.ForegroundColor = ConsoleColor.DarkYellow;
                                Console.WriteLine("Info:        {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
                                Console.ForegroundColor = ConsoleColor.Gray;
                            }
                        }
                        else
                        {
                            controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
                            resultsVisitor.Finished.WaitOne();
                        }
                    }
            }
            catch (Exception ex)
            {
                Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
                failed = true;
            }

            return(assemblyElement);
        }
예제 #11
0
        public static void Main(string[] args)
        {
            OutputDirectory = CommandLine.GetPropertyOrDefault("multinode.output-directory", string.Empty);
            TestRunSystem   = ActorSystem.Create("TestRunnerLogging");

            var teamCityFormattingOn = CommandLine.GetPropertyOrDefault("multinode.teamcity", "false");

            if (!Boolean.TryParse(teamCityFormattingOn, out TeamCityFormattingOn))
            {
                throw new ArgumentException("Invalid argument provided for -Dteamcity");
            }

            SinkCoordinator = TestRunSystem.ActorOf( // mutes ConsoleMessageSinkActor
                Props.Create <SinkCoordinator>(), "sinkCoordinator");

            var listenAddress  = IPAddress.Parse(CommandLine.GetPropertyOrDefault("multinode.listen-address", "127.0.0.1"));
            var listenPort     = CommandLine.GetInt32OrDefault("multinode.listen-port", 6577);
            var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
            var specName       = CommandLine.GetPropertyOrDefault("multinode.spec", "");
            var platform       = CommandLine.GetPropertyOrDefault("multinode.platform", "netcore");


            if (!_validNetCorePlatform.Contains(platform))
            {
                throw new Exception($"Target platform not supported: {platform}. Supported platforms are net and netcore");
            }

            var tcpLogger = TestRunSystem.ActorOf(Props.Create(() => new TcpLoggingServer(SinkCoordinator)), "TcpLogger");

            TestRunSystem.Tcp().Tell(new Tcp.Bind(tcpLogger, listenEndpoint));

            var assemblyPath = Path.GetFullPath(args[0].Trim('"')); //unquote the string first

            EnableAllSinks(assemblyPath, platform);
            PublishRunnerMessage($"Running MultiNodeTests for {assemblyPath}");

            // In NetCore, if the assembly file hasn't been touched,
            // XunitFrontController would fail loading external assemblies and its dependencies.
            var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath);
            var asms     = assembly.GetReferencedAssemblies();
            var basePath = Path.GetDirectoryName(assemblyPath);

            foreach (var asm in asms)
            {
                try
                {
                    Assembly.Load(new AssemblyName(asm.FullName));
                }
                catch (Exception)
                {
                    var path = Path.Combine(basePath, asm.Name + ".dll");
                    try
                    {
                        AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
                    }
                    catch (Exception e)
                    {
                        Console.Out.WriteLine($"Failed to load dll: {path}, {e}");
                    }
                }
            }


            using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath))
            {
                using (var discovery = new Discovery())
                {
                    controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
                    discovery.Finished.WaitOne();

                    if (discovery.WasSuccessful)
                    {
                        foreach (var test in discovery.Tests.Reverse())
                        {
                            if (!string.IsNullOrEmpty(test.Value.First().SkipReason))
                            {
                                PublishRunnerMessage($"Skipping test {test.Value.First().MethodName}. Reason - {test.Value.First().SkipReason}");
                                continue;
                            }

                            if (!string.IsNullOrWhiteSpace(specName) &&
                                CultureInfo.InvariantCulture.CompareInfo.IndexOf(test.Value.First().TestName,
                                                                                 specName,
                                                                                 CompareOptions.IgnoreCase) < 0)
                            {
                                PublishRunnerMessage($"Skipping [{test.Value.First().MethodName}] (Filtering)");
                                continue;
                            }

                            var processes = new List <Process>();

                            PublishRunnerMessage($"Starting test {test.Value.First().MethodName}");
                            Console.Out.WriteLine($"Starting test {test.Value.First().MethodName}");

                            StartNewSpec(test.Value);

                            var ntrNetPath     = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.exe");
                            var ntrNetCorePath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.dll");

                            foreach (var nodeTest in test.Value)
                            {
                                //Loop through each test, work out number of nodes to run on and kick off process
                                var sbArguments = new StringBuilder()
                                                  //.Append($@"-Dmultinode.test-assembly=""{assemblyPath}"" ")
                                                  .Append($@"-Dmultinode.test-class=""{nodeTest.TypeName}"" ")
                                                  .Append($@"-Dmultinode.test-method=""{nodeTest.MethodName}"" ")
                                                  .Append($@"-Dmultinode.max-nodes={test.Value.Count} ")
                                                  .Append($@"-Dmultinode.server-host=""{"localhost"}"" ")
                                                  .Append($@"-Dmultinode.host=""{"localhost"}"" ")
                                                  .Append($@"-Dmultinode.index={nodeTest.Node - 1} ")
                                                  .Append($@"-Dmultinode.role=""{nodeTest.Role}"" ")
                                                  .Append($@"-Dmultinode.listen-address={listenAddress} ")
                                                  .Append($@"-Dmultinode.listen-port={listenPort} ");


                                string fileName = null;
                                switch (platform)
                                {
                                case "net":
                                    fileName = ntrNetPath;
                                    sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
                                    break;

                                case "netcore":
                                    fileName = "dotnet";
                                    sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
                                    sbArguments.Insert(0, ntrNetCorePath);
                                    break;
                                }
                                var process = new Process
                                {
                                    StartInfo = new ProcessStartInfo

                                    {
                                        FileName = fileName ?? throw new NullException(nameof(fileName)),
                                                         UseShellExecute        = false,
                                                         RedirectStandardOutput = true,
                                                         Arguments        = sbArguments.ToString(),
                                                         WorkingDirectory = Path.GetDirectoryName(assemblyPath)
                                    }
                                };

                                processes.Add(process);
                                var nodeIndex = nodeTest.Node;
                                var nodeRole  = nodeTest.Role;


                                if (platform == "netcore")
                                {
                                    process.StartInfo.FileName         = "dotnet";
                                    process.StartInfo.Arguments        = ntrNetCorePath + " " + process.StartInfo.Arguments;
                                    process.StartInfo.WorkingDirectory = Path.GetDirectoryName(assemblyPath);
                                }


                                //TODO: might need to do some validation here to avoid the 260 character max path error on Windows
                                var folder      = Directory.CreateDirectory(Path.Combine(OutputDirectory, nodeTest.TestName));
                                var logFilePath = Path.Combine(folder.FullName, $"node{nodeIndex}__{nodeRole}__{platform}.txt");
                                var fileActor   = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath)));
                                process.OutputDataReceived += (sender, eventArgs) =>
                                {
                                    if (eventArgs?.Data != null)
                                    {
                                        fileActor.Tell(eventArgs.Data);
                                        if (TeamCityFormattingOn)
                                        {
                                            // teamCityTest.WriteStdOutput(eventArgs.Data); TODO: open flood gates
                                        }
                                    }
                                };
                                var closureTest = nodeTest;
                                process.Exited += (sender, eventArgs) =>
                                {
                                    if (process.ExitCode == 0)
                                    {
                                        ReportSpecPassFromExitCode(nodeIndex, nodeRole,
                                                                   closureTest.TestName);
                                    }
                                };

                                process.Start();
                                process.BeginOutputReadLine();
                                PublishRunnerMessage($"Started node {nodeIndex} : {nodeRole} on pid {process.Id}");
                            }

                            foreach (var process in processes)
                            {
                                process.WaitForExit();
                                process.Dispose();
                            }

                            PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected.");
                            Thread.Sleep(TimeSpan.FromSeconds(3));
                            FinishSpec(test.Value);
                        }
                        Console.WriteLine("Complete");
                        PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected.");
                        Thread.Sleep(TimeSpan.FromSeconds(5));
                    }
                    else
                    {
                        var sb = new StringBuilder();
                        sb.AppendLine("One or more exception was thrown while discovering test cases. Test Aborted.");
                        foreach (var err in discovery.Errors)
                        {
                            for (int i = 0; i < err.ExceptionTypes.Length; ++i)
                            {
                                sb.AppendLine();
                                sb.Append($"{err.ExceptionTypes[i]}: {err.Messages[i]}");
                                sb.Append(err.StackTraces[i]);
                            }
                        }
                        PublishRunnerMessage(sb.ToString());
                        Console.Out.WriteLine(sb.ToString());
                    }
                }
            }

            CloseAllSinks();

            //Block until all Sinks have been terminated.
            TestRunSystem.WhenTerminated.Wait(TimeSpan.FromMinutes(1));

            if (Debugger.IsAttached)
            {
                Console.ReadLine(); //block when debugging
            }
            //Return the proper exit code
            Environment.Exit(ExitCodeContainer.ExitCode);
        }
        Task ITestDiscoverer.AnalyzeDocumentAsync(DocumentTestDiscoveryContext context)
        {
            if (context.CancellationToken.IsCancellationRequested)
            {
                return(Task.FromCanceled(context.CancellationToken));
            }

            using (var discoverySink = new TestDiscoveryVisitor())
                using (var xunit2 = new Xunit2Discoverer(
                           AppDomainSupport.Denied,
                           sourceInformationProvider: DummySourceInformationProvider.Instance,
                           assemblyInfo: new SourceAssemblyInfo(context),
                           xunitExecutionAssemblyPath: XunitExecutionAssemblyPath.Value,
                           verifyAssembliesOnDisk: false))
                {
                    xunit2.Find(includeSourceInformation: false, messageSink: discoverySink, discoveryOptions: TestFrameworkOptions.ForDiscovery());

                    while (!discoverySink.Finished.WaitOne(50))
                    {
                        if (context.CancellationToken.IsCancellationRequested)
                        {
                            break;
                        }

                        TrySendDiscoveredTestCases(context, discoverySink);
                    }

                    TrySendDiscoveredTestCases(context, discoverySink);
                }

            return(context.CancellationToken.IsCancellationRequested ?
                   Task.FromCanceled(context.CancellationToken) :
                   Task.CompletedTask);
        }
예제 #13
0
        static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool serialize, bool needsXml, bool teamCity, bool appVeyor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = needsXml ? new XElement("assembly") : null;

            try
            {
                if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
                {
                    return(null);
                }

                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                if (maxThreadCount.HasValue)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
                }

                var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);

                lock (consoleLock)
                {
                    if (assembly.Configuration.DiagnosticMessagesOrDefault)
                    {
                        Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
                                          assemblyDisplayName,
                                          discoveryOptions.GetMethodDisplayOrDefault(),
                                          !executionOptions.GetDisableParallelizationOrDefault(),
                                          executionOptions.GetMaxParallelThreadsOrDefault());
                    }
                    else if (!quiet)
                    {
                        Console.WriteLine("Discovering: {0}", assemblyDisplayName);
                    }
                }

                var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);

                using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
                    using (var discoveryVisitor = new TestDiscoveryVisitor())
                    {
                        controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
                        discoveryVisitor.Finished.WaitOne();

                        if (!quiet)
                        {
                            lock (consoleLock)
                                Console.WriteLine("Discovered:  {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
                        }

                        var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
                        if (filteredTestCases.Count == 0)
                        {
                            completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                        }
                        else
                        {
                            if (serialize)
                            {
                                filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
                            }

                            var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor);
                            controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
                            resultsVisitor.Finished.WaitOne();
                        }
                    }
            }
            catch (Exception ex)
            {
                failed = true;

                var e = ex;
                while (e != null)
                {
                    Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
                    e = e.InnerException;
                }
            }

            return(assemblyElement);
        }
예제 #14
0
        private void ExecuteTests(
            [NotNull] XunitProjectAssembly assembly,
            [NotNull] TestRunOptions options,
            [NotNull] XunitFilters filters,
            [NotNull] XunitFrontController controller,
            [NotNull] TestDiscoverySink discoverySink,
            [NotNull] ITestFrameworkExecutionOptions executionOptions,
            [CanBeNull] XElement assemblyElement,
            [NotNull] DiagnosticMessageSink diagnosticMessageSink)
        {
            var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
            var shadowCopy       = assembly.Configuration.ShadowCopyOrDefault;
            var serialize        = options.Serialize;
            var failSkips        = options.FailSkips;
            var stopOnFail       = options.StopOnFail;

            var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

            var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);

            // Discover & filter the tests
            _reporterMessageHandler.OnMessage(
                new TestAssemblyDiscoveryStarting(
                    assembly,
                    controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
                    shadowCopy,
                    discoveryOptions));

            controller.Find(false, discoverySink, discoveryOptions);
            discoverySink.Finished.WaitOne();

            var testCasesDiscovered = discoverySink.TestCases.Count;
            var filteredTestCases   = discoverySink.TestCases.Where(filters.Filter).ToList();
            var testCasesToRun      = filteredTestCases.Count;

            _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

            // Run the filtered tests
            if (testCasesToRun == 0)
            {
                _completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary());
            }
            else
            {
                if (serialize)
                {
                    filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
                }

                _reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                IExecutionSink resultsSink = new DelegatingExecutionSummarySink(
                    _reporterMessageHandler,
                    () => _cancel,
                    (path, summary) => _completionMessages.TryAdd(path, summary));
                if (assemblyElement != null)
                {
                    resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
                }

                if (longRunningSeconds > 0)
                {
                    resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
                }

                if (failSkips)
                {
                    resultsSink = new DelegatingFailSkipSink(resultsSink);
                }

                controller.RunTests(filteredTestCases, resultsSink, executionOptions);
                resultsSink.Finished.WaitOne();

                _reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));

                if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0)
                {
                    return;
                }

                Console.WriteLine("Canceling due to test failure...");
                _cancel = true;
            }
        }
예제 #15
0
    public List <IMessageSinkMessage> Run(Type[] types)
    {
        Xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), types[0].Assembly.GetLocalCodeBase(), configFileName: null, shadowCopy: true);

        var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>();

        foreach (var type in types)
        {
            Xunit2.Find(type.FullName, includeSourceInformation: false, messageSink: discoverySink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
            discoverySink.Finished.WaitOne();
            discoverySink.Finished.Reset();
        }

        var testCases = discoverySink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(msg => msg.TestCase).ToArray();

        var runSink = new SpyMessageSink <ITestAssemblyFinished>();

        Xunit2.RunTests(testCases, runSink, TestFrameworkOptions.ForExecution());
        runSink.Finished.WaitOne();

        return(runSink.Messages.ToList());
    }
        public EtwPerformanceMetricLogger(XunitPerformanceProject project, Program program)
        {
            _etlPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".etl");
            _program = program;
            _project = project;

            var diagnosticMessageSink = new ConsoleReporter();

            foreach (var assembly in project.Assemblies)
            {
                program.PrintIfVerbose($"Discovering tests for {assembly.AssemblyFilename}.");

                // Note: We do not use shadowCopy because that creates a new AppDomain which can cause
                // assembly load failures with delay-signed or "fake signed" assemblies.
                using (var controller = new XunitFrontController(
                           assemblyFileName: assembly.AssemblyFilename,
                           shadowCopy: false,
                           appDomainSupport: AppDomainSupport.Denied,
                           diagnosticMessageSink: new ConsoleDiagnosticsMessageVisitor())
                       )
                    using (var discoveryVisitor = new PerformanceTestDiscoveryVisitor(assembly, project.Filters, diagnosticMessageSink))
                    {
                        controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: TestFrameworkOptions.ForDiscovery());
                        discoveryVisitor.Finished.WaitOne();
                        _tests.AddRange(discoveryVisitor.Tests);
                    }
            }

            program.PrintIfVerbose($"Discovered a total of {_tests.Count} tests.");
        }
예제 #17
0
    public static int Main(string[] args)
    {
        var asm = typeof(SingleFileTestRunner).Assembly;

        Console.WriteLine("Running assembly:" + asm.FullName);

        var diagnosticSink = new ConsoleDiagnosticMessageSink();
        var testsFinished  = new TaskCompletionSource();
        var testSink       = new TestMessageSink();
        var summarySink    = new DelegatingExecutionSummarySink(testSink,
                                                                () => false,
                                                                (completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"));
        var resultsXmlAssembly = new XElement("assembly");
        var resultsSink        = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);

        testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
        testSink.Execution.TestFailedEvent  += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); };

        testSink.Execution.TestAssemblyFinishedEvent += args =>
        {
            Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}");
            testsFinished.SetResult();
        };

        var assemblyConfig = new TestAssemblyConfiguration()
        {
            // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
            PreEnumerateTheories = false,
        };

        var xunitTestFx = new SingleFileTestRunner(diagnosticSink);
        var asmInfo     = Reflector.Wrap(asm);
        var asmName     = asm.GetName();

        var discoverySink = new TestDiscoverySink();
        var discoverer    = xunitTestFx.CreateDiscoverer(asmInfo);

        discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery(assemblyConfig));
        discoverySink.Finished.WaitOne();

        string       xmlResultFileName = null;
        XunitFilters filters           = new XunitFilters();
        // Quick hack wo much validation to get args that are passed (notrait, xml)
        Dictionary <string, List <string> > noTraits = new Dictionary <string, List <string> >();

        for (int i = 0; i < args.Length; i++)
        {
            if (args[i].Equals("-notrait", StringComparison.OrdinalIgnoreCase))
            {
                var traitKeyValue = args[i + 1].Split("=", StringSplitOptions.TrimEntries);
                if (!noTraits.TryGetValue(traitKeyValue[0], out List <string> values))
                {
                    noTraits.Add(traitKeyValue[0], values = new List <string>());
                }
                values.Add(traitKeyValue[1]);
            }
            if (args[i].Equals("-xml", StringComparison.OrdinalIgnoreCase))
            {
                xmlResultFileName = args[i + 1].Trim();
            }
        }

        foreach (KeyValuePair <string, List <string> > kvp in noTraits)
        {
            filters.ExcludedTraits.Add(kvp.Key, kvp.Value);
        }

        var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
        var executor          = xunitTestFx.CreateExecutor(asmName);

        executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution(assemblyConfig));

        resultsSink.Finished.WaitOne();

        // Helix need to see results file in the drive to detect if the test has failed or not
        if (xmlResultFileName != null)
        {
            resultsXmlAssembly.Save(xmlResultFileName);
        }

        var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;

        return(failed ? 1 : 0);
    }
예제 #18
0
        public RealDiscovery()
        {
            var sourceInformationProvider = new NullSourceInformationProvider();
            var diagnosticMessageSink     = new Xunit.NullMessageSink();
            var assembly     = typeof(TestCaseBulkDeserializerTests).Assembly;
            var assemblyInfo = Reflector.Wrap(assembly);

            discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceInformationProvider, diagnosticMessageSink);

            var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>();

            discoverer.Find("TestCaseDescriptorFactoryTests+TestClass", false, discoverySink, TestFrameworkOptions.ForDiscovery());
            discoverySink.Finished.WaitOne();

            testCases = discoverySink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(m => m.TestCase).ToList();
        }
예제 #19
0
        private bool DiscoverTestsInSource <TVisitor>(XunitFrontController framework,
                                                      LoggerHelper logger,
                                                      TestPlatformContext testPlatformContext,
                                                      RunSettings runSettings,
                                                      Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
                                                      Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete,
                                                      string assemblyFileName,
                                                      bool shadowCopy,
                                                      TestAssemblyConfiguration configuration)
            where TVisitor : IVsDiscoverySink, IDisposable
        {
            if (cancelled)
            {
                return(false);
            }

            var fileName = "(unknown assembly)";

            try
            {
                var reporterMessageHandler = GetRunnerReporter(logger, runSettings, new[] { assemblyFileName }).CreateMessageHandler(new VisualStudioRunnerLogger(logger));
                var assembly = new XunitProjectAssembly {
                    AssemblyFilename = assemblyFileName
                };
                fileName = Path.GetFileNameWithoutExtension(assemblyFileName);

                if (!IsXunitTestAssembly(assemblyFileName))
                {
                    if (configuration.DiagnosticMessagesOrDefault)
                    {
                        logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
                    }
                }
                else
                {
                    var targetFramework = framework.TargetFramework;
                    if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
                        targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
                        targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
                    {
                        if (configuration.DiagnosticMessagesOrDefault)
                        {
                            logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
                        }
                    }
                    else
                    {
                        var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);

                        using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
                        {
                            var totalTests      = 0;
                            var usingAppDomains = framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied;
                            reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, usingAppDomains, shadowCopy, discoveryOptions));

                            try
                            {
                                framework.Find(testPlatformContext.RequireSourceInformation, visitor, discoveryOptions);

                                totalTests = visitor.Finish();

                                visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);
                            }
                            finally
                            {
                                reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
                            }
                        }
                    }
                }
            }
            catch (Exception e)
            {
                var ex = e.Unwrap();

                if (ex is InvalidOperationException)
                {
                    logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
                }
                else if (ex is FileNotFoundException fileNotFound)
                {
                    logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
                }
#if !WINDOWS_UAP
                else if (ex is FileLoadException fileLoad)
                {
                    logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
                }
#endif
                else
                {
                    logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
                }
            }

            return(true);
        }
예제 #20
0
        public static void GuardClauses()
        {
            var framework = TestableXunitTestFrameworkDiscoverer.Create();

            Assert.Throws <ArgumentNullException>("discoveryMessageSink", () => framework.Find(includeSourceInformation: false, discoveryMessageSink: null, discoveryOptions: TestFrameworkOptions.ForDiscovery()));
            Assert.Throws <ArgumentNullException>("discoveryOptions", () => framework.Find(includeSourceInformation: false, discoveryMessageSink: Substitute.For <IMessageSink>(), discoveryOptions: null));
        }
예제 #21
0
        static int Main(string[] args)
        {
            CommandLine.Initialize(args);

            var nodeIndex        = CommandLine.GetInt32("multinode.index");
            var nodeRole         = CommandLine.GetProperty("multinode.role");
            var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly");
            var typeName         = CommandLine.GetProperty("multinode.test-class");
            var testName         = CommandLine.GetProperty("multinode.test-method");
            var displayName      = testName;

            var listenAddress  = IPAddress.Parse(CommandLine.GetProperty("multinode.listen-address"));
            var listenPort     = CommandLine.GetInt32("multinode.listen-port");
            var listenEndpoint = new IPEndPoint(listenAddress, listenPort);

            var system    = ActorSystem.Create("NoteTestRunner-" + nodeIndex);
            var tcpClient = _logger = system.ActorOf <RunnerTcpClient>();

            system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient);

#if CORECLR
            // In NetCore, if the assembly file hasn't been touched,
            // XunitFrontController would fail loading external assemblies and its dependencies.
            AssemblyLoadContext.Default.Resolving += (assemblyLoadContext, assemblyName) => DefaultOnResolving(assemblyLoadContext, assemblyName, assemblyFileName);
            var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyFileName);
            DependencyContext.Load(assembly)
            .CompileLibraries
            .Where(dep => dep.Name.ToLower()
                   .Contains(assembly.FullName.Split(new[] { ',' })[0].ToLower()))
            .Select(dependency => AssemblyLoadContext.Default.LoadFromAssemblyName(new AssemblyName(dependency.Name)));
#endif

            Thread.Sleep(TimeSpan.FromSeconds(10));
            Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, "1");
            try
            {
                using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyFileName))
                {
                    /* need to pass in just the assembly name to Discovery, not the full path
                     * i.e. "Akka.Cluster.Tests.MultiNode.dll"
                     * not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause
                     * the Discovery class to actually not find any individual specs to run
                     */
                    var assemblyName = Path.GetFileName(assemblyFileName);
                    Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName);
                    using (var discovery = new Discovery(assemblyName, typeName))
                    {
                        using (var sink = new Sink(nodeIndex, nodeRole, tcpClient))
                        {
                            try
                            {
                                controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery());
                                discovery.Finished.WaitOne();
                                controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution());
                            }
                            catch (AggregateException ex)
                            {
                                var specFail = new SpecFail(nodeIndex, nodeRole, displayName);
                                specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
                                specFail.FailureMessages.Add(ex.Message);
                                specFail.FailureStackTraces.Add(ex.StackTrace);
                                foreach (var innerEx in ex.Flatten().InnerExceptions)
                                {
                                    specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
                                    specFail.FailureMessages.Add(innerEx.Message);
                                    specFail.FailureStackTraces.Add(innerEx.StackTrace);
                                }
                                _logger.Tell(specFail.ToString());
                                Console.WriteLine(specFail);

                                //make sure message is send over the wire
                                FlushLogMessages();
                                Environment.Exit(1); //signal failure
                                return(1);
                            }
                            catch (Exception ex)
                            {
                                var specFail = new SpecFail(nodeIndex, nodeRole, displayName);
                                specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
                                specFail.FailureMessages.Add(ex.Message);
                                specFail.FailureStackTraces.Add(ex.StackTrace);
                                var innerEx = ex.InnerException;
                                while (innerEx != null)
                                {
                                    specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
                                    specFail.FailureMessages.Add(innerEx.Message);
                                    specFail.FailureStackTraces.Add(innerEx.StackTrace);
                                    innerEx = innerEx.InnerException;
                                }
                                _logger.Tell(specFail.ToString());
                                Console.WriteLine(specFail);

                                //make sure message is send over the wire
                                FlushLogMessages();
                                Environment.Exit(1); //signal failure
                                return(1);
                            }

                            var timedOut = false;
                            if (!sink.Finished.WaitOne(MaxProcessWaitTimeout)) //timed out
                            {
                                var line = string.Format("Timed out while waiting for test to complete after {0} ms",
                                                         MaxProcessWaitTimeout);
                                _logger.Tell(line);
                                Console.WriteLine(line);
                                timedOut = true;
                            }

                            FlushLogMessages();
                            system.Terminate().Wait();

                            Environment.Exit(sink.Passed && !timedOut ? 0 : 1);
                            return(sink.Passed ? 0 : 1);
                        }
                    }
                }
            }
            finally
            {
                Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, null);
            }
        }
예제 #22
0
 public void Find(string typeName, bool includeSourceInformation = false)
 {
     base.Find(typeName, includeSourceInformation, Visitor, TestFrameworkOptions.ForDiscovery());
     Visitor.Finished.WaitOne();
 }
예제 #23
0
        public void NoTestMethods()
        {
            using (var assm = CSharpAcceptanceTestV2Assembly.Create(code: ""))
                using (var controller = new TestableXunit2(assm.FileName, null, true))
                {
                    var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();

                    controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());

                    sink.Finished.WaitOne();

                    Assert.False(sink.Messages.Any(msg => msg is ITestCaseDiscoveryMessage));
                }
        }
예제 #24
0
 public virtual bool FindTestsForClass(ITestClass testClass, bool includeSourceInformation = false)
 {
     using (var messageBus = new MessageBus(Visitor))
         return(base.FindTestsForType(testClass, includeSourceInformation, messageBus, TestFrameworkOptions.ForDiscovery()));
 }
예제 #25
0
        public void FactAcceptanceTest()
        {
            string code = @"
module FSharpTests

open Xunit

[<Fact>]
[<Trait(""Name!"", ""Value!"")>]
let Trait() =
    Assert.True(true)

[<Fact(Skip = ""Skipping"")>]
let Skipped() =
    Assert.True(false)

[<Fact(DisplayName=""Custom Test Name"")>]
let CustomName() =
    Assert.True(true)
";

            using (var assembly = FSharpAcceptanceTestV2Assembly.Create(code))
                using (var controller = new TestableXunit2(assembly.FileName, null, true))
                {
                    var sink = new TestDiscoverySink();

                    controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
                    sink.Finished.WaitOne();

                    Assert.Collection(sink.TestCases.OrderBy(tc => tc.DisplayName),
                                      testCase => Assert.Equal("Custom Test Name", testCase.DisplayName),
                                      testCase =>
                    {
                        Assert.Equal("FSharpTests.Skipped", testCase.DisplayName);
                        Assert.Equal("Skipping", testCase.SkipReason);
                    },
                                      testCase =>
                    {
                        Assert.Equal("FSharpTests.Trait", testCase.DisplayName);
                        Assert.Collection(testCase.Traits,
                                          kvp =>
                        {
                            Assert.Equal("Name!", kvp.Key);
                            Assert.Equal("Value!", kvp.Value.Single());
                        }
                                          );
                    }
                                      );
                }
        }
예제 #26
0
파일: xunit.cs 프로젝트: zabulus/xunit
        protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
        {
            if (cancel)
            {
                return(null);
            }

            var assemblyElement = NeedsXml ? new XElement("assembly") : null;

            try
            {
                // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
                assembly.Configuration.PreEnumerateTheories = false;
                assembly.Configuration.DiagnosticMessages  |= DiagnosticMessages;

                if (appDomains.HasValue)
                {
                    assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
                }

                // Setup discovery and execution options with command-line overrides
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
                {
                    executionOptions.SetMaxParallelThreads(maxThreadCount);
                }
                if (parallelizeTestCollections.HasValue)
                {
                    executionOptions.SetDisableParallelization(!parallelizeTestCollections);
                }

                var assemblyDisplayName      = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
                var appDomainSupport         = assembly.Configuration.AppDomainOrDefault;
                var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;

                using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
                    using (var discoveryVisitor = new TestDiscoveryVisitor())
                    {
                        // Discover & filter the tests
                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));

                        controller.Find(false, discoveryVisitor, discoveryOptions);
                        discoveryVisitor.Finished.WaitOne();

                        var testCasesDiscovered = discoveryVisitor.TestCases.Count;
                        var filteredTestCases   = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
                        var testCasesToRun      = filteredTestCases.Count;

                        reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                        // Run the filtered tests
                        if (testCasesToRun == 0)
                        {
                            completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                        }
                        else
                        {
                            if (SerializeTestCases)
                            {
                                filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
                            }

                            IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
                            if (FailSkips)
                            {
                                resultsVisitor = new FailSkipVisitor(resultsVisitor);
                            }

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                            controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
                            resultsVisitor.Finished.WaitOne();

                            reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));

                            if (resultsVisitor.ExecutionSummary.Failed != 0)
                            {
                                ExitCode = 1;
                            }
                        }
                    }
            }
            catch (Exception ex)
            {
                var e = ex;

                while (e != null)
                {
                    Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);

                    foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
                    {
                        Log.LogError(stackLine);
                    }

                    e = e.InnerException;
                }

                ExitCode = -1;
            }

            return(assemblyElement);
        }
예제 #27
0
        public void SingleTestMethod()
        {
            string code = @"
using Xunit;

public class Foo
{
    [Fact]
    public void Bar() { }
}";

            using (var assm = CSharpAcceptanceTestV2Assembly.Create(code))
                using (var controller = new TestableXunit2(assm.FileName, null, true))
                {
                    var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();

                    controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());

                    sink.Finished.WaitOne();

                    ITestCase testCase = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Single().TestCase;
                    Assert.Equal("Foo.Bar", testCase.DisplayName);
                }
        }
예제 #28
0
        public async void RunTests(string arguments)
        {
            log = await Helpers.GetFileStreamWriterInLocalStorageAsync("stdout.txt");

            string[] args = arguments.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);

            log.WriteLine(arguments);

            var commandLine = CommandLine.Parse(args);

            if (commandLine.Debug)
            {
                Debugger.Launch();
            }
            var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
            var assembliesElement  = new XElement("assemblies");

            foreach (var assembly in commandLine.Project.Assemblies)
            {
                if (cancel)
                {
                    return;
                }
                assembly.Configuration.PreEnumerateTheories = false;
                assembly.Configuration.DiagnosticMessages  |= commandLine.DiagnosticMessages;
                assembly.Configuration.AppDomain            = AppDomainSupport.Denied;
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                executionOptions.SetDisableParallelization(true);

                try
                {
                    using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
                        using (var discoveryVisitor = new TestDiscoveryVisitor())
                        {
                            string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
                            // Discover & filter the tests
                            log.WriteLine($"Discovering: {assemblyName}");
                            xunit.Find(false, discoveryVisitor, discoveryOptions);
                            discoveryVisitor.Finished.WaitOne();

                            var testCasesDiscovered = discoveryVisitor.TestCases.Count;
                            var filteredTestCases   = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
                            var testCasesToRun      = filteredTestCases.Count;

                            log.WriteLine($"Discovered: {assemblyName}");

                            // Run the filtered tests
                            if (testCasesToRun == 0)
                            {
                                log.WriteLine($"Info:        {assemblyName} has no tests to run");
                            }
                            else
                            {
                                if (commandLine.Serialize)
                                {
                                    filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
                                }

                                var assemblyElement = new XElement("assembly");

                                StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, log, completionMessages, commandLine.ShowProgress, commandLine.FailSkips);

                                xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);

                                resultsVisitor.Finished.WaitOne();

                                assembliesElement.Add(assemblyElement);

                                log.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)}  Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}");
                            }
                        }
                }
                catch (Exception e)
                {
                    assembliesElement = new XElement("error");
                    assembliesElement.Add(e);
                    log.WriteLine(e);
                }
                finally
                {
                    await WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement);

                    log.Dispose();
                }
            }

            Application.Current.Exit();
        }
예제 #29
0
        private async void RunTests(string arguments)
        {
            var reporters = new List <IRunnerReporter>();

            string[] args = arguments.Split(new[] { '\x1F' }, StringSplitOptions.RemoveEmptyEntries);
            log  = string.Empty;
            log += "Args: " + string.Join(" ", args) + "\n";
            var commandLine = CommandLine.Parse(args);

            if (commandLine.Debug)
            {
                Debugger.Launch();
            }
            var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger());
            var completionMessages     = new ConcurrentDictionary <string, ExecutionSummary>();
            var assembliesElement      = new XElement("assemblies");

            foreach (var assembly in commandLine.Project.Assemblies)
            {
                if (cancel)
                {
                    return;
                }
                assembly.Configuration.PreEnumerateTheories = false;
                assembly.Configuration.DiagnosticMessages  |= commandLine.DiagnosticMessages;
                assembly.Configuration.AppDomain            = AppDomainSupport.Denied;
                var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
                var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
                executionOptions.SetDisableParallelization(true);

                try
                {
                    using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
                        using (var discoveryVisitor = new TestDiscoveryVisitor())
                        {
                            // Discover & filter the tests
                            reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
                            xunit.Find(false, discoveryVisitor, discoveryOptions);
                            discoveryVisitor.Finished.WaitOne();

                            var testCasesDiscovered = discoveryVisitor.TestCases.Count;
                            var filteredTestCases   = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
                            var testCasesToRun      = filteredTestCases.Count;
                            log += "testCasesToRun: " + testCasesToRun + "\n";

                            reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));

                            // Run the filtered tests
                            if (testCasesToRun == 0)
                            {
                                completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
                            }
                            else
                            {
                                if (commandLine.Serialize)
                                {
                                    filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
                                }

                                reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
                                var assemblyElement = new XElement("assembly");

                                IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
                                if (commandLine.FailSkips)
                                {
                                    resultsVisitor = new FailSkipVisitor(resultsVisitor);
                                }

                                xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);

                                log += "finished running tests \n";
                                resultsVisitor.Finished.WaitOne();

                                reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
                                assembliesElement.Add(assemblyElement);
                            }
                        }
                }
                catch (Exception e)
                {
                    assembliesElement = new XElement("error");
                    assembliesElement.Add(e);

                    log += "logged exec errors: " + e + "\n";
                }
            }
            await WriteResults(assembliesElement);
            await WriteLogs(log);

            Application.Current.Exit();
        }
예제 #30
0
        public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits)
        {
            WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''");

            Log("Starting tests...");

            var filters = new XunitFilters();

            foreach (var trait in excludedTraits ?? Array.Empty <string>())
            {
                ParseEqualSeparatedArgument(filters.ExcludedTraits, trait);
            }

            var configuration = new TestAssemblyConfiguration
            {
                ShadowCopy                 = false,
                ParallelizeAssembly        = false,
                ParallelizeTestCollections = false,
                MaxParallelThreads         = 1,
                PreEnumerateTheories       = false
            };
            var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
            var discoverySink    = new TestDiscoverySink();
            var diagnosticSink   = new ConsoleDiagnosticMessageSink();
            var testOptions      = TestFrameworkOptions.ForExecution(configuration);
            var testSink         = new TestMessageSink();
            var controller       = new Xunit2(
                AppDomainSupport.Denied,
                new NullSourceInformationProvider(),
                assemblyFileName,
                configFileName: null,
                shadowCopy: false,
                shadowCopyFolder: null,
                diagnosticMessageSink: diagnosticSink,
                verifyTestAssemblyExists: false);

            discoveryOptions.SetSynchronousMessageReporting(true);
            testOptions.SetSynchronousMessageReporting(true);

            Log($"Discovering tests for {assemblyFileName}...");
            var assembly     = Assembly.LoadFrom(assemblyFileName);
            var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly);
            var discoverer   = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);

            discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
            discoverySink.Finished.WaitOne();
            var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();

            Log($"Discovery finished.");
            Log("");

            var summarySink = new DelegatingExecutionSummarySink(
                testSink,
                () => false,
                (completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });

            var resultsXmlAssembly = new XElement("assembly");
            var resultsSink        = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);

            testSink.Execution.TestPassedEvent  += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); };
            testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); };
            testSink.Execution.TestFailedEvent  += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); };

            testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); };
            testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };

            controller.RunTests(testCasesToRun, resultsSink, testOptions);
            resultsSink.Finished.WaitOne();

            var resultsXml = new XElement("assemblies");

            resultsXml.Add(resultsXmlAssembly);

            Console.WriteLine(resultsXml.ToString());

            Log("");
            Log("Test results (Base64 encoded):");
            var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString()));

            Log(base64, id: "results");

            return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0);
        }