static void Main(string[] args) { var nodeIndex = CommandLine.GetInt32("multinode.index"); var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly"); var typeName = CommandLine.GetProperty("multinode.test-class"); var testName = CommandLine.GetProperty("multinode.test-method"); var displayName = testName; Thread.Sleep(TimeSpan.FromSeconds(10)); using (var controller = new XunitFrontController(assemblyFileName)) { /* need to pass in just the assembly name to Discovery, not the full path * i.e. "Akka.Cluster.Tests.MultiNode.dll" * not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause * the Discovery class to actually not find any indivudal specs to run */ var assemblyName = Path.GetFileName(assemblyFileName); Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName); using (var discovery = new Discovery(assemblyName, typeName)) { using (var sink = new Sink(nodeIndex)) { Thread.Sleep(10000); try { controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution()); } catch (AggregateException ex) { var specFail = new SpecFail(nodeIndex, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); foreach (var innerEx in ex.Flatten().InnerExceptions) { specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString()); specFail.FailureMessages.Add(innerEx.Message); specFail.FailureStackTraces.Add(innerEx.StackTrace); } Console.WriteLine(specFail); Environment.Exit(1); //signal failure } catch (Exception ex) { var specFail = new SpecFail(nodeIndex, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); Console.WriteLine(specFail); Environment.Exit(1); //signal failure } sink.Finished.WaitOne(); Environment.Exit(sink.Passed ? 0 : 1); } } } }
public EtwPerformanceMetricLogger(XunitPerformanceProject project, Program program) { _etlPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".etl"); _program = program; _project = project; var diagnosticMessageSink = new ConsoleReporter(); foreach (var assembly in project.Assemblies) { program.PrintIfVerbose($"Discovering tests for {assembly.AssemblyFilename}."); // Note: We do not use shadowCopy because that creates a new AppDomain which can cause // assembly load failures with delay-signed or "fake signed" assemblies. using (var controller = new XunitFrontController( assemblyFileName: assembly.AssemblyFilename, shadowCopy: false, appDomainSupport: AppDomainSupport.Denied, diagnosticMessageSink: new ConsoleDiagnosticsMessageVisitor()) ) using (var discoveryVisitor = new PerformanceTestDiscoveryVisitor(assembly, project.Filters, diagnosticMessageSink)) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: TestFrameworkOptions.ForDiscovery()); discoveryVisitor.Finished.WaitOne(); _tests.AddRange(discoveryVisitor.Tests); } } program.PrintIfVerbose($"Discovered a total of {_tests.Count} tests."); }
static void Main(string[] args) { var nodeIndex = CommandLine.GetInt32("multinode.index"); var assemblyName = CommandLine.GetProperty("multinode.test-assembly"); var typeName = CommandLine.GetProperty("multinode.test-class"); var testName = CommandLine.GetProperty("multinode.test-method"); var displayName = testName; Thread.Sleep(TimeSpan.FromSeconds(10)); using (var controller = new XunitFrontController(assemblyName)) { using (var discovery = new Discovery(assemblyName, typeName)) { using (var sink = new Sink(nodeIndex)) { Thread.Sleep(10000); try { controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution()); } catch (AggregateException ex) { var specFail = new SpecFail(nodeIndex, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); foreach (var innerEx in ex.Flatten().InnerExceptions) { specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString()); specFail.FailureMessages.Add(innerEx.Message); specFail.FailureStackTraces.Add(innerEx.StackTrace); } Console.WriteLine(specFail); Environment.Exit(1); //signal failure } catch (Exception ex) { var specFail = new SpecFail(nodeIndex, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); Console.WriteLine(specFail); Environment.Exit(1); //signal failure } sink.Finished.WaitOne(); Environment.Exit(sink.Passed ? 0 : 1); } } } }
public override Task Load(ITestSource source) { _source = source; _tcs = new TaskCompletionSource <bool>(); controller = new xu.XunitFrontController(xu.AppDomainSupport.Denied, source.Assembly.Location, diagnosticMessageSink: xu.MessageSinkAdapter.Wrap(this)); _assembly = source.Assembly; _testWrapper = new TestCollectionWrapper(_assembly); var discoveryOptions = xu.TestFrameworkOptions.ForDiscovery(); messageHandler = DiscoverMessageHandler; controller.Find(true, this, discoveryOptions); return(_tcs.Task); }
internal static void Go(string fileName, Stream stream) { using (AssemblyHelper.SubscribeResolve()) using (var xunit = new XunitFrontController( AppDomainSupport.Denied, assemblyFileName: fileName, diagnosticMessageSink: new MessageVisitor(), shadowCopy: false)) using (var writer = new ClientWriter(stream)) using (var impl = new Impl(xunit, writer)) { xunit.Find(includeSourceInformation: false, messageSink: impl, discoveryOptions: TestFrameworkOptions.ForDiscovery()); impl.Finished.WaitOne(); writer.Write(TestDataKind.EndOfData); } }
/// <summary> /// MultiNodeTestRunner takes the following <see cref="args"/>: /// /// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}] /// /// <list type="number"> /// <listheader> /// <term>Argument</term> /// <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description> /// </listheader> /// <item> /// <term>AssemblyName</term> /// <description> /// The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory. /// /// i.e. "Akka.Cluster.Tests.MultiNode.dll" /// "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll" /// </description> /// </item> /// <item> /// <term>-Dmultinode.enable-filesink</term> /// <description>Having this flag set means that the contents of this test run will be saved in the /// current working directory as a .JSON file. /// </description> /// </item> /// <item> /// <term>-Dmultinode.multinode.output-directory</term> /// <description>Setting this flag means that any persistent multi-node test runner output files /// will be written to this directory instead of the default, which is the same folder /// as the test binary. /// </description> /// </item> /// </list> /// </summary> static void Main(string[] args) { TestRunSystem = ActorSystem.Create("TestRunnerLogging"); SinkCoordinator = TestRunSystem.ActorOf(Props.Create<SinkCoordinator>(), "sinkCoordinator"); var assemblyName = Path.GetFullPath(args[0]); EnableAllSinks(assemblyName); PublishRunnerMessage(String.Format("Running MultiNodeTests for {0}", assemblyName)); using (var controller = new XunitFrontController(assemblyName)) { using (var discovery = new Discovery()) { controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); foreach (var test in discovery.Tests.Reverse()) { if (!string.IsNullOrEmpty(test.Value.First().SkipReason)) { PublishRunnerMessage(string.Format("Skipping test {0}. Reason - {1}", test.Value.First().MethodName, test.Value.First().SkipReason)); continue; } PublishRunnerMessage(string.Format("Starting test {0}", test.Value.First().MethodName)); var processes = new List<Process>(); StartNewSpec(test.Value); foreach (var nodeTest in test.Value) { //Loop through each test, work out number of nodes to run on and kick off process var process = new Process(); processes.Add(process); process.StartInfo.UseShellExecute = false; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.FileName = "Akka.NodeTestRunner.exe"; process.StartInfo.Arguments = String.Format(@"-Dmultinode.test-assembly=""{0}"" -Dmultinode.test-class=""{1}"" -Dmultinode.test-method=""{2}"" -Dmultinode.max-nodes={3} -Dmultinode.server-host=""{4}"" -Dmultinode.host=""{5}"" -Dmultinode.index={6}", assemblyName, nodeTest.TypeName, nodeTest.MethodName, test.Value.Count, "localhost", "localhost", nodeTest.Node - 1); var nodeIndex = nodeTest.Node; process.OutputDataReceived += (sender, line) => { //ignore any trailing whitespace if (string.IsNullOrEmpty(line.Data) || string.IsNullOrWhiteSpace(line.Data)) return; string message = line.Data; if (!message.StartsWith("[NODE", true, CultureInfo.InvariantCulture)) { message = "[NODE" + nodeIndex + "]" + message; } PublishToAllSinks(message); }; var closureTest = nodeTest; process.Exited += (sender, eventArgs) => { if (process.ExitCode == 0) { ReportSpecPassFromExitCode(nodeIndex, closureTest.TestName); } }; process.Start(); process.BeginOutputReadLine(); PublishRunnerMessage(string.Format("Started node {0} on pid {1}", nodeTest.Node, process.Id)); } foreach (var process in processes) { process.WaitForExit(); var exitCode = process.ExitCode; process.Close(); } PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(3)); FinishSpec(); } } } Console.WriteLine("Complete"); PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(5)); CloseAllSinks(); //Block until all Sinks have been terminated. TestRunSystem.AwaitTermination(TimeSpan.FromMinutes(1)); //Return the proper exit code Environment.Exit(ExitCodeContainer.ExitCode); }
/// <summary> /// MultiNodeTestRunner takes the following <see cref="args"/>: /// /// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}] [-Dmultinode.spec={spec name}] /// /// <list type="number"> /// <listheader> /// <term>Argument</term> /// <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description> /// </listheader> /// <item> /// <term>AssemblyName</term> /// <description> /// The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory. /// /// i.e. "Akka.Cluster.Tests.MultiNode.dll" /// "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll" /// </description> /// </item> /// <item> /// <term>-Dmultinode.enable-filesink</term> /// <description>Having this flag set means that the contents of this test run will be saved in the /// current working directory as a .JSON file. /// </description> /// </item> /// <item> /// <term>-Dmultinode.multinode.output-directory</term> /// <description>Setting this flag means that any persistent multi-node test runner output files /// will be written to this directory instead of the default, which is the same folder /// as the test binary. /// </description> /// </item> /// <item> /// <term>-Dmultinode.listen-address={ip}</term> /// <description> /// Determines the address that this multi-node test runner will use to listen for log messages from /// individual NodeTestRunner.exe processes. /// /// Defaults to 127.0.0.1 /// </description> /// </item> /// <item> /// <term>-Dmultinode.listen-port={port}</term> /// <description> /// Determines the port number that this multi-node test runner will use to listen for log messages from /// individual NodeTestRunner.exe processes. /// /// Defaults to 6577 /// </description> /// </item> /// <item> /// <term>-Dmultinode.spec={spec name}</term> /// <description> /// Setting this flag means that only tests which contains the spec name will be executed /// otherwise all tests will be executed /// </description> /// </item> /// </list> /// </summary> static void Main(string[] args) { OutputDirectory = CommandLine.GetProperty("multinode.output-directory") ?? string.Empty; TestRunSystem = ActorSystem.Create("TestRunnerLogging"); SinkCoordinator = TestRunSystem.ActorOf(Props.Create<SinkCoordinator>(), "sinkCoordinator"); var listenAddress = IPAddress.Parse(CommandLine.GetPropertyOrDefault("multinode.listen-address", "127.0.0.1")); var listenPort = CommandLine.GetInt32OrDefault("multinode.listen-port", 6577); var listenEndpoint = new IPEndPoint(listenAddress, listenPort); var specName = CommandLine.GetPropertyOrDefault("multinode.spec", ""); var tcpLogger = TestRunSystem.ActorOf(Props.Create(() => new TcpLoggingServer(SinkCoordinator)), "TcpLogger"); TestRunSystem.Tcp().Tell(new Tcp.Bind(tcpLogger, listenEndpoint)); var assemblyName = Path.GetFullPath(args[0]); EnableAllSinks(assemblyName); PublishRunnerMessage(String.Format("Running MultiNodeTests for {0}", assemblyName)); using (var controller = new XunitFrontController(assemblyName)) { using (var discovery = new Discovery()) { controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); foreach (var test in discovery.Tests.Reverse()) { if (!string.IsNullOrEmpty(test.Value.First().SkipReason)) { PublishRunnerMessage(string.Format("Skipping test {0}. Reason - {1}", test.Value.First().MethodName, test.Value.First().SkipReason)); continue; } if (!string.IsNullOrWhiteSpace(specName) && !test.Value[0].MethodName.Contains(specName)) continue; PublishRunnerMessage(string.Format("Starting test {0}", test.Value.First().MethodName)); var processes = new List<Process>(); StartNewSpec(test.Value); foreach (var nodeTest in test.Value) { //Loop through each test, work out number of nodes to run on and kick off process var process = new Process(); processes.Add(process); process.StartInfo.UseShellExecute = false; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.FileName = "Akka.NodeTestRunner.exe"; process.StartInfo.Arguments = $@"-Dmultinode.test-assembly=""{assemblyName}"" -Dmultinode.test-class=""{ nodeTest.TypeName}"" -Dmultinode.test-method=""{nodeTest.MethodName }"" -Dmultinode.max-nodes={test.Value.Count} -Dmultinode.server-host=""{"localhost" }"" -Dmultinode.host=""{"localhost"}"" -Dmultinode.index={nodeTest.Node - 1 } -Dmultinode.listen-address={listenAddress} -Dmultinode.listen-port={listenPort}"; var nodeIndex = nodeTest.Node; //TODO: might need to do some validation here to avoid the 260 character max path error on Windows var folder = Directory.CreateDirectory(Path.Combine(OutputDirectory, nodeTest.MethodName)); var logFilePath = Path.Combine(folder.FullName, "node" + nodeIndex + ".txt"); var fileActor = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath))); process.OutputDataReceived += (sender, eventArgs) => { if(eventArgs?.Data != null) fileActor.Tell(eventArgs.Data); }; var closureTest = nodeTest; process.Exited += (sender, eventArgs) => { if (process.ExitCode == 0) { ReportSpecPassFromExitCode(nodeIndex, closureTest.TestName); } }; process.Start(); process.BeginOutputReadLine(); PublishRunnerMessage(string.Format("Started node {0} on pid {1}", nodeTest.Node, process.Id)); } foreach (var process in processes) { process.WaitForExit(); var exitCode = process.ExitCode; process.Close(); } PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(3)); FinishSpec(); } } } Console.WriteLine("Complete"); PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(5)); CloseAllSinks(); //Block until all Sinks have been terminated. TestRunSystem.WhenTerminated.Wait(TimeSpan.FromMinutes(1)); if (Debugger.IsAttached) { Console.ReadLine(); //block when debugging } //Return the proper exit code Environment.Exit(ExitCodeContainer.ExitCode); }
static int Main(string[] args) { var nodeIndex = CommandLine.GetInt32("multinode.index"); var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly"); var typeName = CommandLine.GetProperty("multinode.test-class"); var testName = CommandLine.GetProperty("multinode.test-method"); var displayName = testName; var listenAddress = IPAddress.Parse(CommandLine.GetProperty("multinode.listen-address")); var listenPort = CommandLine.GetInt32("multinode.listen-port"); var listenEndpoint = new IPEndPoint(listenAddress, listenPort); var system = ActorSystem.Create("NoteTestRunner-" + nodeIndex); var tcpClient = _logger = system.ActorOf<RunnerTcpClient>(); system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient); Thread.Sleep(TimeSpan.FromSeconds(10)); using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyFileName)) { /* need to pass in just the assembly name to Discovery, not the full path * i.e. "Akka.Cluster.Tests.MultiNode.dll" * not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause * the Discovery class to actually not find any indivudal specs to run */ var assemblyName = Path.GetFileName(assemblyFileName); Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName); using (var discovery = new Discovery(assemblyName, typeName)) { using (var sink = new Sink(nodeIndex, tcpClient)) { try { controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution()); } catch (AggregateException ex) { var specFail = new SpecFail(nodeIndex, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); foreach (var innerEx in ex.Flatten().InnerExceptions) { specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString()); specFail.FailureMessages.Add(innerEx.Message); specFail.FailureStackTraces.Add(innerEx.StackTrace); } _logger.Tell(specFail.ToString()); Console.WriteLine(specFail); //make sure message is send over the wire FlushLogMessages(); Environment.Exit(1); //signal failure return 1; } catch (Exception ex) { var specFail = new SpecFail(nodeIndex, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); _logger.Tell(specFail.ToString()); Console.WriteLine(specFail); //make sure message is send over the wire FlushLogMessages(); Environment.Exit(1); //signal failure return 1; } var timedOut = false; if (!sink.Finished.WaitOne(MaxProcessWaitTimeout)) //timed out { var line = string.Format("Timed out while waiting for test to complete after {0} ms", MaxProcessWaitTimeout); _logger.Tell(line); Console.WriteLine(line); timedOut = true; } FlushLogMessages(); system.Terminate().Wait(); Environment.Exit(sink.Passed && !timedOut ? 0 : 1); return sink.Passed ? 0 : 1; } } } }
private async void RunTests(string arguments) { var reporters = new List<IRunnerReporter>(); string[] args = arguments.Split(new[] { '\x1F' }, StringSplitOptions.RemoveEmptyEntries); log = string.Empty; log += "Args: " + args + "\n"; var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger()); var completionMessages = new ConcurrentDictionary<string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { return; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; log += "testCasesToRun: " + testCasesToRun + "\n"; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); var assemblyElement = new XElement("assembly"); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (commandLine.FailSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); log += "finished running tests \n"; resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); assembliesElement.Add(assemblyElement); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); log += "logged exec errors: " + e + "\n"; } } await WriteResults(assembliesElement); await WriteLogs(log); Application.Current.Exit(); }