private static async Task Handler(int iterations, int duration, int concurrency, int rampUp, string reportFile, string target) { var reportWriter = new ReportWriter(reportFile); var userStepTime = rampUp / concurrency; var testTasks = new List <Task>(); var startTime = DateTime.UtcNow; var testAssembly = System.Reflection.Assembly.LoadFrom(target); for (var i = 1; i <= concurrency; ++i) { var threadName = $"worker_{i}"; var eventListener = new XUnitTestEventListener(reportWriter, threadName); var runner = AssemblyRunner.WithoutAppDomain(target); runner.OnDiscoveryComplete = XUnitTestEventListener.OnDiscoveryComplete; runner.OnTestFailed = eventListener.OnTestFailed; runner.OnTestSkipped = eventListener.OnTestSkipped; runner.OnTestFinished = eventListener.OnTestFinished; testTasks.Add(Task.Run(() => StartWorker(runner, startTime, iterations, duration))); Thread.Sleep(userStepTime * 1000); } await Task.WhenAll(testTasks); await reportWriter.StopWritingAsync(); }
static int Main(string[] args) { if (args.Length == 0 || args.Length > 2) { Console.WriteLine("usage: TestRunner <assembly> [typeName]"); return(2); } var testAssembly = args[0]; var typeName = args.Length == 2 ? args[1] : null; using (var runner = AssemblyRunner.WithoutAppDomain(testAssembly)) { runner.OnDiscoveryComplete = OnDiscoveryComplete; runner.OnExecutionComplete = OnExecutionComplete; runner.OnTestFailed = OnTestFailed; runner.OnTestSkipped = OnTestSkipped; Console.WriteLine("Discovering..."); runner.Start(typeName); finished.WaitOne(); finished.Dispose(); return(result); } }
/// <summary> Runs the tests. </summary> /// <returns> The Results of the test run. </returns> public TestRunnerResults Run() { var assembly = Assembly.LoadFrom(this.runnerSettings.TestAssemblyFullName); using var runner = AssemblyRunner.WithoutAppDomain(assembly.Location); runner.OnDiscoveryComplete = OnDiscoveryComplete; runner.OnExecutionComplete = OnExecutionComplete; runner.OnTestPassed = OnTestPassed; runner.OnTestFailed = OnTestFailed; runner.OnTestSkipped = OnTestSkipped; runner.TestCaseFilter = f => this.runnerSettings.TestsToRun.Any( t => t.TestClassName == f.TestMethod.TestClass.Class.Name && t.TestName == f.TestMethod.Method.Name); Console.WriteLine("Discovering..."); runner.Start(parallel: false); Finished.WaitOne(); Finished.Dispose(); return(new TestRunnerResults(true, TestRunnerType.XUnit)); }
public void TestDisposal() { AssemblyRunner runner = new AssemblyRunner(Exe); runner.Dispose(); Assert.IsTrue(runner.IsDisposed); }
protected override void RunTestCases(IEnumerable <IXunitTestCase> testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) { using (var assemblyRunner = new AssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions)) { assemblyRunner.RunAsync().GetAwaiter().GetResult(); } }
static async Task Main(string[] args) { MSBuildLocator.RegisterDefaults(); var projectFilePath = @"C:\Code\AnalyzingSourceCodeUsingRoslyn\TestRunner.Exercise\TestRunner.Exercise.csproj"; Process.Start("dotnet", $"restore {projectFilePath}").WaitForExit(); var workspace = MSBuildWorkspace.Create(); var project = await workspace.OpenProjectAsync(projectFilePath); var tests = project.Documents.Single(document => document.Name == "TestRunnerExerciseTests.cs"); var root = await tests.GetSyntaxRootAsync(); root = new UnskipTests().Visit(root); tests = tests.WithSyntaxRoot(root); var compilation = await tests.Project.GetCompilationAsync(); var diagnostics = compilation.GetDiagnostics(); var errors = diagnostics.Count(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error); Console.WriteLine($"Diagnostics: {diagnostics.Length}. Errors: {errors}"); compilation.Emit(compilation.SourceModule.Name); Assembly.LoadFrom(compilation.SourceModule.Name); var finished = new ManualResetEventSlim(); var runner = AssemblyRunner.WithoutAppDomain(compilation.SourceModule.Name); runner.OnTestFailed += info => Console.WriteLine($"[FAIL] {info.TestDisplayName}: {info.ExceptionMessage}"); runner.OnTestPassed += info => Console.WriteLine($"[PASS] {info.TestDisplayName}"); runner.OnTestSkipped += info => Console.WriteLine($"[SKIP] {info.TestDisplayName}"); runner.OnExecutionComplete += info => finished.Set(); runner.Start(); finished.Wait(); }
public static int Main(string[] args) { string solutionDir = Path.GetDirectoryName( // sw2urdf Path.GetDirectoryName( // TestRunner Path.GetDirectoryName( // bin Path.GetDirectoryName( // x64 Path.GetDirectoryName( // net452 AppDomain.CurrentDomain.BaseDirectory // Debug ))))); string testAssembly = Path.Combine(solutionDir, "SW2URDF\\bin\\x64\\Debug\\SW2URDF.dll"); string typeName = null; using (var runner = AssemblyRunner.WithAppDomain(testAssembly)) { if (null != args && args.Length > 0) { TestNameFilter = args[0]; runner.TestCaseFilter += FilterByClass; } runner.OnDiscoveryComplete = OnDiscoveryComplete; runner.OnExecutionComplete = OnExecutionComplete; runner.OnTestFailed = OnTestFailed; runner.OnTestSkipped = OnTestSkipped; Console.WriteLine("Discovering..."); runner.Start(typeName); finished.WaitOne(); finished.Dispose(); return(result); } }
static void Main(string[] args) { if (args.Length == 0 || args.Length > 2) { Console.WriteLine("usage: TestRunner <assembly> [typeName]"); } //var testAssembly = args[0]; var testAssembly = System.Diagnostics.Process.GetCurrentProcess().MainModule.FileName; var typeName = args.Length == 2 ? args[1] : null; using (var runner = AssemblyRunner.WithAppDomain(testAssembly)) { runner.OnDiscoveryComplete = OnDiscoveryComplete; runner.OnExecutionComplete = OnExecutionComplete; runner.OnTestFailed = OnTestFailed; runner.OnTestSkipped = OnTestSkipped; runner.OnTestFinished = OnTestFinished; runner.OnTestPassed = OnTestPassed; Console.WriteLine("Discovering..."); runner.Start(typeName); finished.WaitOne(); finished.Dispose(); Console.WriteLine(result); } }
public async Task <TestSuiteResult> RunTestsAsync(string dllPath) { _executionTime = 0; _finished = new ManualResetEvent(false); _results = new List <TestResult>(); using (var runner = AssemblyRunner.WithAppDomain(dllPath, shadowCopy: false)) { runner.OnExecutionComplete = OnExecutionComplete; runner.OnTestFailed = OnTestFailed; runner.OnTestPassed = OnTestPassed; await Task.Run(() => RunTests(runner)); _finished.WaitOne(); _finished.Dispose(); return(new TestSuiteResult { Name = Path.GetFileName(dllPath), TestResults = _results, ExecutionTime = TimeSpan.FromSeconds((int)_executionTime), IsSuccess = _results.All(r => r.IsSuccess) }); } }
public bool RunAllTestsInTypeUsingXunit(Type typeToRunTestsFrom) { new { typeToRunTestsFrom }.Must().NotBeNull(); var typeNameToRunTestsFrom = typeToRunTestsFrom.FullName; var testAssemblyPath = typeToRunTestsFrom.Assembly.GetCodeBaseAsPathInsteadOfUri(); using (var runner = AssemblyRunner.WithAppDomain(testAssemblyPath)) { runner.OnDiscoveryComplete = this.OnDiscoveryComplete; runner.OnTestOutput = this.OnTestOutput; runner.OnTestPassed = this.OnTestPassed; runner.OnTestFailed = this.OnTestFailed; runner.OnTestSkipped = this.OnTestSkipped; runner.OnExecutionComplete = this.OnExecutionComplete; lock (this.announcementLock) { this.announcer(" * Discovering tests in type."); } runner.Start(typeNameToRunTestsFrom); this.finished.WaitOne(); this.finished.Dispose(); return(!this.seenFailures); } }
static int Main(string[] args) { string testAssembly = typeof(Program).Assembly.Location; var typeName = args.Length == 2 ? args[1] : null; try { using (var runner = AssemblyRunner.WithoutAppDomain(testAssembly)) { runner.OnDiscoveryComplete = OnDiscoveryComplete; runner.OnExecutionComplete = OnExecutionComplete; runner.OnTestFailed = OnTestFailed; runner.OnTestSkipped = OnTestSkipped; Console.WriteLine("Discovering..."); runner.Start(typeName); finished.WaitOne(); finished.Dispose(); } } catch (InvalidOperationException) { // Swallow } return(result); }
public void TestStdOutputAndStdError() { using (AssemblyRunner runner = new AssemblyRunner(Exe)) { StringWriter wtr = new StringWriter(); StringWriter err = new StringWriter(); ProcessOutputEventHandler handler = delegate(object o, ProcessOutputEventArgs e) { if (e.Error) { err.WriteLine(e.Data); } else { wtr.WriteLine(e.Data); } }; runner.OutputReceived += handler; Assert.AreEqual(1, runner.Run("command-line-argrument")); Assert.AreEqual("std-err", err.ToString().Trim()); StringReader rdr = new StringReader(wtr.ToString()); Assert.AreEqual("WorkingDirectory = " + Environment.CurrentDirectory, rdr.ReadLine()); Assert.AreEqual("argument[0] = command-line-argrument", rdr.ReadLine()); Assert.AreEqual("std-input:", rdr.ReadLine()); Assert.AreEqual(null, rdr.ReadLine()); } }
public static int Run(string assemblyPath, List <string> typeNames = null) { // Create a runner for the specifid assembly. using (AssemblyRunner runner = AssemblyRunner.WithoutAppDomain(assemblyPath)) { // Start out assuming success; we'll set this to 1 if we get a failed test var result = new int[] { 0 }; if (typeNames?.Count == 0) { Run(runner, result, null); } else { typeNames.ForEach(typeName => Run(runner, result, typeName)); } // Wait for the assembly runner to go idle. using (var waitHandle = new ManualResetEvent(false)) { while (runner.Status != AssemblyRunnerStatus.Idle) { waitHandle.WaitOne(10); } } // Return overall status. return(result[0]); } }
protected override void OnCreate(Bundle savedInstanceState) { base.OnCreate(savedInstanceState); TestParentActivity = this; // Set our view from the "main" layout resource SetContentView(Resource.Layout.Main); listView = FindViewById <ListView>(Resource.Id.listView); adapter = new TestResultsAdapter { Parent = this }; listView.Adapter = adapter; assemblyRunner = AssemblyRunner.WithoutAppDomain(Assembly.GetExecutingAssembly().GetName().Name + ".dll"); assemblyRunner.OnDiscoveryComplete = DiscoveryCompleteHandler;; assemblyRunner.OnExecutionComplete = ExecutionCompleteHandler;; assemblyRunner.OnTestFailed = TestFailedHandler;; assemblyRunner.OnTestSkipped = TestSkippedHandler; assemblyRunner.OnTestPassed = TestPassedHandler; assemblyRunner.OnTestOutput = TestOutputHandler; Console.WriteLine("Discovering..."); assemblyRunner.Start(); }
public TestResult RunTests() { _totalTests = 0; _testsFailed = 0; using (finished = new ManualResetEvent(false)) using (AssemblyRunner runner = AssemblyRunner.WithoutAppDomain(_assemblyFileName)) { runner.OnExecutionComplete = TestAssemblyExecutionFinished; runner.Start(); finished.WaitOne(); while (runner.Status != AssemblyRunnerStatus.Idle) { Thread.Sleep(500); } } TestResult result = new TestResult { TotalTests = _totalTests, TestsFailed = _testsFailed }; return(result); }
public void TestKill() { Environment.CurrentDirectory = Path.GetDirectoryName(FileUtils.FindFullPath("cmd.exe")); using (AssemblyRunner runner = new AssemblyRunner(Exe)) { ManualResetEvent gotExit = new ManualResetEvent(false); runner.ProcessExited += delegate(Object o, ProcessExitedEventArgs e) { gotExit.Set(); }; Assert.IsFalse(runner.IsRunning); runner.Kill(); // always safe to call runner.Start("-wait"); Assert.IsTrue(runner.IsRunning); try //make sure we can't start twice. { runner.Start(); Assert.Fail(); } catch (InvalidOperationException) { } Assert.IsFalse(runner.WaitForExit(TimeSpan.FromSeconds(1), false)); runner.Kill(); Assert.IsFalse(runner.IsRunning); Assert.IsTrue(gotExit.WaitOne(30000, false)); } }
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] args) { if (!testsToRun.Any()) { // discovery cannot be done // When running with empty list of tests XUnit hangs return(new TestExecutionResults()); } executedTests.Clear(); this.testsToRun = testsToRun; executionCanceled = false; result = new TestExecutionResults(); runner = AssemblyRunner.WithAppDomain(testModule); logger.Debug($"Running XUnit executor in App domain: {AppDomain.CurrentDomain.FriendlyName}"); runner.TestCaseFilter = TestCaseFilterHandler; runner.OnExecutionComplete = ExecutionCompleteHandler; runner.OnTestFailed = TestFailedHandler; runner.OnTestFinished += TestFinishedHandler; runner.OnDiscoveryComplete += DiscoveryCompleteHandler; runner.OnErrorMessage += ErrorMessageHandler; logger.Debug("Starting tests"); runner.Start(maxParallelThreads: 1); finished.WaitOne(); finished.Dispose(); while (runner.Status != AssemblyRunnerStatus.Idle && !executionCanceled) { // spin until runner finishes } if (executionCanceled) { try { runner.Dispose(); } catch (InvalidOperationException e) { // for now ignore this exception // due to XUnit bug of not resetting Status to Idle when cancelling run } } else { runner.Dispose(); } var temp = result; // reset result variable result = null; return(temp); }
public void TestStandardInputFail() { using (IRunner runner = new AssemblyRunner(Exe)) { runner.Start("-wait"); runner.StandardInput.WriteLine(); } }
public void AddsNonFilteredCorrectly() { var runner = new AssemblyRunner(new TestLogger(), "test"); runner.AddTestCases(fakeTest1); runner.AddTestCases(fakeTest2); Assert.That(runner.NUnitFilter.IsEmpty, Is.True, "NUnitfilter has been touched"); Assert.That(runner.LoadedTestCases.Count, Is.EqualTo(2), "We should have had 2 test cases here"); }
public void TestUnhandledExceptionRun() { using (IRunner runner = new AssemblyRunner(Exe)) { runner.Run("-throw", "System.ArgumentOutOfRangeException"); Assert.Fail(); } }
public void AddsNonFilteredCorrectly() { var runner = new AssemblyRunner(new TestLogger(), "test"); runner.AddTestCases(fakeTest1); runner.AddTestCases(fakeTest2); Assert.That(runner.NUnitFilter.IsEmpty, Is.True, "NUnitfilter has been touched"); Assert.That(runner.LoadedTestCases.Count, Is.EqualTo(2), "We should have had 2 test cases here"); }
/// <summary> /// Run the provided donatello program and return the result. /// </summary> /// <typeparam name="T">The expected return type</typeparam> /// <param name="program">The donatello source code</param> /// <returns>the return value of the program</returns> public static T Run <T>(string program, string assemblyName = null, params string[] references) { const string namespaceName = "DonatelloRun"; const string className = "Runner"; const string methodName = "Run"; CompilationUnitSyntax syntaxTree = AntlrParser.ParseAsClass(program, namespaceName, className, methodName); Stream result = Compiler.Compile(assemblyName ?? namespaceName, references, OutputType.DynamicallyLinkedLibrary, syntaxTree); return(AssemblyRunner.Run <T>(result, namespaceName, className, methodName)); }
public void AddsFilteredCorrectly() { var t1 = new TestCase(fakeTest1.TestName.FullName, ExecutorUri, "test"); var t2 = new TestCase(fakeTest2.TestName.FullName, ExecutorUri, "test"); var list = new List<TestCase> {t1, t2}; var runner = new AssemblyRunner(new TestLogger(), "test", list); runner.AddTestCases(fakeTest1); runner.AddTestCases(fakeTest2); Assert.That(runner.NUnitFilter.IsEmpty, Is.False, "NUnitfilter should not be empty, we have added testcases"); Assert.That(runner.LoadedTestCases.Count, Is.EqualTo(2), "We should have had 2 converted MS test cases here"); }
public void VerifyConstruction2() { var t1 = new TestCase(fakeTest1.TestName.FullName, ExecutorUri, "test"); var t2 = new TestCase(fakeTest2.TestName.FullName, ExecutorUri, "test"); var list = new List<TestCase> {t1, t2}; var runner = new AssemblyRunner(new TestLogger(), "test", list); Assert.False(runner.NUnitFilter.IsEmpty); Assert.That(runner.NUnitFilter, Is.TypeOf<SimpleNameFilter>()); Assert.True(runner.NUnitFilter.Match(fakeTest1)); Assert.True(runner.NUnitFilter.Match(fakeTest2)); }
private static void RunAllTests(AssemblyRunner runner) { using var executionCompleteEvent = new ManualResetEvent(false); var onExecutionComplete = runner.OnExecutionComplete; runner.OnExecutionComplete += info => executionCompleteEvent.Set(); runner.Start(); executionCompleteEvent.WaitOne(); runner.OnExecutionComplete = onExecutionComplete; }
private void RunTests(AssemblyRunner runner) { try { runner.Start(); } catch (Exception ex) { LogTo.ErrorException("Failed to unload test runner", ex); _finished.Set(); } }
public void TestExecuteInstructions() { var instructions = Testing.GetTestFileContents("TestInput.txt").Split(Environment.NewLine); var(programOneOutput, programTwoOutput) = (new Queue <long>(), new Queue <long>()); var firstRunner = new AssemblyRunner(instructions, programTwoOutput, programOneOutput, 0); var secondRunner = new AssemblyRunner(instructions, programOneOutput, programTwoOutput, 1); Program.RunTwoRunnersInParallel(firstRunner, secondRunner); Assert.AreEqual(3, firstRunner.ValuesSent); }
private async Task WaitRunnerExitAsync(AssemblyRunner runner, CancellationToken token) { while (runner.Status != AssemblyRunnerStatus.Idle) { // Wait for assembly runner to finish. // If we try to dispose while runner is executing, // it will throw an error. await Task.Delay(5, token); token.ThrowIfCancellationRequested(); } }
public void AddsFilteredCorrectly() { var t1 = new TestCase(fakeTest1.TestName.FullName, ExecutorUri, "test"); var t2 = new TestCase(fakeTest2.TestName.FullName, ExecutorUri, "test"); var list = new List <TestCase> { t1, t2 }; var runner = new AssemblyRunner(new TestLogger(), "test", list); runner.AddTestCases(fakeTest1); runner.AddTestCases(fakeTest2); Assert.That(runner.NUnitFilter.IsEmpty, Is.False, "NUnitfilter should not be empty, we have added testcases"); Assert.That(runner.LoadedTestCases.Count, Is.EqualTo(2), "We should have had 2 converted MS test cases here"); }
public void VerifyConstruction2() { var t1 = new TestCase(fakeTest1.TestName.FullName, ExecutorUri, "test"); var t2 = new TestCase(fakeTest2.TestName.FullName, ExecutorUri, "test"); var list = new List <TestCase> { t1, t2 }; var runner = new AssemblyRunner(new TestLogger(), "test", list); Assert.False(runner.NUnitFilter.IsEmpty); Assert.That(runner.NUnitFilter, Is.TypeOf <SimpleNameFilter>()); Assert.True(runner.NUnitFilter.Match(fakeTest1)); Assert.True(runner.NUnitFilter.Match(fakeTest2)); }
private static void SetupRunnerCallbacks(AssemblyRunner runner, ManualResetEvent manualResetEvent, object consoleLock, int[] result) { runner.TestCaseFilter = (ITestCase testCase) => testCase.GetType() == typeof(BenchmarkTestCase); runner.OnDiscoveryComplete = info => { lock (consoleLock) { WriteInfoLine($"Running {info.TestCasesToRun} Benchmarks out of {info.TestCasesDiscovered} Xunit Facts..."); } }; runner.OnExecutionComplete = info => { lock (consoleLock) { WriteInfoLine($"Finished {info.TotalTests} tests in {Math.Round(info.ExecutionTime, 3)}s ({info.TestsFailed} failed, {info.TestsSkipped} skipped)"); } manualResetEvent.Set(); }; runner.OnTestStarting = info => { lock (consoleLock) { WriteInfoLine($" {info.TestDisplayName}"); } }; runner.OnTestFailed = info => { lock (consoleLock) { WriteErrorLine($"{info.TestDisplayName}: {info.ExceptionMessage}"); if (info.ExceptionStackTrace != null) { WriteErrorLine(info.ExceptionStackTrace); } } result[0] += 1; }; runner.OnTestSkipped = info => { lock (consoleLock) { WriteWarningLine($"[SKIP] {info.TestDisplayName}: {info.SkipReason}"); } }; }
public void TestRunWithInput() { using (AssemblyRunner runner = new AssemblyRunner(Exe)) { List <string> lines = new List <string>(); runner.OutputReceived += delegate(Object o, ProcessOutputEventArgs e) { lines.Add(e.Data); }; int exitCode = runner.Run(new StringReader("Hello World\r\nWhatever!\r\nAnother line.")); Assert.AreEqual(0, exitCode); // 0 == WorkingDirectory = // 1 == std-input: Assert.AreEqual("Hello World", lines[2]); Assert.AreEqual("Whatever!", lines[3]); Assert.AreEqual("Another line.", lines[4]); } }
public void TestOutputEventUnsubscribe() { using (AssemblyRunner runner = new AssemblyRunner(Exe)) { bool outputReceived = false; ProcessOutputEventHandler handler = delegate(object o, ProcessOutputEventArgs e) { outputReceived = true; }; runner.OutputReceived += handler; runner.OutputReceived -= handler; Assert.AreEqual(0, runner.Run()); Assert.IsFalse(outputReceived); } }
public void VerifyConstruction1() { var runner = new AssemblyRunner(new TestLogger(), "test"); Assert.That(runner.NUnitFilter.IsEmpty, Is.True); }