private static async Task Handler(int iterations, int duration, int concurrency, int rampUp, string reportFile,
string target)
{
var reportWriter = new ReportWriter(reportFile);
var userStepTime = rampUp / concurrency;
var testTasks = new List <Task>();
var startTime = DateTime.UtcNow;
var testAssembly = System.Reflection.Assembly.LoadFrom(target);
for (var i = 1; i <= concurrency; ++i)
{
var threadName = $"worker_{i}";
var eventListener = new XUnitTestEventListener(reportWriter, threadName);
var runner = AssemblyRunner.WithoutAppDomain(target);
runner.OnDiscoveryComplete = XUnitTestEventListener.OnDiscoveryComplete;
runner.OnTestFailed = eventListener.OnTestFailed;
runner.OnTestSkipped = eventListener.OnTestSkipped;
runner.OnTestFinished = eventListener.OnTestFinished;
testTasks.Add(Task.Run(() => StartWorker(runner, startTime, iterations, duration)));
Thread.Sleep(userStepTime * 1000);
}
await Task.WhenAll(testTasks);
await reportWriter.StopWritingAsync();
}
static int Main(string[] args)
{
if (args.Length == 0 || args.Length > 2)
{
Console.WriteLine("usage: TestRunner <assembly> [typeName]");
return(2);
}
var testAssembly = args[0];
var typeName = args.Length == 2 ? args[1] : null;
using (var runner = AssemblyRunner.WithoutAppDomain(testAssembly))
{
runner.OnDiscoveryComplete = OnDiscoveryComplete;
runner.OnExecutionComplete = OnExecutionComplete;
runner.OnTestFailed = OnTestFailed;
runner.OnTestSkipped = OnTestSkipped;
Console.WriteLine("Discovering...");
runner.Start(typeName);
finished.WaitOne();
finished.Dispose();
return(result);
}
}
/// <summary> Runs the tests. </summary>
/// <returns> The Results of the test run. </returns>
public TestRunnerResults Run()
{
var assembly = Assembly.LoadFrom(this.runnerSettings.TestAssemblyFullName);
using var runner = AssemblyRunner.WithoutAppDomain(assembly.Location);
runner.OnDiscoveryComplete = OnDiscoveryComplete;
runner.OnExecutionComplete = OnExecutionComplete;
runner.OnTestPassed = OnTestPassed;
runner.OnTestFailed = OnTestFailed;
runner.OnTestSkipped = OnTestSkipped;
runner.TestCaseFilter =
f => this.runnerSettings.TestsToRun.Any(
t => t.TestClassName == f.TestMethod.TestClass.Class.Name &&
t.TestName == f.TestMethod.Method.Name);
Console.WriteLine("Discovering...");
runner.Start(parallel: false);
Finished.WaitOne();
Finished.Dispose();
return(new TestRunnerResults(true, TestRunnerType.XUnit));
}
public void TestDisposal()
{
AssemblyRunner runner = new AssemblyRunner(Exe);
runner.Dispose();
Assert.IsTrue(runner.IsDisposed);
}
protected override void RunTestCases(IEnumerable <IXunitTestCase> testCases, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions)
{
using (var assemblyRunner = new AssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions))
{
assemblyRunner.RunAsync().GetAwaiter().GetResult();
}
}
static async Task Main(string[] args)
{
MSBuildLocator.RegisterDefaults();
var projectFilePath = @"C:\Code\AnalyzingSourceCodeUsingRoslyn\TestRunner.Exercise\TestRunner.Exercise.csproj";
Process.Start("dotnet", $"restore {projectFilePath}").WaitForExit();
var workspace = MSBuildWorkspace.Create();
var project = await workspace.OpenProjectAsync(projectFilePath);
var tests = project.Documents.Single(document => document.Name == "TestRunnerExerciseTests.cs");
var root = await tests.GetSyntaxRootAsync();
root = new UnskipTests().Visit(root);
tests = tests.WithSyntaxRoot(root);
var compilation = await tests.Project.GetCompilationAsync();
var diagnostics = compilation.GetDiagnostics();
var errors = diagnostics.Count(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error);
Console.WriteLine($"Diagnostics: {diagnostics.Length}. Errors: {errors}");
compilation.Emit(compilation.SourceModule.Name);
Assembly.LoadFrom(compilation.SourceModule.Name);
var finished = new ManualResetEventSlim();
var runner = AssemblyRunner.WithoutAppDomain(compilation.SourceModule.Name);
runner.OnTestFailed += info => Console.WriteLine($"[FAIL] {info.TestDisplayName}: {info.ExceptionMessage}");
runner.OnTestPassed += info => Console.WriteLine($"[PASS] {info.TestDisplayName}");
runner.OnTestSkipped += info => Console.WriteLine($"[SKIP] {info.TestDisplayName}");
runner.OnExecutionComplete += info => finished.Set();
runner.Start();
finished.Wait();
}
public static int Main(string[] args)
{
string solutionDir =
Path.GetDirectoryName( // sw2urdf
Path.GetDirectoryName( // TestRunner
Path.GetDirectoryName( // bin
Path.GetDirectoryName( // x64
Path.GetDirectoryName( // net452
AppDomain.CurrentDomain.BaseDirectory // Debug
)))));
string testAssembly = Path.Combine(solutionDir, "SW2URDF\\bin\\x64\\Debug\\SW2URDF.dll");
string typeName = null;
using (var runner = AssemblyRunner.WithAppDomain(testAssembly))
{
if (null != args && args.Length > 0)
{
TestNameFilter = args[0];
runner.TestCaseFilter += FilterByClass;
}
runner.OnDiscoveryComplete = OnDiscoveryComplete;
runner.OnExecutionComplete = OnExecutionComplete;
runner.OnTestFailed = OnTestFailed;
runner.OnTestSkipped = OnTestSkipped;
Console.WriteLine("Discovering...");
runner.Start(typeName);
finished.WaitOne();
finished.Dispose();
return(result);
}
}
static void Main(string[] args)
{
if (args.Length == 0 || args.Length > 2)
{
Console.WriteLine("usage: TestRunner <assembly> [typeName]");
}
//var testAssembly = args[0];
var testAssembly = System.Diagnostics.Process.GetCurrentProcess().MainModule.FileName;
var typeName = args.Length == 2 ? args[1] : null;
using (var runner = AssemblyRunner.WithAppDomain(testAssembly))
{
runner.OnDiscoveryComplete = OnDiscoveryComplete;
runner.OnExecutionComplete = OnExecutionComplete;
runner.OnTestFailed = OnTestFailed;
runner.OnTestSkipped = OnTestSkipped;
runner.OnTestFinished = OnTestFinished;
runner.OnTestPassed = OnTestPassed;
Console.WriteLine("Discovering...");
runner.Start(typeName);
finished.WaitOne();
finished.Dispose();
Console.WriteLine(result);
}
}
public async Task <TestSuiteResult> RunTestsAsync(string dllPath)
{
_executionTime = 0;
_finished = new ManualResetEvent(false);
_results = new List <TestResult>();
using (var runner = AssemblyRunner.WithAppDomain(dllPath, shadowCopy: false))
{
runner.OnExecutionComplete = OnExecutionComplete;
runner.OnTestFailed = OnTestFailed;
runner.OnTestPassed = OnTestPassed;
await Task.Run(() => RunTests(runner));
_finished.WaitOne();
_finished.Dispose();
return(new TestSuiteResult
{
Name = Path.GetFileName(dllPath),
TestResults = _results,
ExecutionTime = TimeSpan.FromSeconds((int)_executionTime),
IsSuccess = _results.All(r => r.IsSuccess)
});
}
}
public bool RunAllTestsInTypeUsingXunit(Type typeToRunTestsFrom)
{
new { typeToRunTestsFrom }.Must().NotBeNull();
var typeNameToRunTestsFrom = typeToRunTestsFrom.FullName;
var testAssemblyPath = typeToRunTestsFrom.Assembly.GetCodeBaseAsPathInsteadOfUri();
using (var runner = AssemblyRunner.WithAppDomain(testAssemblyPath))
{
runner.OnDiscoveryComplete = this.OnDiscoveryComplete;
runner.OnTestOutput = this.OnTestOutput;
runner.OnTestPassed = this.OnTestPassed;
runner.OnTestFailed = this.OnTestFailed;
runner.OnTestSkipped = this.OnTestSkipped;
runner.OnExecutionComplete = this.OnExecutionComplete;
lock (this.announcementLock)
{
this.announcer(" * Discovering tests in type.");
}
runner.Start(typeNameToRunTestsFrom);
this.finished.WaitOne();
this.finished.Dispose();
return(!this.seenFailures);
}
}
static int Main(string[] args)
{
string testAssembly = typeof(Program).Assembly.Location;
var typeName = args.Length == 2 ? args[1] : null;
try
{
using (var runner = AssemblyRunner.WithoutAppDomain(testAssembly))
{
runner.OnDiscoveryComplete = OnDiscoveryComplete;
runner.OnExecutionComplete = OnExecutionComplete;
runner.OnTestFailed = OnTestFailed;
runner.OnTestSkipped = OnTestSkipped;
Console.WriteLine("Discovering...");
runner.Start(typeName);
finished.WaitOne();
finished.Dispose();
}
}
catch (InvalidOperationException)
{
// Swallow
}
return(result);
}
public void TestStdOutputAndStdError()
{
using (AssemblyRunner runner = new AssemblyRunner(Exe))
{
StringWriter wtr = new StringWriter();
StringWriter err = new StringWriter();
ProcessOutputEventHandler handler =
delegate(object o, ProcessOutputEventArgs e)
{ if (e.Error)
{
err.WriteLine(e.Data);
}
else
{
wtr.WriteLine(e.Data);
} };
runner.OutputReceived += handler;
Assert.AreEqual(1, runner.Run("command-line-argrument"));
Assert.AreEqual("std-err", err.ToString().Trim());
StringReader rdr = new StringReader(wtr.ToString());
Assert.AreEqual("WorkingDirectory = " + Environment.CurrentDirectory, rdr.ReadLine());
Assert.AreEqual("argument[0] = command-line-argrument", rdr.ReadLine());
Assert.AreEqual("std-input:", rdr.ReadLine());
Assert.AreEqual(null, rdr.ReadLine());
}
}
public static int Run(string assemblyPath, List <string> typeNames = null)
{
// Create a runner for the specifid assembly.
using (AssemblyRunner runner = AssemblyRunner.WithoutAppDomain(assemblyPath))
{
// Start out assuming success; we'll set this to 1 if we get a failed test
var result = new int[] { 0 };
if (typeNames?.Count == 0)
{
Run(runner, result, null);
}
else
{
typeNames.ForEach(typeName => Run(runner, result, typeName));
}
// Wait for the assembly runner to go idle.
using (var waitHandle = new ManualResetEvent(false))
{
while (runner.Status != AssemblyRunnerStatus.Idle)
{
waitHandle.WaitOne(10);
}
}
// Return overall status.
return(result[0]);
}
}
protected override void OnCreate(Bundle savedInstanceState)
{
base.OnCreate(savedInstanceState);
TestParentActivity = this;
// Set our view from the "main" layout resource
SetContentView(Resource.Layout.Main);
listView = FindViewById <ListView>(Resource.Id.listView);
adapter = new TestResultsAdapter {
Parent = this
};
listView.Adapter = adapter;
assemblyRunner = AssemblyRunner.WithoutAppDomain(Assembly.GetExecutingAssembly().GetName().Name + ".dll");
assemblyRunner.OnDiscoveryComplete = DiscoveryCompleteHandler;;
assemblyRunner.OnExecutionComplete = ExecutionCompleteHandler;;
assemblyRunner.OnTestFailed = TestFailedHandler;;
assemblyRunner.OnTestSkipped = TestSkippedHandler;
assemblyRunner.OnTestPassed = TestPassedHandler;
assemblyRunner.OnTestOutput = TestOutputHandler;
Console.WriteLine("Discovering...");
assemblyRunner.Start();
}
public TestResult RunTests()
{
_totalTests = 0;
_testsFailed = 0;
using (finished = new ManualResetEvent(false))
using (AssemblyRunner runner = AssemblyRunner.WithoutAppDomain(_assemblyFileName))
{
runner.OnExecutionComplete = TestAssemblyExecutionFinished;
runner.Start();
finished.WaitOne();
while (runner.Status != AssemblyRunnerStatus.Idle)
{
Thread.Sleep(500);
}
}
TestResult result = new TestResult {
TotalTests = _totalTests, TestsFailed = _testsFailed
};
return(result);
}
public void TestKill()
{
Environment.CurrentDirectory = Path.GetDirectoryName(FileUtils.FindFullPath("cmd.exe"));
using (AssemblyRunner runner = new AssemblyRunner(Exe))
{
ManualResetEvent gotExit = new ManualResetEvent(false);
runner.ProcessExited += delegate(Object o, ProcessExitedEventArgs e) { gotExit.Set(); };
Assert.IsFalse(runner.IsRunning);
runner.Kill(); // always safe to call
runner.Start("-wait");
Assert.IsTrue(runner.IsRunning);
try //make sure we can't start twice.
{
runner.Start();
Assert.Fail();
}
catch (InvalidOperationException)
{ }
Assert.IsFalse(runner.WaitForExit(TimeSpan.FromSeconds(1), false));
runner.Kill();
Assert.IsFalse(runner.IsRunning);
Assert.IsTrue(gotExit.WaitOne(30000, false));
}
}
protected override TestExecutionResults ExecuteImpl(string testModule, IEnumerable <IMemberDefinition> testsToRun, string[] args)
{
if (!testsToRun.Any())
{
// discovery cannot be done
// When running with empty list of tests XUnit hangs
return(new TestExecutionResults());
}
executedTests.Clear();
this.testsToRun = testsToRun;
executionCanceled = false;
result = new TestExecutionResults();
runner = AssemblyRunner.WithAppDomain(testModule);
logger.Debug($"Running XUnit executor in App domain: {AppDomain.CurrentDomain.FriendlyName}");
runner.TestCaseFilter = TestCaseFilterHandler;
runner.OnExecutionComplete = ExecutionCompleteHandler;
runner.OnTestFailed = TestFailedHandler;
runner.OnTestFinished += TestFinishedHandler;
runner.OnDiscoveryComplete += DiscoveryCompleteHandler;
runner.OnErrorMessage += ErrorMessageHandler;
logger.Debug("Starting tests");
runner.Start(maxParallelThreads: 1);
finished.WaitOne();
finished.Dispose();
while (runner.Status != AssemblyRunnerStatus.Idle && !executionCanceled)
{
// spin until runner finishes
}
if (executionCanceled)
{
try
{
runner.Dispose();
}
catch (InvalidOperationException e)
{
// for now ignore this exception
// due to XUnit bug of not resetting Status to Idle when cancelling run
}
}
else
{
runner.Dispose();
}
var temp = result;
// reset result variable
result = null;
return(temp);
}
public void TestStandardInputFail()
{
using (IRunner runner = new AssemblyRunner(Exe))
{
runner.Start("-wait");
runner.StandardInput.WriteLine();
}
}
public void AddsNonFilteredCorrectly()
{
var runner = new AssemblyRunner(new TestLogger(), "test");
runner.AddTestCases(fakeTest1);
runner.AddTestCases(fakeTest2);
Assert.That(runner.NUnitFilter.IsEmpty, Is.True, "NUnitfilter has been touched");
Assert.That(runner.LoadedTestCases.Count, Is.EqualTo(2), "We should have had 2 test cases here");
}
public void TestUnhandledExceptionRun()
{
using (IRunner runner = new AssemblyRunner(Exe))
{
runner.Run("-throw", "System.ArgumentOutOfRangeException");
Assert.Fail();
}
}
public void AddsNonFilteredCorrectly()
{
var runner = new AssemblyRunner(new TestLogger(), "test");
runner.AddTestCases(fakeTest1);
runner.AddTestCases(fakeTest2);
Assert.That(runner.NUnitFilter.IsEmpty, Is.True, "NUnitfilter has been touched");
Assert.That(runner.LoadedTestCases.Count, Is.EqualTo(2), "We should have had 2 test cases here");
}
/// <summary>
/// Run the provided donatello program and return the result.
/// </summary>
/// <typeparam name="T">The expected return type</typeparam>
/// <param name="program">The donatello source code</param>
/// <returns>the return value of the program</returns>
public static T Run <T>(string program, string assemblyName = null, params string[] references)
{
const string namespaceName = "DonatelloRun";
const string className = "Runner";
const string methodName = "Run";
CompilationUnitSyntax syntaxTree = AntlrParser.ParseAsClass(program, namespaceName, className, methodName);
Stream result = Compiler.Compile(assemblyName ?? namespaceName, references, OutputType.DynamicallyLinkedLibrary, syntaxTree);
return(AssemblyRunner.Run <T>(result, namespaceName, className, methodName));
}
public void AddsFilteredCorrectly()
{
var t1 = new TestCase(fakeTest1.TestName.FullName, ExecutorUri, "test");
var t2 = new TestCase(fakeTest2.TestName.FullName, ExecutorUri, "test");
var list = new List<TestCase> {t1, t2};
var runner = new AssemblyRunner(new TestLogger(), "test", list);
runner.AddTestCases(fakeTest1);
runner.AddTestCases(fakeTest2);
Assert.That(runner.NUnitFilter.IsEmpty, Is.False, "NUnitfilter should not be empty, we have added testcases");
Assert.That(runner.LoadedTestCases.Count, Is.EqualTo(2), "We should have had 2 converted MS test cases here");
}
public void VerifyConstruction2()
{
var t1 = new TestCase(fakeTest1.TestName.FullName, ExecutorUri, "test");
var t2 = new TestCase(fakeTest2.TestName.FullName, ExecutorUri, "test");
var list = new List<TestCase> {t1, t2};
var runner = new AssemblyRunner(new TestLogger(), "test", list);
Assert.False(runner.NUnitFilter.IsEmpty);
Assert.That(runner.NUnitFilter, Is.TypeOf<SimpleNameFilter>());
Assert.True(runner.NUnitFilter.Match(fakeTest1));
Assert.True(runner.NUnitFilter.Match(fakeTest2));
}
private static void RunAllTests(AssemblyRunner runner)
{
using var executionCompleteEvent = new ManualResetEvent(false);
var onExecutionComplete = runner.OnExecutionComplete;
runner.OnExecutionComplete += info => executionCompleteEvent.Set();
runner.Start();
executionCompleteEvent.WaitOne();
runner.OnExecutionComplete = onExecutionComplete;
}
private void RunTests(AssemblyRunner runner)
{
try
{
runner.Start();
}
catch (Exception ex)
{
LogTo.ErrorException("Failed to unload test runner", ex);
_finished.Set();
}
}
public void TestExecuteInstructions()
{
var instructions = Testing.GetTestFileContents("TestInput.txt").Split(Environment.NewLine);
var(programOneOutput, programTwoOutput) = (new Queue <long>(), new Queue <long>());
var firstRunner = new AssemblyRunner(instructions, programTwoOutput, programOneOutput, 0);
var secondRunner = new AssemblyRunner(instructions, programOneOutput, programTwoOutput, 1);
Program.RunTwoRunnersInParallel(firstRunner, secondRunner);
Assert.AreEqual(3, firstRunner.ValuesSent);
}
private async Task WaitRunnerExitAsync(AssemblyRunner runner, CancellationToken token)
{
while (runner.Status != AssemblyRunnerStatus.Idle)
{
// Wait for assembly runner to finish.
// If we try to dispose while runner is executing,
// it will throw an error.
await Task.Delay(5, token);
token.ThrowIfCancellationRequested();
}
}
public void AddsFilteredCorrectly()
{
var t1 = new TestCase(fakeTest1.TestName.FullName, ExecutorUri, "test");
var t2 = new TestCase(fakeTest2.TestName.FullName, ExecutorUri, "test");
var list = new List <TestCase> {
t1, t2
};
var runner = new AssemblyRunner(new TestLogger(), "test", list);
runner.AddTestCases(fakeTest1);
runner.AddTestCases(fakeTest2);
Assert.That(runner.NUnitFilter.IsEmpty, Is.False, "NUnitfilter should not be empty, we have added testcases");
Assert.That(runner.LoadedTestCases.Count, Is.EqualTo(2), "We should have had 2 converted MS test cases here");
}
public void VerifyConstruction2()
{
var t1 = new TestCase(fakeTest1.TestName.FullName, ExecutorUri, "test");
var t2 = new TestCase(fakeTest2.TestName.FullName, ExecutorUri, "test");
var list = new List <TestCase> {
t1, t2
};
var runner = new AssemblyRunner(new TestLogger(), "test", list);
Assert.False(runner.NUnitFilter.IsEmpty);
Assert.That(runner.NUnitFilter, Is.TypeOf <SimpleNameFilter>());
Assert.True(runner.NUnitFilter.Match(fakeTest1));
Assert.True(runner.NUnitFilter.Match(fakeTest2));
}
private static void SetupRunnerCallbacks(AssemblyRunner runner, ManualResetEvent manualResetEvent, object consoleLock, int[] result)
{
runner.TestCaseFilter = (ITestCase testCase) => testCase.GetType() == typeof(BenchmarkTestCase);
runner.OnDiscoveryComplete = info =>
{
lock (consoleLock)
{
WriteInfoLine($"Running {info.TestCasesToRun} Benchmarks out of {info.TestCasesDiscovered} Xunit Facts...");
}
};
runner.OnExecutionComplete = info =>
{
lock (consoleLock)
{
WriteInfoLine($"Finished {info.TotalTests} tests in {Math.Round(info.ExecutionTime, 3)}s ({info.TestsFailed} failed, {info.TestsSkipped} skipped)");
}
manualResetEvent.Set();
};
runner.OnTestStarting = info =>
{
lock (consoleLock)
{
WriteInfoLine($" {info.TestDisplayName}");
}
};
runner.OnTestFailed = info =>
{
lock (consoleLock)
{
WriteErrorLine($"{info.TestDisplayName}: {info.ExceptionMessage}");
if (info.ExceptionStackTrace != null)
{
WriteErrorLine(info.ExceptionStackTrace);
}
}
result[0] += 1;
};
runner.OnTestSkipped = info =>
{
lock (consoleLock)
{
WriteWarningLine($"[SKIP] {info.TestDisplayName}: {info.SkipReason}");
}
};
}
public void TestRunWithInput()
{
using (AssemblyRunner runner = new AssemblyRunner(Exe))
{
List <string> lines = new List <string>();
runner.OutputReceived += delegate(Object o, ProcessOutputEventArgs e) { lines.Add(e.Data); };
int exitCode = runner.Run(new StringReader("Hello World\r\nWhatever!\r\nAnother line."));
Assert.AreEqual(0, exitCode);
// 0 == WorkingDirectory =
// 1 == std-input:
Assert.AreEqual("Hello World", lines[2]);
Assert.AreEqual("Whatever!", lines[3]);
Assert.AreEqual("Another line.", lines[4]);
}
}
public void TestOutputEventUnsubscribe()
{
using (AssemblyRunner runner = new AssemblyRunner(Exe))
{
bool outputReceived = false;
ProcessOutputEventHandler handler =
delegate(object o, ProcessOutputEventArgs e)
{ outputReceived = true; };
runner.OutputReceived += handler;
runner.OutputReceived -= handler;
Assert.AreEqual(0, runner.Run());
Assert.IsFalse(outputReceived);
}
}
public void VerifyConstruction1()
{
var runner = new AssemblyRunner(new TestLogger(), "test");
Assert.That(runner.NUnitFilter.IsEmpty, Is.True);
}
