public UnitTestResult GetLocalTestResult (TestResult t)
{
UnitTestResult res = new UnitTestResult ();
res.Message = t.Message;
if (t is TestSuiteResult) {
int s=0, f=0, i=0;
CountResults ((TestSuiteResult)t, ref s, ref f, ref i);
res.TotalFailures = f;
res.TotalSuccess = s;
res.TotalIgnored = i;
if (f > 0)
res.Status |= ResultStatus.Failure;
if (s > 0)
res.Status |= ResultStatus.Success;
if (i > 0)
res.Status |= ResultStatus.Ignored;
} else {
if (t.IsFailure) {
res.Status = ResultStatus.Failure;
res.TotalFailures = 1;
}
else if (!t.Executed) {
res.Status = ResultStatus.Ignored;
res.TotalIgnored = 1;
}
else {
res.Status = ResultStatus.Success;
res.TotalSuccess = 1;
}
if (string.IsNullOrEmpty (res.Message)) {
if (t.IsFailure)
res.Message = GettextCatalog.GetString ("Test failed");
else if (!t.Executed)
res.Message = GettextCatalog.GetString ("Test ignored");
else {
res.Message = GettextCatalog.GetString ("Test successful") + "\n\n";
res.Message += GettextCatalog.GetString ("Execution time: {0:0.00}ms", t.Time);
}
}
}
res.StackTrace = t.StackTrace;
res.Time = TimeSpan.FromSeconds (t.Time);
if (consoleOutput != null) {
res.ConsoleOutput = consoleOutput.ToString ();
res.ConsoleError = consoleError.ToString ();
consoleOutput = null;
consoleError = null;
}
return res;
}
void IRemoteEventListener.TestFinished (string test, UnitTestResult result)
{
if (Canceled)
return;
if (singleTestRun) {
SingleTestResult = result;
return;
}
UnitTest t = GetLocalTest (test);
if (t == null)
return;
t.RegisterResult (context, result);
context.Monitor.EndTest (t, result);
t.Status = TestStatus.Ready;
runningTest = null;
}
void ProcessResult (UnitTestResult res)
{
if (string.IsNullOrEmpty (res.Message)) {
if (res.IsFailure)
res.Message = GettextCatalog.GetString ("Test failed");
else if (res.IsNotRun)
res.Message = GettextCatalog.GetString ("Test ignored");
else {
res.Message = GettextCatalog.GetString ("Test successful") + "\n\n";
res.Message += GettextCatalog.GetString ("Execution time: {0:0.00}ms", res.Time.TotalMilliseconds);
}
}
}
public UnitTestResult GetLocalTestResult (TestResult t)
{
UnitTestResult res = new UnitTestResult ();
var summary = new ResultSummarizer (t);
res.Failures = summary.Failures;
res.Errors = summary.Errors;
res.Ignored = summary.Ignored;
res.Inconclusive = summary.Inconclusive;
res.NotRunnable = summary.NotRunnable;
res.Passed = summary.Passed;
res.StackTrace = t.StackTrace;
res.Time = TimeSpan.FromSeconds (t.Time);
res.Message = t.Message;
if (string.IsNullOrEmpty (res.Message)) {
if (res.IsFailure)
res.Message = GettextCatalog.GetString ("Test failed");
else if (!t.Executed)
res.Message = GettextCatalog.GetString ("Test ignored");
else {
res.Message = GettextCatalog.GetString ("Test successful") + "\n\n";
res.Message += GettextCatalog.GetString ("Execution time: {0:0.00}ms", t.Time);
}
}
if (consoleOutput != null) {
res.ConsoleOutput = consoleOutput.ToString ();
res.ConsoleError = consoleError.ToString ();
consoleOutput = null;
consoleError = null;
}
return res;
}
UnitTestResult ReportXmlResult(IRemoteEventListener listener, XElement elem, string testPrefix, bool macunitStyle)
{
UnitTestResult result = new UnitTestResult();
var time = (string)elem.Attribute("time");
if (time != null)
{
result.Time = TimeSpan.FromSeconds(double.Parse(time, CultureInfo.InvariantCulture));
}
result.TestDate = DateTime.Now;
var reason = elem.Element("reason");
if (reason != null)
{
result.Message = (string)reason;
}
var failure = elem.Element("failure");
if (failure != null)
{
var msg = failure.Element("message");
if (msg != null)
{
result.Message = (string)msg;
}
var stack = failure.Element("stack-trace");
if (stack != null)
{
result.StackTrace = (string)stack;
}
}
switch ((string)elem.Attribute("result"))
{
case "Error":
case "Failure":
result.Status = ResultStatus.Failure;
break;
case "Success":
result.Status = ResultStatus.Success;
break;
case "Ignored":
result.Status = ResultStatus.Ignored;
break;
default:
result.Status = ResultStatus.Inconclusive;
break;
}
if (elem.Name == "test-suite")
{
// nunitlite does not emit <test-suite type="Namespace" elements so we need to fake
// them by deconstructing the full type name and emitting the suite started events manually
var names = new List <string> ();
if (!macunitStyle || (string)elem.Attribute("type") == "Assembly")
{
names.Add("<root>");
}
else
{
names.AddRange(elem.Attribute("name").Value.Split('.'));
}
for (int i = 0; i < names.Count; i++)
{
listener.SuiteStarted(testPrefix + string.Join(".", names.Take(i + 1)));
}
var name = (string)elem.Attribute("type") == "Assembly" ? "<root>" : (string)elem.Attribute("name");
var cts = elem.Element("results");
if (cts != null)
{
foreach (var ct in cts.Elements())
{
var r = ReportXmlResult(listener, ct, name != "<root>" ? testPrefix + name + "." : "", macunitStyle);
result.Add(r);
}
}
for (int i = 0; i < names.Count; i++)
{
listener.SuiteFinished(testPrefix + string.Join(".", names.Take(i + 1)), result);
}
}
else
{
string name = (string)elem.Attribute("name");
switch (result.Status)
{
case ResultStatus.Success:
result.Passed++;
break;
case ResultStatus.Failure:
result.Failures++;
break;
case ResultStatus.Ignored:
result.Ignored++;
break;
case ResultStatus.Inconclusive:
result.Inconclusive++;
break;
}
listener.TestStarted(name);
listener.TestFinished(name, result);
}
return(result);
}
UnitTestResult ReportXmlResult (IRemoteEventListener listener, XElement elem, string testPrefix, bool macunitStyle)
{
UnitTestResult result = new UnitTestResult ();
var time = (string)elem.Attribute ("time");
if (time != null)
result.Time = TimeSpan.FromSeconds (double.Parse (time, CultureInfo.InvariantCulture));
result.TestDate = DateTime.Now;
var reason = elem.Element ("reason");
if (reason != null)
result.Message = (string) reason;
var failure = elem.Element ("failure");
if (failure != null) {
var msg = failure.Element ("message");
if (msg != null)
result.Message = (string)msg;
var stack = failure.Element ("stack-trace");
if (stack != null)
result.StackTrace = (string)stack;
}
switch ((string)elem.Attribute ("result")) {
case "Error":
case "Failure":
result.Status = ResultStatus.Failure;
break;
case "Success":
result.Status = ResultStatus.Success;
break;
case "Ignored":
result.Status = ResultStatus.Ignored;
break;
default:
result.Status = ResultStatus.Inconclusive;
break;
}
if (elem.Name == "test-suite") {
// nunitlite does not emit <test-suite type="Namespace" elements so we need to fake
// them by deconstructing the full type name and emitting the suite started events manually
var names = new List<string> ();
if (!macunitStyle || (string)elem.Attribute ("type") == "Assembly")
names.Add ("<root>");
else
names.AddRange (elem.Attribute ("name").Value.Split ('.'));
for (int i = 0; i < names.Count; i ++)
listener.SuiteStarted (testPrefix + string.Join (".", names.Take (i + 1)));
var name = (string)elem.Attribute ("type") == "Assembly" ? "<root>" : (string) elem.Attribute ("name");
var cts = elem.Element ("results");
if (cts != null) {
foreach (var ct in cts.Elements ()) {
var r = ReportXmlResult (listener, ct, name != "<root>" ? testPrefix + name + "." : "", macunitStyle);
result.Add (r);
}
}
for (int i = 0; i < names.Count; i ++)
listener.SuiteFinished (testPrefix + string.Join (".", names.Take (i + 1)), result);
} else {
string name = (string)elem.Attribute ("name");
switch (result.Status) {
case ResultStatus.Success:
result.Passed++;
break;
case ResultStatus.Failure:
result.Failures++;
break;
case ResultStatus.Ignored:
result.Ignored++;
break;
case ResultStatus.Inconclusive:
result.Inconclusive++;
break;
}
listener.TestStarted (name);
listener.TestFinished (name, result);
}
return result;
}
internal UnitTestResult RunUnitTest(UnitTest test, string suiteName, string pathName, string testName, TestContext testContext)
{
var runnerExe = GetCustomConsoleRunnerCommand();
if (runnerExe != null)
{
return(RunWithConsoleRunner(runnerExe, test, suiteName, pathName, testName, testContext));
}
var console = testContext.ExecutionContext.ConsoleFactory.CreateConsole(
OperationConsoleFactory.CreateConsoleOptions.Default.WithTitle(GettextCatalog.GetString("Unit Tests")));
ExternalTestRunner runner = new ExternalTestRunner();
runner.Connect(NUnitVersion, testContext.ExecutionContext.ExecutionHandler, console).Wait();
LocalTestMonitor localMonitor = new LocalTestMonitor(testContext, test, suiteName, testName != null);
string[] filter = null;
if (test != null)
{
if (test is UnitTestGroup && NUnitVersion == NUnitVersion.NUnit2)
{
filter = CollectTests((UnitTestGroup)test);
}
else if (test.TestId != null)
{
filter = new string [] { test.TestId };
}
}
RunData rd = new RunData();
rd.Runner = runner;
rd.Test = this;
rd.LocalMonitor = localMonitor;
var cancelReg = testContext.Monitor.CancellationToken.Register(rd.Cancel);
UnitTestResult result;
var crashLogFile = Path.GetTempFileName();
try {
if (string.IsNullOrEmpty(AssemblyPath))
{
string msg = GettextCatalog.GetString("Could not get a valid path to the assembly. There may be a conflict in the project configurations.");
throw new Exception(msg);
}
string testRunnerAssembly, testRunnerType;
GetCustomTestRunner(out testRunnerAssembly, out testRunnerType);
testContext.Monitor.CancellationToken.ThrowIfCancellationRequested();
result = runner.Run(localMonitor, filter, AssemblyPath, "", new List <string> (SupportAssemblies), testRunnerType, testRunnerAssembly, crashLogFile).Result;
if (testName != null)
{
result = localMonitor.SingleTestResult;
}
ReportCrash(testContext, crashLogFile);
} catch (Exception ex) {
if (ReportCrash(testContext, crashLogFile))
{
result = UnitTestResult.CreateFailure(GettextCatalog.GetString("Unhandled exception"), null);
}
else if (!localMonitor.Canceled)
{
LoggingService.LogError(ex.ToString());
if (localMonitor.RunningTest != null)
{
RuntimeErrorCleanup(testContext, localMonitor.RunningTest, ex);
}
else
{
testContext.Monitor.ReportRuntimeError(null, ex);
throw;
}
result = UnitTestResult.CreateFailure(ex);
}
else
{
result = UnitTestResult.CreateFailure(GettextCatalog.GetString("Canceled"), null);
}
} finally {
// Dispose the runner before the console, to make sure the console is available until the runner is disposed.
runner.Disconnect().Wait();
runner.Dispose();
if (console != null)
{
console.Dispose();
}
cancelReg.Dispose();
File.Delete(crashLogFile);
}
return(result);
}
public void RegisterResult(string configuration, UnitTest test, UnitTestResult result) => cache.Add(test, result);
internal UnitTestResult[] Execute()
{
string initLogs = string.Empty;
string initTrace = string.Empty;
string initErrorLogs = string.Empty;
string inittestContextMessages = string.Empty;
UnitTestResult[] result = null;
string ignoreMessage = null;
var isIgnoreAttributeOnClass = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.Parent.ClassType, typeof(UTF.IgnoreAttribute), false);
var isIgnoreAttributeOnMethod = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.TestMethod, typeof(UTF.IgnoreAttribute), false);
if (isIgnoreAttributeOnClass)
{
ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.Parent.ClassType.GetTypeInfo());
}
if (string.IsNullOrEmpty(ignoreMessage) && isIgnoreAttributeOnMethod)
{
ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.TestMethod);
}
if (isIgnoreAttributeOnClass || isIgnoreAttributeOnMethod)
{
return(new[] { new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage) });
}
try
{
using (LogMessageListener logListener = new LogMessageListener(this.captureDebugTraces))
{
try
{
// Run the assembly and class Initialize methods if required.
// Assembly or class initialize can throw exceptions in which case we need to ensure that we fail the test.
this.testMethodInfo.Parent.Parent.RunAssemblyInitialize(this.testContext.Context);
this.testMethodInfo.Parent.RunClassInitialize(this.testContext.Context);
}
finally
{
initLogs = logListener.StandardOutput;
initTrace = logListener.DebugTrace;
initErrorLogs = logListener.StandardError;
inittestContextMessages = this.testContext.GetAndClearDiagnosticMessages();
}
}
// Listening to log messages when running the test method with its Test Initialize and cleanup later on in the stack.
// This allows us to differentiate logging when data driven methods are used.
result = this.RunTestMethod();
}
catch (TestFailedException ex)
{
result = new[] { new UnitTestResult(ex) };
}
catch (Exception ex)
{
if (result == null || result.Length == 0)
{
result = new[] { new UnitTestResult() };
}
var newResult =
new UnitTestResult(new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()));
newResult.StandardOut = result[result.Length - 1].StandardOut;
newResult.StandardError = result[result.Length - 1].StandardError;
newResult.DebugTrace = result[result.Length - 1].DebugTrace;
newResult.TestContextMessages = result[result.Length - 1].TestContextMessages;
newResult.Duration = result[result.Length - 1].Duration;
result[result.Length - 1] = newResult;
}
finally
{
var firstResult = result[0];
firstResult.StandardOut = initLogs + firstResult.StandardOut;
firstResult.StandardError = initErrorLogs + firstResult.StandardError;
firstResult.DebugTrace = initTrace + firstResult.DebugTrace;
firstResult.TestContextMessages = inittestContextMessages + firstResult.TestContextMessages;
}
return(result);
}
public UnitTestResultReport(UnitTestResult result)
{
Result = result;
if (!string.IsNullOrEmpty(Result.Output.StdOut))
{
if (Result.Output.StdOut.Contains("-> done") ||
Result.Output.StdOut.Contains("-> error") ||
Result.Output.StdOut.Contains("-> skipped"))
{
//set cucumber output
cucumberStdOut = new List <KeyValuePair <string, string> >();
var rows = Result.Output.StdOut.Split(new char[] { '\n' });
for (int i = 1; i < rows.Length; i++)
{
if (rows[i].StartsWith("-> done"))
{
cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i - 1], "success"));
cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i], "success"));
}
else if (rows[i].StartsWith("-> error"))
{
cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i - 1], "danger"));
cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i], "danger"));
}
else if (rows[i].StartsWith("-> skipped"))
{
cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i - 1], "warning"));
cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i], "warning"));
}
}
}
else
{
//set standard output
StdOutRows = Result.Output.StdOut.Split(new char[] { '\n' }).ToList();
}
}
if (!string.IsNullOrEmpty(Result.Output.StdErr))
{
StdErrRows = Result.Output.StdErr.Split(new char[] { '\n' }).ToList();
}
if (result.Output.ErrorInfo != null)
{
if (!string.IsNullOrEmpty(Result.Output.ErrorInfo.Message))
{
//set MessageRows
ErrorMessageRows = Result.Output.ErrorInfo.Message.Split(new char[] { '\n' }).ToList();
}
if (!string.IsNullOrEmpty(Result.Output.ErrorInfo.StackTrace))
{
//set StackTraceRows
ErrorStackTraceRows = Result.Output.ErrorInfo.StackTrace.Split(new char[] { '\n' }).ToList();
}
}
ErrorImage = null;
}
private static void ReportTestMessages(SpecSession session, Dictionary <int, UnitTestResult> nodeResults, UnitTestResult specResult, SpecLog log)
{
foreach (var(_, message) in session.Messages)
{
var textMessage = $"[{message.When:G}] {message.Message}";
Output output;
if (nodeResults.TryGetValue(message.NodeIndex, out var result))
{
if (result.Output == null)
{
result.Output = new Output();
}
output = result.Output;
}
else
{
if (specResult.Output == null)
{
specResult.Output = new Output();
}
output = specResult.Output;
}
output.StdOut.Add(textMessage);
output.DebugTrace.Add(textMessage);
}
specResult.Output = specResult.Output ?? new Output();
specResult.Output.StdErr.AddRange(log.AggregatedTimelineLog);
}
public void CreateFailure(Exception ex)
{
TestResult = UnitTestResult.CreateFailure(ex);
TaskSource.SetResult(TestResult);
}
void UpdateCounts(UnitTestResult result)
{
UpdateCounts(result, result);
}
void IRemoteEventListener.SuiteFinished (string suite, UnitTestResult result)
{
if (singleTestRun || Canceled)
return;
UnitTest t = GetLocalTest (suite);
if (t == null)
return;
t.RegisterResult (context, result);
t.Status = TestStatus.Ready;
context.Monitor.EndTest (t, result);
}
UnitTestResult RunWithConsoleRunner(ProcessExecutionCommand cmd, UnitTest test, string suiteName, string pathName, string testName, TestContext testContext)
{
var outFile = Path.GetTempFileName();
var xmlOutputConsole = new LocalConsole();
var appDebugOutputConsole = testContext.ExecutionContext.ConsoleFactory.CreateConsole(
OperationConsoleFactory.CreateConsoleOptions.Default.WithTitle(GettextCatalog.GetString("Unit Tests")));
OperationConsole cons;
if (appDebugOutputConsole != null)
{
cons = new MultipleOperationConsoles(appDebugOutputConsole, xmlOutputConsole);
}
else
{
cons = xmlOutputConsole;
}
try {
MonoDevelop.UnitTesting.NUnit.External.TcpTestListener tcpListener = null;
LocalTestMonitor localMonitor = new LocalTestMonitor(testContext, test, suiteName, testName != null);
if (!string.IsNullOrEmpty(cmd.Arguments))
{
cmd.Arguments += " ";
}
cmd.Arguments += "\"-xml=" + outFile + "\" " + AssemblyPath;
bool automaticUpdates = cmd.Command != null && (cmd.Command.Contains("GuiUnit") || (cmd.Command.Contains("mdtool.exe") && cmd.Arguments.Contains("run-md-tests")));
if (!string.IsNullOrEmpty(pathName))
{
cmd.Arguments += " -run=\"" + test.TestId.Replace("\"", "\\\"") + "\"";
}
if (automaticUpdates)
{
tcpListener = new MonoDevelop.UnitTesting.NUnit.External.TcpTestListener(localMonitor, suiteName);
cmd.Arguments += " -port=" + tcpListener.Port;
}
cmd.WorkingDirectory = Path.GetDirectoryName(AssemblyPath);
// Note that we always dispose the tcp listener as we don't want it listening
// forever if the test runner does not try to connect to it
using (tcpListener) {
var handler = testContext.ExecutionContext.ExecutionHandler;
if (handler == null)
{
handler = Runtime.ProcessService.DefaultExecutionHandler;
}
var p = handler.Execute(cmd, cons);
using (testContext.Monitor.CancellationToken.Register(p.Cancel))
p.Task.Wait();
if (new FileInfo(outFile).Length == 0)
{
throw new Exception("Command failed");
}
}
// mdtool.exe does not necessarily guarantee we get automatic updates. It just guarantees
// that if guiunit is being used then it will give us updates. If you have a regular test
// assembly compiled against nunit.framework.dll
if (automaticUpdates && tcpListener.HasReceivedConnection)
{
if (testName != null)
{
return(localMonitor.SingleTestResult);
}
return(test.GetLastResult());
}
XDocument doc = XDocument.Load(outFile);
if (doc.Root != null)
{
var root = doc.Root.Elements("test-suite").FirstOrDefault();
if (root != null)
{
xmlOutputConsole.SetDone();
var ot = xmlOutputConsole.OutReader.ReadToEnd();
var et = xmlOutputConsole.ErrorReader.ReadToEnd();
testContext.Monitor.WriteGlobalLog(ot);
if (!string.IsNullOrEmpty(et))
{
testContext.Monitor.WriteGlobalLog("ERROR:\n");
testContext.Monitor.WriteGlobalLog(et);
}
bool macunitStyle = doc.Root.Element("environment") != null && doc.Root.Element("environment").Attribute("macunit-version") != null;
var result = ReportXmlResult(localMonitor, root, "", macunitStyle);
if (testName != null)
{
result = localMonitor.SingleTestResult;
}
return(result);
}
}
throw new Exception("Test results could not be parsed.");
} catch (Exception ex) {
xmlOutputConsole.SetDone();
var ot = xmlOutputConsole.OutReader.ReadToEnd();
var et = xmlOutputConsole.ErrorReader.ReadToEnd();
testContext.Monitor.WriteGlobalLog(ot);
if (!string.IsNullOrEmpty(et))
{
testContext.Monitor.WriteGlobalLog("ERROR:\n");
testContext.Monitor.WriteGlobalLog(et);
}
testContext.Monitor.ReportRuntimeError("Test execution failed.\n" + ot + "\n" + et, ex);
return(UnitTestResult.CreateIgnored("Test execution failed"));
} finally {
File.Delete(outFile);
cons.Dispose();
}
}
void AddTestResult (UnitTestResult res)
{
foreach (var r in suiteStack)
r.Item2.Add (res);
}
public static UnitTest GetUnitTest(this UnitTestResult unitTestResult, TrxDocument trxDocument)
{
return(trxDocument.TestDefinitions.FirstOrDefault(x => x.Id == unitTestResult.TestId));
}
