public UnitTestResult GetLocalTestResult (TestResult t) { UnitTestResult res = new UnitTestResult (); res.Message = t.Message; if (t is TestSuiteResult) { int s=0, f=0, i=0; CountResults ((TestSuiteResult)t, ref s, ref f, ref i); res.TotalFailures = f; res.TotalSuccess = s; res.TotalIgnored = i; if (f > 0) res.Status |= ResultStatus.Failure; if (s > 0) res.Status |= ResultStatus.Success; if (i > 0) res.Status |= ResultStatus.Ignored; } else { if (t.IsFailure) { res.Status = ResultStatus.Failure; res.TotalFailures = 1; } else if (!t.Executed) { res.Status = ResultStatus.Ignored; res.TotalIgnored = 1; } else { res.Status = ResultStatus.Success; res.TotalSuccess = 1; } if (string.IsNullOrEmpty (res.Message)) { if (t.IsFailure) res.Message = GettextCatalog.GetString ("Test failed"); else if (!t.Executed) res.Message = GettextCatalog.GetString ("Test ignored"); else { res.Message = GettextCatalog.GetString ("Test successful") + "\n\n"; res.Message += GettextCatalog.GetString ("Execution time: {0:0.00}ms", t.Time); } } } res.StackTrace = t.StackTrace; res.Time = TimeSpan.FromSeconds (t.Time); if (consoleOutput != null) { res.ConsoleOutput = consoleOutput.ToString (); res.ConsoleError = consoleError.ToString (); consoleOutput = null; consoleError = null; } return res; }
void IRemoteEventListener.TestFinished (string test, UnitTestResult result) { if (Canceled) return; if (singleTestRun) { SingleTestResult = result; return; } UnitTest t = GetLocalTest (test); if (t == null) return; t.RegisterResult (context, result); context.Monitor.EndTest (t, result); t.Status = TestStatus.Ready; runningTest = null; }
void ProcessResult (UnitTestResult res) { if (string.IsNullOrEmpty (res.Message)) { if (res.IsFailure) res.Message = GettextCatalog.GetString ("Test failed"); else if (res.IsNotRun) res.Message = GettextCatalog.GetString ("Test ignored"); else { res.Message = GettextCatalog.GetString ("Test successful") + "\n\n"; res.Message += GettextCatalog.GetString ("Execution time: {0:0.00}ms", res.Time.TotalMilliseconds); } } }
public UnitTestResult GetLocalTestResult (TestResult t) { UnitTestResult res = new UnitTestResult (); var summary = new ResultSummarizer (t); res.Failures = summary.Failures; res.Errors = summary.Errors; res.Ignored = summary.Ignored; res.Inconclusive = summary.Inconclusive; res.NotRunnable = summary.NotRunnable; res.Passed = summary.Passed; res.StackTrace = t.StackTrace; res.Time = TimeSpan.FromSeconds (t.Time); res.Message = t.Message; if (string.IsNullOrEmpty (res.Message)) { if (res.IsFailure) res.Message = GettextCatalog.GetString ("Test failed"); else if (!t.Executed) res.Message = GettextCatalog.GetString ("Test ignored"); else { res.Message = GettextCatalog.GetString ("Test successful") + "\n\n"; res.Message += GettextCatalog.GetString ("Execution time: {0:0.00}ms", t.Time); } } if (consoleOutput != null) { res.ConsoleOutput = consoleOutput.ToString (); res.ConsoleError = consoleError.ToString (); consoleOutput = null; consoleError = null; } return res; }
UnitTestResult ReportXmlResult(IRemoteEventListener listener, XElement elem, string testPrefix, bool macunitStyle) { UnitTestResult result = new UnitTestResult(); var time = (string)elem.Attribute("time"); if (time != null) { result.Time = TimeSpan.FromSeconds(double.Parse(time, CultureInfo.InvariantCulture)); } result.TestDate = DateTime.Now; var reason = elem.Element("reason"); if (reason != null) { result.Message = (string)reason; } var failure = elem.Element("failure"); if (failure != null) { var msg = failure.Element("message"); if (msg != null) { result.Message = (string)msg; } var stack = failure.Element("stack-trace"); if (stack != null) { result.StackTrace = (string)stack; } } switch ((string)elem.Attribute("result")) { case "Error": case "Failure": result.Status = ResultStatus.Failure; break; case "Success": result.Status = ResultStatus.Success; break; case "Ignored": result.Status = ResultStatus.Ignored; break; default: result.Status = ResultStatus.Inconclusive; break; } if (elem.Name == "test-suite") { // nunitlite does not emit <test-suite type="Namespace" elements so we need to fake // them by deconstructing the full type name and emitting the suite started events manually var names = new List <string> (); if (!macunitStyle || (string)elem.Attribute("type") == "Assembly") { names.Add("<root>"); } else { names.AddRange(elem.Attribute("name").Value.Split('.')); } for (int i = 0; i < names.Count; i++) { listener.SuiteStarted(testPrefix + string.Join(".", names.Take(i + 1))); } var name = (string)elem.Attribute("type") == "Assembly" ? "<root>" : (string)elem.Attribute("name"); var cts = elem.Element("results"); if (cts != null) { foreach (var ct in cts.Elements()) { var r = ReportXmlResult(listener, ct, name != "<root>" ? testPrefix + name + "." : "", macunitStyle); result.Add(r); } } for (int i = 0; i < names.Count; i++) { listener.SuiteFinished(testPrefix + string.Join(".", names.Take(i + 1)), result); } } else { string name = (string)elem.Attribute("name"); switch (result.Status) { case ResultStatus.Success: result.Passed++; break; case ResultStatus.Failure: result.Failures++; break; case ResultStatus.Ignored: result.Ignored++; break; case ResultStatus.Inconclusive: result.Inconclusive++; break; } listener.TestStarted(name); listener.TestFinished(name, result); } return(result); }
UnitTestResult ReportXmlResult (IRemoteEventListener listener, XElement elem, string testPrefix, bool macunitStyle) { UnitTestResult result = new UnitTestResult (); var time = (string)elem.Attribute ("time"); if (time != null) result.Time = TimeSpan.FromSeconds (double.Parse (time, CultureInfo.InvariantCulture)); result.TestDate = DateTime.Now; var reason = elem.Element ("reason"); if (reason != null) result.Message = (string) reason; var failure = elem.Element ("failure"); if (failure != null) { var msg = failure.Element ("message"); if (msg != null) result.Message = (string)msg; var stack = failure.Element ("stack-trace"); if (stack != null) result.StackTrace = (string)stack; } switch ((string)elem.Attribute ("result")) { case "Error": case "Failure": result.Status = ResultStatus.Failure; break; case "Success": result.Status = ResultStatus.Success; break; case "Ignored": result.Status = ResultStatus.Ignored; break; default: result.Status = ResultStatus.Inconclusive; break; } if (elem.Name == "test-suite") { // nunitlite does not emit <test-suite type="Namespace" elements so we need to fake // them by deconstructing the full type name and emitting the suite started events manually var names = new List<string> (); if (!macunitStyle || (string)elem.Attribute ("type") == "Assembly") names.Add ("<root>"); else names.AddRange (elem.Attribute ("name").Value.Split ('.')); for (int i = 0; i < names.Count; i ++) listener.SuiteStarted (testPrefix + string.Join (".", names.Take (i + 1))); var name = (string)elem.Attribute ("type") == "Assembly" ? "<root>" : (string) elem.Attribute ("name"); var cts = elem.Element ("results"); if (cts != null) { foreach (var ct in cts.Elements ()) { var r = ReportXmlResult (listener, ct, name != "<root>" ? testPrefix + name + "." : "", macunitStyle); result.Add (r); } } for (int i = 0; i < names.Count; i ++) listener.SuiteFinished (testPrefix + string.Join (".", names.Take (i + 1)), result); } else { string name = (string)elem.Attribute ("name"); switch (result.Status) { case ResultStatus.Success: result.Passed++; break; case ResultStatus.Failure: result.Failures++; break; case ResultStatus.Ignored: result.Ignored++; break; case ResultStatus.Inconclusive: result.Inconclusive++; break; } listener.TestStarted (name); listener.TestFinished (name, result); } return result; }
internal UnitTestResult RunUnitTest(UnitTest test, string suiteName, string pathName, string testName, TestContext testContext) { var runnerExe = GetCustomConsoleRunnerCommand(); if (runnerExe != null) { return(RunWithConsoleRunner(runnerExe, test, suiteName, pathName, testName, testContext)); } var console = testContext.ExecutionContext.ConsoleFactory.CreateConsole( OperationConsoleFactory.CreateConsoleOptions.Default.WithTitle(GettextCatalog.GetString("Unit Tests"))); ExternalTestRunner runner = new ExternalTestRunner(); runner.Connect(NUnitVersion, testContext.ExecutionContext.ExecutionHandler, console).Wait(); LocalTestMonitor localMonitor = new LocalTestMonitor(testContext, test, suiteName, testName != null); string[] filter = null; if (test != null) { if (test is UnitTestGroup && NUnitVersion == NUnitVersion.NUnit2) { filter = CollectTests((UnitTestGroup)test); } else if (test.TestId != null) { filter = new string [] { test.TestId }; } } RunData rd = new RunData(); rd.Runner = runner; rd.Test = this; rd.LocalMonitor = localMonitor; var cancelReg = testContext.Monitor.CancellationToken.Register(rd.Cancel); UnitTestResult result; var crashLogFile = Path.GetTempFileName(); try { if (string.IsNullOrEmpty(AssemblyPath)) { string msg = GettextCatalog.GetString("Could not get a valid path to the assembly. There may be a conflict in the project configurations."); throw new Exception(msg); } string testRunnerAssembly, testRunnerType; GetCustomTestRunner(out testRunnerAssembly, out testRunnerType); testContext.Monitor.CancellationToken.ThrowIfCancellationRequested(); result = runner.Run(localMonitor, filter, AssemblyPath, "", new List <string> (SupportAssemblies), testRunnerType, testRunnerAssembly, crashLogFile).Result; if (testName != null) { result = localMonitor.SingleTestResult; } ReportCrash(testContext, crashLogFile); } catch (Exception ex) { if (ReportCrash(testContext, crashLogFile)) { result = UnitTestResult.CreateFailure(GettextCatalog.GetString("Unhandled exception"), null); } else if (!localMonitor.Canceled) { LoggingService.LogError(ex.ToString()); if (localMonitor.RunningTest != null) { RuntimeErrorCleanup(testContext, localMonitor.RunningTest, ex); } else { testContext.Monitor.ReportRuntimeError(null, ex); throw; } result = UnitTestResult.CreateFailure(ex); } else { result = UnitTestResult.CreateFailure(GettextCatalog.GetString("Canceled"), null); } } finally { // Dispose the runner before the console, to make sure the console is available until the runner is disposed. runner.Disconnect().Wait(); runner.Dispose(); if (console != null) { console.Dispose(); } cancelReg.Dispose(); File.Delete(crashLogFile); } return(result); }
public void RegisterResult(string configuration, UnitTest test, UnitTestResult result) => cache.Add(test, result);
internal UnitTestResult[] Execute() { string initLogs = string.Empty; string initTrace = string.Empty; string initErrorLogs = string.Empty; string inittestContextMessages = string.Empty; UnitTestResult[] result = null; string ignoreMessage = null; var isIgnoreAttributeOnClass = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.Parent.ClassType, typeof(UTF.IgnoreAttribute), false); var isIgnoreAttributeOnMethod = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.TestMethod, typeof(UTF.IgnoreAttribute), false); if (isIgnoreAttributeOnClass) { ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.Parent.ClassType.GetTypeInfo()); } if (string.IsNullOrEmpty(ignoreMessage) && isIgnoreAttributeOnMethod) { ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.TestMethod); } if (isIgnoreAttributeOnClass || isIgnoreAttributeOnMethod) { return(new[] { new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage) }); } try { using (LogMessageListener logListener = new LogMessageListener(this.captureDebugTraces)) { try { // Run the assembly and class Initialize methods if required. // Assembly or class initialize can throw exceptions in which case we need to ensure that we fail the test. this.testMethodInfo.Parent.Parent.RunAssemblyInitialize(this.testContext.Context); this.testMethodInfo.Parent.RunClassInitialize(this.testContext.Context); } finally { initLogs = logListener.StandardOutput; initTrace = logListener.DebugTrace; initErrorLogs = logListener.StandardError; inittestContextMessages = this.testContext.GetAndClearDiagnosticMessages(); } } // Listening to log messages when running the test method with its Test Initialize and cleanup later on in the stack. // This allows us to differentiate logging when data driven methods are used. result = this.RunTestMethod(); } catch (TestFailedException ex) { result = new[] { new UnitTestResult(ex) }; } catch (Exception ex) { if (result == null || result.Length == 0) { result = new[] { new UnitTestResult() }; } var newResult = new UnitTestResult(new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation())); newResult.StandardOut = result[result.Length - 1].StandardOut; newResult.StandardError = result[result.Length - 1].StandardError; newResult.DebugTrace = result[result.Length - 1].DebugTrace; newResult.TestContextMessages = result[result.Length - 1].TestContextMessages; newResult.Duration = result[result.Length - 1].Duration; result[result.Length - 1] = newResult; } finally { var firstResult = result[0]; firstResult.StandardOut = initLogs + firstResult.StandardOut; firstResult.StandardError = initErrorLogs + firstResult.StandardError; firstResult.DebugTrace = initTrace + firstResult.DebugTrace; firstResult.TestContextMessages = inittestContextMessages + firstResult.TestContextMessages; } return(result); }
public UnitTestResultReport(UnitTestResult result) { Result = result; if (!string.IsNullOrEmpty(Result.Output.StdOut)) { if (Result.Output.StdOut.Contains("-> done") || Result.Output.StdOut.Contains("-> error") || Result.Output.StdOut.Contains("-> skipped")) { //set cucumber output cucumberStdOut = new List <KeyValuePair <string, string> >(); var rows = Result.Output.StdOut.Split(new char[] { '\n' }); for (int i = 1; i < rows.Length; i++) { if (rows[i].StartsWith("-> done")) { cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i - 1], "success")); cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i], "success")); } else if (rows[i].StartsWith("-> error")) { cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i - 1], "danger")); cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i], "danger")); } else if (rows[i].StartsWith("-> skipped")) { cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i - 1], "warning")); cucumberStdOut.Add(new KeyValuePair <string, string>(rows[i], "warning")); } } } else { //set standard output StdOutRows = Result.Output.StdOut.Split(new char[] { '\n' }).ToList(); } } if (!string.IsNullOrEmpty(Result.Output.StdErr)) { StdErrRows = Result.Output.StdErr.Split(new char[] { '\n' }).ToList(); } if (result.Output.ErrorInfo != null) { if (!string.IsNullOrEmpty(Result.Output.ErrorInfo.Message)) { //set MessageRows ErrorMessageRows = Result.Output.ErrorInfo.Message.Split(new char[] { '\n' }).ToList(); } if (!string.IsNullOrEmpty(Result.Output.ErrorInfo.StackTrace)) { //set StackTraceRows ErrorStackTraceRows = Result.Output.ErrorInfo.StackTrace.Split(new char[] { '\n' }).ToList(); } } ErrorImage = null; }
private static void ReportTestMessages(SpecSession session, Dictionary <int, UnitTestResult> nodeResults, UnitTestResult specResult, SpecLog log) { foreach (var(_, message) in session.Messages) { var textMessage = $"[{message.When:G}] {message.Message}"; Output output; if (nodeResults.TryGetValue(message.NodeIndex, out var result)) { if (result.Output == null) { result.Output = new Output(); } output = result.Output; } else { if (specResult.Output == null) { specResult.Output = new Output(); } output = specResult.Output; } output.StdOut.Add(textMessage); output.DebugTrace.Add(textMessage); } specResult.Output = specResult.Output ?? new Output(); specResult.Output.StdErr.AddRange(log.AggregatedTimelineLog); }
public void CreateFailure(Exception ex) { TestResult = UnitTestResult.CreateFailure(ex); TaskSource.SetResult(TestResult); }
void UpdateCounts(UnitTestResult result) { UpdateCounts(result, result); }
void IRemoteEventListener.SuiteFinished (string suite, UnitTestResult result) { if (singleTestRun || Canceled) return; UnitTest t = GetLocalTest (suite); if (t == null) return; t.RegisterResult (context, result); t.Status = TestStatus.Ready; context.Monitor.EndTest (t, result); }
UnitTestResult RunWithConsoleRunner(ProcessExecutionCommand cmd, UnitTest test, string suiteName, string pathName, string testName, TestContext testContext) { var outFile = Path.GetTempFileName(); var xmlOutputConsole = new LocalConsole(); var appDebugOutputConsole = testContext.ExecutionContext.ConsoleFactory.CreateConsole( OperationConsoleFactory.CreateConsoleOptions.Default.WithTitle(GettextCatalog.GetString("Unit Tests"))); OperationConsole cons; if (appDebugOutputConsole != null) { cons = new MultipleOperationConsoles(appDebugOutputConsole, xmlOutputConsole); } else { cons = xmlOutputConsole; } try { MonoDevelop.UnitTesting.NUnit.External.TcpTestListener tcpListener = null; LocalTestMonitor localMonitor = new LocalTestMonitor(testContext, test, suiteName, testName != null); if (!string.IsNullOrEmpty(cmd.Arguments)) { cmd.Arguments += " "; } cmd.Arguments += "\"-xml=" + outFile + "\" " + AssemblyPath; bool automaticUpdates = cmd.Command != null && (cmd.Command.Contains("GuiUnit") || (cmd.Command.Contains("mdtool.exe") && cmd.Arguments.Contains("run-md-tests"))); if (!string.IsNullOrEmpty(pathName)) { cmd.Arguments += " -run=\"" + test.TestId.Replace("\"", "\\\"") + "\""; } if (automaticUpdates) { tcpListener = new MonoDevelop.UnitTesting.NUnit.External.TcpTestListener(localMonitor, suiteName); cmd.Arguments += " -port=" + tcpListener.Port; } cmd.WorkingDirectory = Path.GetDirectoryName(AssemblyPath); // Note that we always dispose the tcp listener as we don't want it listening // forever if the test runner does not try to connect to it using (tcpListener) { var handler = testContext.ExecutionContext.ExecutionHandler; if (handler == null) { handler = Runtime.ProcessService.DefaultExecutionHandler; } var p = handler.Execute(cmd, cons); using (testContext.Monitor.CancellationToken.Register(p.Cancel)) p.Task.Wait(); if (new FileInfo(outFile).Length == 0) { throw new Exception("Command failed"); } } // mdtool.exe does not necessarily guarantee we get automatic updates. It just guarantees // that if guiunit is being used then it will give us updates. If you have a regular test // assembly compiled against nunit.framework.dll if (automaticUpdates && tcpListener.HasReceivedConnection) { if (testName != null) { return(localMonitor.SingleTestResult); } return(test.GetLastResult()); } XDocument doc = XDocument.Load(outFile); if (doc.Root != null) { var root = doc.Root.Elements("test-suite").FirstOrDefault(); if (root != null) { xmlOutputConsole.SetDone(); var ot = xmlOutputConsole.OutReader.ReadToEnd(); var et = xmlOutputConsole.ErrorReader.ReadToEnd(); testContext.Monitor.WriteGlobalLog(ot); if (!string.IsNullOrEmpty(et)) { testContext.Monitor.WriteGlobalLog("ERROR:\n"); testContext.Monitor.WriteGlobalLog(et); } bool macunitStyle = doc.Root.Element("environment") != null && doc.Root.Element("environment").Attribute("macunit-version") != null; var result = ReportXmlResult(localMonitor, root, "", macunitStyle); if (testName != null) { result = localMonitor.SingleTestResult; } return(result); } } throw new Exception("Test results could not be parsed."); } catch (Exception ex) { xmlOutputConsole.SetDone(); var ot = xmlOutputConsole.OutReader.ReadToEnd(); var et = xmlOutputConsole.ErrorReader.ReadToEnd(); testContext.Monitor.WriteGlobalLog(ot); if (!string.IsNullOrEmpty(et)) { testContext.Monitor.WriteGlobalLog("ERROR:\n"); testContext.Monitor.WriteGlobalLog(et); } testContext.Monitor.ReportRuntimeError("Test execution failed.\n" + ot + "\n" + et, ex); return(UnitTestResult.CreateIgnored("Test execution failed")); } finally { File.Delete(outFile); cons.Dispose(); } }
void AddTestResult (UnitTestResult res) { foreach (var r in suiteStack) r.Item2.Add (res); }
public static UnitTest GetUnitTest(this UnitTestResult unitTestResult, TrxDocument trxDocument) { return(trxDocument.TestDefinitions.FirstOrDefault(x => x.Id == unitTestResult.TestId)); }