public override void Invoke(object[] parameters, ITestResult result)
{
var x = Invoke(parameters);
if(ReturnType == typeof(void) || expectedResult.Matches(x))
result.Success();
else result.TestFailure(new Exception("\n" + string.Format(ExpectMessages.EqualFormat, x, expectedResult)));
}
private void Visit(ITestResult result)
{
if (result.Test is TestSuite)
{
foreach (ITestResult r in result.Children)
Visit(r);
}
else
{
testCount++;
switch (result.ResultState.Status)
{
case TestStatus.Skipped:
notRunCount++;
break;
case TestStatus.Failed:
if (result.ResultState == ResultState.Failure)
failureCount++;
else
errorCount++;
break;
default:
break;
}
return;
}
}
private void InitializeXmlFile(ITestResult result)
{
ResultSummary summary = new ResultSummary(result);
xmlWriter.WriteStartDocument(false);
xmlWriter.WriteComment("This file represents the results of running a test suite");
xmlWriter.WriteStartElement("test-results");
xmlWriter.WriteAttributeString("name", result.FullName);
xmlWriter.WriteAttributeString("total", summary.TestCount.ToString());
xmlWriter.WriteAttributeString("errors", summary.ErrorCount.ToString());
xmlWriter.WriteAttributeString("failures", summary.FailureCount.ToString());
var notRunTotal = summary.SkipCount + summary.FailureCount + summary.InvalidCount;
xmlWriter.WriteAttributeString("not-run", notRunTotal.ToString());
xmlWriter.WriteAttributeString("inconclusive", summary.InconclusiveCount.ToString());
xmlWriter.WriteAttributeString("ignored", summary.IgnoreCount.ToString());
xmlWriter.WriteAttributeString("skipped", summary.SkipCount.ToString());
xmlWriter.WriteAttributeString("invalid", summary.InvalidCount.ToString());
xmlWriter.WriteAttributeString("date", result.StartTime.ToString("yyyy-MM-dd"));
xmlWriter.WriteAttributeString("time", result.StartTime.ToString("HH:mm:ss"));
WriteEnvironment();
WriteCultureInfo();
}
/// <summary>
/// Writes a test result to a file
/// </summary>
/// <param name="result">The result to be written</param>
/// <param name="outputPath">Path to the file to which the result is written</param>
/// <param name="runSettings">A dictionary of settings used for this test run</param>
public void WriteResultFile(ITestResult result, string outputPath, IDictionary runSettings, TestFilter filter)
{
using (StreamWriter writer = new StreamWriter(outputPath, false, Encoding.UTF8))
{
WriteResultFile(result, writer, runSettings, filter);
}
}
public NewTestResultSummaryEventArgs(ITest test, ITestResult result, ITestOutcomeFilter outcomeFilter, ITestResultSummary summary)
{
Test = test;
Result = result;
OutcomeFilter = outcomeFilter;
Summary = summary;
}
public void TestFinished(ITestResult fullName)
{
foreach (ITestRunnerCallback callback in this.m_CallbackList)
{
callback.TestFinished(fullName);
}
}
public void TestFinished(ITestResult result)
{
#if PORTABLE
if(result.Test is TestMethod)
{
if(result.ResultState == ResultState.Inconclusive)
{
Console.ForegroundColor = ConsoleColor.Yellow;
}
else if(result.ResultState != ResultState.Success)
{
Console.ForegroundColor = ConsoleColor.Red;
}
else if(result.ResultState == ResultState.Success)
{
Console.ForegroundColor = ConsoleColor.Green;
}
Console.WriteLine(result.ResultState);
Console.ResetColor();
}
#else
Events += string.Format(":{0}>", result.ResultState);
#endif
}
/// <summary>
/// Called when a test has finished
/// </summary>
/// <param name="result">The result of the test</param>
public void TestFinished(ITestResult result)
{
string testName = result.Test.Name;
if (result.Test.IsSuite)
TC_TestSuiteFinished(testName);
else
switch (result.ResultState.Status)
{
case TestStatus.Passed:
TC_TestFinished(testName, result.Duration);
break;
case TestStatus.Inconclusive:
TC_TestIgnored(testName, "Inconclusive");
break;
case TestStatus.Skipped:
TC_TestIgnored(testName, result.Message);
break;
case TestStatus.Warning:
// TODO: No action at this time. May need to be added.
break;
case TestStatus.Failed:
TC_TestFailed(testName, result.Message, result.StackTrace);
TC_TestFinished(testName, result.Duration);
break;
}
}
public ResultViewModel(ITestResult result)
{
TestResult = result;
Name = result.Name;
Parent = result.Test.Parent.FullName;
Message = result.Message;
}
public void TestFinished(ITestResult fullName)
{
foreach (var unitTestRunnerCallback in m_CallbackList)
{
unitTestRunnerCallback.TestFinished(fullName);
}
}
/// <summary>
/// Constructs an instance of ResultReporter
/// </summary>
/// <param name="result">The top-level result being reported</param>
/// <param name="writer">A TextWriter to which the report is written</param>
public ResultReporter(ITestResult result, TextWriter writer)
{
this.result = result;
this.writer = writer;
this.summary = new ResultSummary(this.result);
}
private void InitializeXmlFile(ITestResult result)
{
xmlWriter.WriteStartDocument(false);
// In order to match the format used by NUnit 3.0, we
// wrap the entire result from the framework in a
// <test-run> element.
xmlWriter.WriteStartElement("test-run");
xmlWriter.WriteAttributeString("id", "2"); // TODO: Should not be hard-coded
xmlWriter.WriteAttributeString("name", result.Name);
xmlWriter.WriteAttributeString("fullname", result.FullName);
xmlWriter.WriteAttributeString("testcasecount", result.Test.TestCaseCount.ToString());
xmlWriter.WriteAttributeString("result", result.ResultState.Status.ToString());
if (result.ResultState.Label != string.Empty) // && result.ResultState.Label != ResultState.Status.ToString())
xmlWriter.WriteAttributeString("label", result.ResultState.Label);
xmlWriter.WriteAttributeString("start-time", result.StartTime.ToString("u"));
xmlWriter.WriteAttributeString("end-time", result.EndTime.ToString("u"));
xmlWriter.WriteAttributeString("duration", result.Duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
xmlWriter.WriteAttributeString("total", (result.PassCount + result.FailCount + result.SkipCount + result.InconclusiveCount).ToString());
xmlWriter.WriteAttributeString("passed", result.PassCount.ToString());
xmlWriter.WriteAttributeString("failed", result.FailCount.ToString());
xmlWriter.WriteAttributeString("inconclusive", result.InconclusiveCount.ToString());
xmlWriter.WriteAttributeString("skipped", result.SkipCount.ToString());
xmlWriter.WriteAttributeString("asserts", result.AssertCount.ToString());
xmlWriter.WriteAttributeString("random-seed", Randomizer.InitialSeed.ToString());
WriteEnvironmentElement();
}
/// <summary>
/// Write the result of a test run according to a spec.
/// </summary>
/// <param name="result">The test result</param>
/// <param name="spec">An output specification</param>
public void WriteResultFile(ITestResult result, OutputSpecification spec, IDictionary runSettings, TestFilter filter)
{
string outputPath = Path.Combine(_workDirectory, spec.OutputPath);
OutputWriter outputWriter = null;
switch (spec.Format)
{
case "nunit3":
outputWriter = new NUnit3XmlOutputWriter();
break;
case "nunit2":
outputWriter = new NUnit2XmlOutputWriter();
break;
//case "user":
// Uri uri = new Uri(Assembly.GetExecutingAssembly().CodeBase);
// string dir = Path.GetDirectoryName(uri.LocalPath);
// outputWriter = new XmlTransformOutputWriter(Path.Combine(dir, spec.Transform));
// break;
default:
throw new ArgumentException(
string.Format("Invalid XML output format '{0}'", spec.Format),
"spec");
}
outputWriter.WriteResultFile(result, outputPath, runSettings, filter);
Console.WriteLine("Results ({0}) saved as {1}", spec.Format, outputPath);
}
/// <summary>
/// Writes a test result to a file
/// </summary>
/// <param name="result">The result to be written</param>
/// <param name="outputPath">Path to the file to which the result is written</param>
public void WriteResultFile(ITestResult result, string outputPath)
{
using (StreamWriter writer = new StreamWriter(outputPath, false, Encoding.UTF8))
{
WriteResultFile(result, writer);
}
}
public override void OnTestFinished(ITestResult result)
{
var testName = result.Identity.Absolute;
switch (result.State)
{
case State.Passed:
_writer.WriteTestFinished(testName, TimeSpan.Zero);
break;
case State.Failed:
var operations = result.OperationResults.ToList();
var exceptions = GetExceptions(operations).ToList();
var message = GetGeneralMessage(exceptions, operations);
var details = GetDetails(operations, result.OutputEntries, exceptions);
_writer.WriteTestFailed(testName, message, details);
break;
case State.NotImplemented:
_writer.WriteTestIgnored(testName, string.Empty);
break;
case State.Ignored:
_writer.WriteTestIgnored(testName, string.Empty);
break;
}
}
public ResultViewModel(ITestResult result)
{
TestResult = result;
Result = result.ResultState.Status.ToString().ToUpperInvariant();
Name = result.Name;
Parent = result.Test.Parent.FullName;
Message = result.Message;
}
public TextValidationSerializer(
IValidationRuleset ruleset,
ITestResult[] testResults) :
this()
{
Ruleset = ruleset;
TestResults = testResults;
}
public void AddResults(string sceneName, ITestResult result)
{
if (!m_TestCollection.ContainsKey(sceneName))
m_TestCollection.Add(sceneName, new List<ITestResult>());
m_TestCollection[sceneName].Add(result);
if (result.Executed && !result.IsSuccess)
m_FailureCount++;
}
public void Update(ITestResult source, bool outdated) {
this.ResultState = source.ResultState;
this.Duration = source.Duration;
this.Message = source.Message;
this.StackTrace = source.StackTrace;
this.Executed = source.Executed;
this.Outdated = outdated;
}
void After(ITestResult result)
{
try {
EachInterceptor(x => x.After(result));
} catch(Exception ex) {
result.AfterFailure(ex);
}
}
private static IOperationResult GetFailingOperation(ITestResult testResult, string failingOperation)
{
var operationResults = testResult.OperationResults.Where(x => x.State == State.Failed && x.Text == failingOperation).ToList();
if (operationResults.Count != 1)
Assert.Fail("There is no single failing operation with text '{0}'.", failingOperation);
return operationResults.Single();
}
void ITestListener.TestFinished(ITestResult result)
{
if (result.FailCount > 0)
{
Environment.ExitCode = 1;
_writer.WriteLine(" FAIL");
}
}
/// <summary>
/// Writes a test result to a file
/// </summary>
/// <param name="result">The result to be written</param>
/// <param name="outputPath">Path to the file to which the result is written</param>
/// <param name="runSettings">A dictionary of settings used for this test run</param>
public void WriteResultFile(ITestResult result, string outputPath, IDictionary<string, object> runSettings, TestFilter filter)
{
using (var stream = new FileStream(outputPath, FileMode.Create))
using (var writer = new StreamWriter(stream, Encoding.UTF8))
{
WriteResultFile(result, writer, runSettings, filter);
}
}
/// <summary>
/// Writes the result of a test run to a specified TextWriter.
/// </summary>
/// <param name="result">The test result for the run</param>
/// <param name="writer">The TextWriter to which the xml will be written</param>
/// <param name="runSettings"></param>
/// <param name="filter"></param>
public override void WriteResultFile(ITestResult result, TextWriter writer, IDictionary<string, object> runSettings, TestFilter filter)
{
var settings = new XmlWriterSettings { Indent = true };
using (var xmlWriter = XmlWriter.Create(writer, settings))
{
WriteXmlOutput(result, xmlWriter);
}
}
private void WriteXmlOutput(ITestResult result, XmlWriter xmlWriter)
{
this.xmlWriter = xmlWriter;
InitializeXmlFile(result);
WriteResultElement(result);
TerminateXmlFile();
}
public TextValidationSerializer(
IResourceManager mgr,
IValidationRuleset ruleset,
ITestResult[] testResults)
: this(mgr)
{
Ruleset = ruleset;
TestResults = testResults;
}
public XmlValidationSerializer(
IXmlDocumentProvider docProvider,
IValidationRuleset ruleset,
ITestResult[] testResults) :
this(docProvider)
{
Ruleset = ruleset;
TestResults = testResults;
}
public void SaveTestResult(string resultsName, ITestResult[] results)
{
InitializeXmlFile(resultsName, new ResultSummarizer(results));
foreach (var result in results)
{
WriteResultElement(result);
}
TerminateXmlFile();
}
public void Update(ITestResult source, bool outdated)
{
ResultState = source.ResultState;
Duration = source.Duration;
Message = source.Message;
StackTrace = source.StackTrace;
Executed = source.Executed;
Outdated = outdated;
}
/// <summary>
/// Add a child result
/// </summary>
/// <param name="result">The child result to be added</param>
public override void AddResult(ITestResult result)
{
base.AddResult(result);
this.assertCount += result.AssertCount;
this.passCount += result.PassCount;
this.failCount += result.FailCount;
this.skipCount += result.SkipCount;
this.inconclusiveCount += result.InconclusiveCount;
}
static public bool IsFailure(this ITestResult result)
{
return(result.ResultState.Status == TestStatus.Failed);
}
public void RunFinished(ITestResult testResults)
{
SyncTestRunEventsHandler.instance.OnRunFinished();
}
public void TestFinishedEvent(ITestResult testResult)
{
m_AlreadyStartedTests.Remove(testResult.FullName);
m_ExecutedTests.Add(TestResultSerializer.MakeFromTestResult(testResult));
}
/// <summary>
/// Creates test result
/// </summary>
/// <param name="executionId"></param>
/// <param name="parentExecutionId"></param>
/// <param name="testType"></param>
/// <param name="testElement"></param>
/// <param name="parentTestElement"></param>
/// <param name="parentTestResult"></param>
/// <param name="rocksteadyTestResult"></param>
/// <param name="ticketNumber"></param>
/// <returns>Trx test result</returns>
private ITestResult CreateTestResult(Guid executionId, Guid parentExecutionId, TestType testType,
ITestElement testElement, ITestElement parentTestElement, ITestResult parentTestResult, Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult rocksteadyTestResult, string ticketNumber)
{
// Create test result
TrxLoggerObjectModel.TestOutcome testOutcome = Converter.ToOutcome(rocksteadyTestResult.Outcome);
var testResult = Converter.ToTestResult(testElement.Id.Id, executionId, parentExecutionId, testElement.Name,
this.testResultsDirPath, ticketNumber, testType, testElement.CategoryId, testOutcome, this.testRun, rocksteadyTestResult);
// Normal result scenario
if (parentTestResult == null)
{
this.results.TryAdd(executionId, testResult);
return(testResult);
}
// Ordered test inner result scenario
if (parentTestElement != null && parentTestElement.TestType.Equals(TrxLoggerConstants.OrderedTestType))
{
(parentTestResult as TestResultAggregation).InnerResults.Add(testResult);
this.innerResults.TryAdd(executionId, testResult);
return(testResult);
}
// Data driven inner result scenario
if (parentTestElement != null && parentTestElement.TestType.Equals(TrxLoggerConstants.UnitTestType))
{
(parentTestResult as TestResultAggregation).InnerResults.Add(testResult);
testResult.DataRowInfo = (parentTestResult as TestResultAggregation).InnerResults.Count;
testResult.ResultType = TrxLoggerConstants.InnerDataDrivenResultType;
parentTestResult.ResultType = TrxLoggerConstants.ParentDataDrivenResultType;
return(testResult);
}
return(testResult);
}
private static void AssertIgnoreResult(ITestResult result)
{
AssertOutcome(result, TestStatus.Skipped, IgnoreMessage);
}
public override void OnTestSkipped(TestMethodInstance instance, ITestResult result)
{
}
static public bool IsInconclusive(this ITestResult result)
{
return(result.ResultState.Status == TestStatus.Inconclusive);
}
private static void AssertOutcome(ITestResult result, TestStatus status, string message)
{
Assert.That(result.ResultState.Status, Is.EqualTo(status));
Assert.That(result.ResultState.Site, Is.EqualTo(FailureSite.Test));
Assert.That(result.Message, Is.EqualTo(message));
}
public void RunFinished(ITestResult testResults)
{
saved = false;
}
public void TestFinished(ITestResult result)
{
}
public virtual void TestFinished(ITestResult r)
{
TestResult result = r as TestResult;
if (result.Test is TestSuite)
{
if (!result.IsFailure() && !result.IsSuccess() && !result.IsInconclusive() && !result.IsIgnored())
{
Writer.WriteLine("\t[INFO] {0}", result.Message);
}
string name = result.Test.Name;
if (!String.IsNullOrEmpty(name))
{
Writer.WriteLine("{0} : {1} ms", name, result.Duration.TotalMilliseconds);
}
}
else
{
if (result.IsSuccess())
{
Writer.Write("\t[PASS] ");
PassedCount++;
}
else if (result.IsIgnored())
{
Writer.Write("\t[IGNORED] ");
IgnoredCount++;
}
else if (result.IsFailure())
{
Writer.Write("\t[FAIL] ");
FailedCount++;
}
else if (result.IsInconclusive())
{
Writer.Write("\t[INCONCLUSIVE] ");
InconclusiveCount++;
}
else
{
Writer.Write("\t[INFO] ");
}
Writer.Write(result.Test.FixtureType.Name);
Writer.Write(".");
Writer.Write(result.Test.Name);
string message = result.Message;
if (!String.IsNullOrEmpty(message))
{
Writer.Write(" : {0}", message.Replace("\r\n", "\\r\\n"));
}
Writer.WriteLine();
string stacktrace = result.StackTrace;
if (!String.IsNullOrEmpty(result.StackTrace))
{
string[] lines = stacktrace.Split(new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string line in lines)
{
Writer.WriteLine("\t\t{0}", line);
}
}
}
}
/// <summary> /// Abstract method that writes a test result to a TextWriter /// </summary> /// <param name="result">The result to be written</param> /// <param name="writer">A TextWriter to which the result is written</param> public abstract void WriteResultFile(ITestResult result, TextWriter writer);
public bool TryGetTestResult(Guid testCaseId, out ITestResult testResult)
{
return(_testResultsByTestCaseId.TryGetValue(testCaseId, out testResult));
}
private static void AssertInconclusiveResult(ITestResult result)
{
AssertOutcome(result, TestStatus.Inconclusive, InconclusiveMessage);
}
static public bool IsSuccess(this ITestResult result)
{
return(result.ResultState.Status == TestStatus.Passed);
}
private static void AssertFailResult(ITestResult result)
{
AssertOutcome(result, TestStatus.Failed, FailureMessage);
}
static public bool IsIgnored(this ITestResult result)
{
return(result.ResultState.Status == TestStatus.Skipped);
}
/// <summary>
/// A test case finished
/// </summary>
/// <param name="result">Result of the test case</param>
public void TestFinished(ITestResult result)
{
events.Enqueue(new TestFinishedEvent(result));
}
public ResultSummary(ITestResult result)
{
Summarize(result);
}
private void WriteTestLabel(ITestResult result)
{
_outWriter.WriteLine(ColorStyle.SectionHeader, "=> " + result.Test.Name);
}
public void TestFinished(ITestResult result)
{
AndroidRunner.Results [result.Test.FullName ?? result.Test.Name] = result as TestResult;
if (result.Test is TestSuite)
{
//if (!result.IsError && !result.IsFailure && !result.IsSuccess && !result.Executed)
//Writer.WriteLine ("\t[INFO] {0}", result.Message);
if (result.ResultState.Status != TestStatus.Failed &&
result.ResultState.Status != TestStatus.Skipped &&
result.ResultState.Status != TestStatus.Passed &&
result.ResultState.Status != TestStatus.Inconclusive)
{
Writer.WriteLine("\t[INFO] {0}", result.Message);
}
var diff = DateTime.UtcNow - time.Pop();
Writer.WriteLine("{0} : {1} ms", result.Test.Name, diff.TotalMilliseconds);
if (GCAfterEachFixture)
{
GC.Collect();
}
}
else
{
if (result.ResultState.Status == TestStatus.Passed)
{
//Writer.Write ("\t{0} ", result.Executed ? "[PASS]" : "[IGNORED]");
Writer.Write("{0}", result.ResultState.ToString());
passed++;
}
else if (result.ResultState.Status == TestStatus.Failed)
{
Writer.Write("[FAIL]");
failed++;
}
else
{
Writer.Write("[INFO]");
if (result.ResultState.Status == TestStatus.Skipped)
{
skipped++;
}
else if (result.ResultState.Status == TestStatus.Inconclusive)
{
inconclusive++;
}
}
string message = result.Message;
if (!String.IsNullOrEmpty(message))
{
Writer.Write(" : {0}", message.Replace("\r\n", "\\r\\n"));
}
Writer.WriteLine();
string stacktrace = result.StackTrace;
if (!String.IsNullOrEmpty(result.StackTrace))
{
string[] lines = stacktrace.Split(new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string line in lines)
{
Writer.WriteLine("\t\t{0}", line);
}
}
}
}
private void PrintFullReport(ITestResult result)
{
writer.WriteLine();
writer.WriteLine("All Test Results:");
PrintAllResults(result, " ");
}
private static void AssertExceptionResult(ITestResult result)
{
Assert.That(result.ResultState.Status, Is.EqualTo(TestStatus.Failed));
Assert.That(result.ResultState.Site, Is.EqualTo(FailureSite.Test));
Assert.That(result.ResultState.Label, Is.EqualTo(ResultState.Error.Label));
}
public void AddResult(ITestResult result)
{
_results.Add(result);
}
private static void AssertPassedResult(ITestResult result)
{
Assert.That(result.ResultState, Is.EqualTo(ResultState.Success));
}
/// <summary>
/// Initializes a new instance of the <see cref="ResultSummary"/> class.
/// </summary>
/// <param name="result">The result.</param>
public ResultSummary(ITestResult result)
{
Visit(result);
}
/// <summary>
/// Initializes a new instance of the <see cref="TestFinishedEvent"/> class.
/// </summary>
/// <param name="result">The result.</param>
public TestFinishedEvent(ITestResult result)
{
_result = result;
}
