public void TestFinished(NUnitTestEventTestCase resultNode)
{
var testId = resultNode.Id;
if (_outputNodes.TryGetValue(testId, out var outputNodes))
{
_outputNodes.Remove(testId);
}
var result = _testConverter.GetVsTestResults(resultNode, outputNodes ?? EmptyNodes);
if (_settings.ConsoleOut == 1 && !string.IsNullOrEmpty(result.ConsoleOutput))
{
_recorder.SendMessage(TestMessageLevel.Informational, result.ConsoleOutput);
}
if (_settings.ConsoleOut == 1 && !string.IsNullOrEmpty(resultNode.ReasonMessage))
{
_recorder.SendMessage(TestMessageLevel.Informational, $"{resultNode.Name}: {resultNode.ReasonMessage}");
}
_recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
{
_recorder.RecordResult(vsResult);
}
if (result.TestCaseResult.Outcome == TestOutcome.Failed && _settings.StopOnError)
{
executor.StopRun();
}
}
public void Attachments_CorrectAmountOfConvertedAttachments()
{
var cachedTestCase = testConverter.ConvertTestCase(fakeTestNode);
var fakeResultNode = new NUnitTestEventTestCase(FakeTestData.GetResultNode());
var testResults = testConverter.GetVsTestResults(fakeResultNode, Enumerable.Empty <INUnitTestEventTestOutput>().ToList());
var fakeAttachments = fakeResultNode.NUnitAttachments
.Where(n => !string.IsNullOrEmpty(n.FilePath))
.ToArray();
TestContext.Out.WriteLine("Incoming attachments");
foreach (var attachment in fakeAttachments)
{
TestContext.Out.WriteLine($"{attachment.FilePath}");
}
var convertedAttachments = testResults.TestResults
.SelectMany(tr => tr.Attachments.SelectMany(ats => ats.Attachments))
.ToArray();
TestContext.Out.WriteLine("\nConverted attachments (Uri, path)");
foreach (var attachment in convertedAttachments)
{
TestContext.Out.WriteLine($"{attachment.Uri.AbsoluteUri} : {attachment.Uri.LocalPath}");
}
Assert.Multiple(() =>
{
Assert.That(convertedAttachments.Length, Is.GreaterThan(0), "Some converted attachments were expected");
Assert.That(convertedAttachments.Length, Is.EqualTo(fakeAttachments.Length), "Attachments are not converted");
});
}
public void CannotMakeTestResultWhenTestCaseIsNotInCache()
{
var fakeResultNode = new NUnitTestEventTestCase(FakeTestData.GetResultNode());
var results = testConverter.GetVsTestResults(fakeResultNode, Enumerable.Empty <XmlNode>().ToList());
Assert.That(results.TestResults.Count, Is.EqualTo(0));
}
private VSTestResult MakeTestResultFromLegacyXmlNode(NUnitTestEventTestCase resultNode, IEnumerable <XmlNode> outputNodes)
{
var ourResult = GetBasicResult(resultNode, outputNodes);
if (ourResult == null)
{
return(null);
}
string message = resultNode.HasFailure
? resultNode.Failure.Message
: resultNode.HasReason
? resultNode.ReasonMessage
: null;
// If we're running in the IDE, remove any caret line from the message
// since it will be displayed using a variable font and won't make sense.
if (!string.IsNullOrEmpty(message) && NUnitTestAdapter.IsRunningUnderIde)
{
string pattern = NL + " -*\\^" + NL;
message = Regex.Replace(message, pattern, NL, RegexOptions.Multiline);
}
ourResult.ErrorMessage = message;
ourResult.ErrorStackTrace = resultNode.Failure?.Stacktrace;
return(ourResult);
}
public void ThatTestEventIsParsedForTestCaseWithReason()
{
var sut = new NUnitTestEventTestCase(testCaseSucceedsWithOutputAndReason);
Assert.That(sut.HasReason);
Assert.That(sut.ReasonMessage, Is.EqualTo("One or more child tests had warnings"));
}
public void CanMakeTestResultFromNUnitTestResult2(string output, string expectedMessages)
{
var cachedTestCase = testConverter.ConvertTestCase(fakeTestNode);
var fakeResultNode = new NUnitTestEventTestCase(FakeTestData.GetResultNode());
var outputNodes = output.Split(';').Select(i => XmlHelper.CreateXmlNode(i.Trim())).ToList();
var testResults = testConverter.GetVsTestResults(fakeResultNode, outputNodes);
var testResult = testResults.TestResults[0];
var actualMessages = string.Join(";", testResult.Messages.Select(i => i.Category + ":" + i.Text));
Assert.That(actualMessages, Is.EqualTo(expectedMessages));
}
public void ThatTestCaseFailsCanBeParsed()
{
var sut = new NUnitTestEventTestCase(testCaseFails);
Assert.That(sut.Properties.Count, Is.EqualTo(1));
Assert.That(sut.HasFailure);
Assert.Multiple(() =>
{
Assert.That(sut.Failure.Message.Trim(), Does.StartWith("Expected: 5"), $"Failure.Message is: {sut.Failure.Message}");
Assert.That(sut.Failure.Stacktrace, Does.StartWith(" at NUnitTestDemo.SimpleTests.TestFails()"));
});
}
public void ThatTestCaseFailureIsParsedWhenFailing()
{
var sut = new NUnitTestEventTestCase(testCaseFailing);
var failure = sut.Failure;
Assert.That(failure, Is.Not.Null);
Assert.Multiple(() =>
{
Assert.That(failure.Message.Trim(), Does.StartWith("Expected: 5"), $"Failure.Message is: {failure.Message}");
Assert.That(failure.Stacktrace, Does.StartWith(" at NUnitTestDemo.SimpleTests.TestFails()"), $"Stacktrace:{failure.Stacktrace}");
});
}
public void ThatTestCasePropertiesAreParsedWhenFailing()
{
var sut = new NUnitTestEventTestCase(testCaseFailing);
Assert.That(sut.Properties, Is.Not.Null);
Assert.That(sut.Properties.Count, Is.EqualTo(1));
var property = sut.Properties.FirstOrDefault();
Assert.That(property, Is.Not.Null);
Assert.That(property.Name, Is.EqualTo("Expect"));
Assert.That(property.Value, Is.EqualTo("Failure"));
}
public void ThatTestEventIsParsedForSuccessTestCase()
{
var sut = new NUnitTestEventTestCase(testCaseSucceeds);
Assert.Multiple(() =>
{
Assert.That(sut.FullName, Is.EqualTo("NUnitTestDemo.AsyncTests.AsyncTaskTestSucceeds"));
Assert.That(sut.TestType(), Is.EqualTo(NUnitTestEvent.TestTypes.NoIdea));
Assert.That(sut.Name, Is.EqualTo("AsyncTaskTestSucceeds"));
Assert.That(sut.Id, Is.EqualTo("0-1006"));
Assert.That(sut.Result, Is.EqualTo(NUnitTestEvent.ResultType.Success));
});
}
public void ResultStateToTestOutcome(string result, TestOutcome expected)
{
var resultNode = new NUnitTestEventTestCase(XmlHelper.CreateXmlNode(result));
var logger = Substitute.For <ITestLogger>();
var settings = Substitute.For <IAdapterSettings>();
settings.MapWarningTo.Returns(TestOutcome.Skipped);
var converter = new TestConverterForXml(logger, "whatever", settings);
var res = converter.GetTestOutcome(resultNode);
Assert.That(res, Is.EqualTo(expected), $"In: {result}, out: {res.ToString()} expected: {expected.ToString()} ");
}
public TestResultSet GetVsTestResults(NUnitTestEventTestCase resultNode, ICollection <XmlNode> outputNodes)
{
var results = new List <VSTestResult>();
var testCaseResult = GetBasicResult(resultNode, outputNodes);
if (testCaseResult != null)
{
switch (testCaseResult.Outcome)
{
case TestOutcome.Failed:
case TestOutcome.NotFound:
{
testCaseResult.ErrorMessage = resultNode.Failure?.Message;
testCaseResult.ErrorStackTrace = resultNode.Failure?.Stacktrace ?? resultNode.StackTrace;
break;
}
case TestOutcome.Skipped:
case TestOutcome.None:
testCaseResult.ErrorMessage = resultNode.ReasonMessage;
testCaseResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, resultNode.ReasonMessage));
break;
default:
{
if (adapterSettings.ConsoleOut > 0 && !string.IsNullOrEmpty(resultNode.ReasonMessage))
{
testCaseResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, resultNode.ReasonMessage));
}
break;
}
}
results.Add(testCaseResult);
}
if (results.Count == 0)
{
var result = MakeTestResultFromLegacyXmlNode(resultNode, outputNodes);
if (result != null)
{
results.Add(result);
}
}
return(new TestResultSet {
TestCaseResult = testCaseResult, TestResults = results, ConsoleOutput = resultNode.Output
});
}
public void ThatTestEventIsParsedForSuccessTestCaseWithOutput()
{
var sut = new NUnitTestEventTestCase(testCaseSucceedsWithOutput);
Assert.Multiple(() =>
{
Assert.That(sut.FullName, Is.EqualTo("NUnitTestDemo.SimpleTests.TestSucceeds"));
Assert.That(sut.TestType(), Is.EqualTo(NUnitTestEvent.TestTypes.NoIdea));
Assert.That(sut.Name, Is.EqualTo("TestSucceeds"));
Assert.That(sut.Id, Is.EqualTo("0-1074"));
Assert.That(sut.Result, Is.EqualTo(NUnitTestEvent.ResultType.Success));
Assert.That(sut.MethodName, Is.EqualTo("TestSucceeds"));
Assert.That(sut.ClassName, Is.EqualTo("NUnitTestDemo.SimpleTests"));
});
}
public void ThatTestEventIsParsedForFailingTestCase()
{
var sut = new NUnitTestEventTestCase(testCaseFailing);
Assert.Multiple(() =>
{
Assert.That(sut.FullName, Is.EqualTo("NUnitTestDemo.SimpleTests.TestFails"));
Assert.That(sut.TestType(), Is.EqualTo(NUnitTestEvent.TestTypes.NoIdea));
Assert.That(sut.Name, Is.EqualTo("TestFails"));
Assert.That(sut.Id, Is.EqualTo("0-1076"));
Assert.That(sut.Result, Is.EqualTo(NUnitTestEvent.ResultType.Failed));
Assert.That(sut.Duration.TotalMilliseconds, Is.GreaterThanOrEqualTo(1.0));
Assert.That(sut.StartTime().Ok);
Assert.That(sut.EndTime().Ok);
});
}
public void CanMakeTestResultFromNUnitTestResult()
{
// This should put the TestCase in the cache
var cachedTestCase = testConverter.ConvertTestCase(fakeTestNode);
var fakeResultNode = new NUnitTestEventTestCase(FakeTestData.GetResultNode());
var testResults = testConverter.GetVsTestResults(fakeResultNode, Enumerable.Empty <XmlNode>().ToList());
var testResult = testResults.TestResults[0];
var testCase = testResult.TestCase;
Assert.That(testCase, Is.SameAs(cachedTestCase));
CheckTestCase(testCase);
Assert.That(testResult.Outcome, Is.EqualTo(TestOutcome.Passed));
Assert.That(testResult.ErrorMessage, Is.EqualTo(null));
Assert.That(testResult.Duration, Is.EqualTo(TimeSpan.FromSeconds(1.234)));
}
public TestResultSet GetVsTestResults(NUnitTestEventTestCase resultNode, ICollection <XmlNode> outputNodes)
{
var results = new List <VSTestResult>();
var testCaseResult = GetBasicResult(resultNode, outputNodes);
if (testCaseResult != null)
{
if (testCaseResult.Outcome == TestOutcome.Failed || testCaseResult.Outcome == TestOutcome.NotFound)
{
testCaseResult.ErrorMessage = resultNode.Failure?.Message;
testCaseResult.ErrorStackTrace = resultNode.Failure?.Stacktrace;
// find stacktrace in assertion nodes if not defined (seems .netcore2.0 doesn't provide stack-trace for Assert.Fail("abc"))
if (testCaseResult.ErrorStackTrace == null)
{
string stackTrace = string.Empty;
foreach (XmlNode assertionStacktraceNode in resultNode.Node.SelectNodes("assertions/assertion/stack-trace"))
{
stackTrace += assertionStacktraceNode.InnerText;
}
testCaseResult.ErrorStackTrace = stackTrace;
}
}
else if (testCaseResult.Outcome == TestOutcome.Skipped || testCaseResult.Outcome == TestOutcome.None)
{
testCaseResult.ErrorMessage = resultNode.ReasonMessage;
}
results.Add(testCaseResult);
}
if (results.Count == 0)
{
var result = MakeTestResultFromLegacyXmlNode(resultNode, outputNodes);
if (result != null)
{
results.Add(result);
}
}
return(new TestResultSet {
TestCaseResult = testCaseResult, TestResults = results, ConsoleOutput = resultNode.Output
});
}
public void Attachments_AllFilePathesStartWithFileScheme()
{
const string fileUriScheme = "file://";
const string errorMessage = "Path must start with file:// uri scheme";
var cachedTestCase = testConverter.ConvertTestCase(fakeTestNode);
var fakeResultNode = new NUnitTestEventTestCase(FakeTestData.GetResultNode().AsString());
var testResults = testConverter.GetVsTestResults(fakeResultNode, Enumerable.Empty <XmlNode>().ToList());
var convertedAttachments = testResults.TestResults
.SelectMany(tr => tr.Attachments.SelectMany(ats => ats.Attachments))
.ToArray();
foreach (var attachment in convertedAttachments)
{
var originalPath = attachment.Uri.OriginalString;
Assert.That(originalPath.LastIndexOf(fileUriScheme), Is.EqualTo(0), errorMessage);
}
}
public void Attachments_CorrectAmountOfConvertedAttachments()
{
var cachedTestCase = testConverterForXml.ConvertTestCase(fakeTestNode);
var fakeResultNode = new NUnitTestEventTestCase(FakeTestData.GetResultNode());
var testResults = testConverterForXml.GetVsTestResults(fakeResultNode, Enumerable.Empty <INUnitTestEventTestOutput>().ToList());
var fakeAttachments = fakeResultNode.NUnitAttachments
.Where(n => !string.IsNullOrEmpty(n.FilePath))
.ToArray();
var convertedAttachments = testResults.TestResults
.SelectMany(tr => tr.Attachments.SelectMany(ats => ats.Attachments))
.ToArray();
Assert.Multiple(() =>
{
Assert.That(convertedAttachments.Length, Is.GreaterThan(0), "Some converted attachments were expected");
Assert.That(convertedAttachments.Length, Is.EqualTo(fakeAttachments.Length), "Attachments are not converted");
});
}
public void OnTestEvent(string report)
{
var node = new NUnitTestEventHeader(report);
dumpXml?.AddTestEvent(node.AsString());
try
{
switch (node.Type)
{
case NUnitTestEventHeader.EventType.StartTest:
var startNode = new NUnitTestEventStartTest(node);
TestStarted(startNode);
break;
case NUnitTestEventHeader.EventType.TestCase:
var testFinishedNode = new NUnitTestEventTestCase(node);
TestFinished(testFinishedNode);
break;
case NUnitTestEventHeader.EventType.TestSuite:
var suiteFinishedNode = new NUnitTestEventSuiteFinished(node);
SuiteFinished(suiteFinishedNode);
break;
case NUnitTestEventHeader.EventType.TestOutput:
var outputNode = new NUnitTestEventTestOutput(node);
TestOutput(outputNode);
break;
}
}
catch (Exception ex)
{
_recorder.SendMessage(TestMessageLevel.Warning, $"Error processing {node.Name} event for {node.FullName}");
_recorder.SendMessage(TestMessageLevel.Warning, ex.ToString());
}
}
public void ThatTestCaseFailsCanBeParsedWithReason()
{
var sut = new NUnitTestEventTestCase(testCaseFailsWithReason);
Assert.That(sut.ReasonMessage, Is.EqualTo("Ignoring this test deliberately"));
}
public TestOutcome ResultStateToTestOutcome(string result)
{
var resultNode = new NUnitTestEventTestCase(XmlHelper.CreateXmlNode(result));
return(TestConverter.GetTestOutcome(resultNode));
}
public void ThatExplicitTestFixtureWorksWithZeroStartTime()
{
var sut = new NUnitTestEventTestCase(testEvent: testCaseExplicitFixtureTime);
Assert.DoesNotThrow(code: () => sut.StartTime());
}
