public void ConvertExceptionToVSTestResult()
{
LogEntryException exception = new LogEntryException("C string: some error", new SourceFileInfo("unknown location", 0));
exception.LastCheckpoint = new SourceFileInfo("boostunittestsample.cpp", 13);
exception.CheckpointDetail = "Going to throw an exception";
BoostTestResult testCaseResult = new BoostTestResultBuilder().
For(this.TestCase).
Aborted().
Duration(0).
Log(exception).
Build();
VSTestResult result = testCaseResult.AsVSTestResult(this.TestCase);
AssertVSTestModelProperties(result);
Assert.That(result.Outcome, Is.EqualTo(TestOutcome.Failed));
// A 0 duration should list as 1 tick to avoid UI issues in the test adapter
Assert.That(result.Duration, Is.EqualTo(TimeSpan.FromTicks(1)));
AssertVsTestModelError(result, exception);
Assert.That(result.Messages.Count, Is.EqualTo(1));
TestResultMessage message = result.Messages.First();
Assert.That(message.Category, Is.EqualTo(TestResultMessage.StandardErrorCategory));
}
public void OutcomeOfRunWillBeCompletedIfNoTestsFails()
{
ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1");
ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2");
ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1");
ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1);
passResult1.Outcome = ObjectModel.TestOutcome.Passed;
ObjectModel.TestResult passResult2 = new ObjectModel.TestResult(passTestCase2);
passResult2.Outcome = ObjectModel.TestOutcome.Passed;
ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1);
skipResult1.Outcome = ObjectModel.TestOutcome.Skipped;
Mock <TestResultEventArgs> pass1 = new Mock <TestResultEventArgs>(passResult1);
Mock <TestResultEventArgs> pass2 = new Mock <TestResultEventArgs>(passResult2);
Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1);
this.testableTrxLogger.TestResultHandler(new object(), pass1.Object);
this.testableTrxLogger.TestResultHandler(new object(), pass2.Object);
this.testableTrxLogger.TestResultHandler(new object(), skip1.Object);
var testRunCompleteEventArgs = CreateTestRunCompleteEventArgs();
this.testableTrxLogger.TestRunCompleteHandler(new object(), testRunCompleteEventArgs);
Assert.AreEqual(TrxLoggerObjectModel.TestOutcome.Completed, this.testableTrxLogger.TestResultOutcome);
}
public void ConvertFailToVSTestResult()
{
LogEntryError error = new LogEntryError("Error: 1 != 2", new SourceFileInfo("file.cpp", 10));
BoostTestResult testCaseResult = new BoostTestResultBuilder().
For(this.TestCase).
Failed().
Duration(2500).
Log(error).
Build();
VSTestResult result = testCaseResult.AsVSTestResult(this.TestCase);
AssertVSTestModelProperties(result);
Assert.That(result.Outcome, Is.EqualTo(TestOutcome.Failed));
Assert.That(result.Duration, Is.EqualTo(Microseconds(2500)));
AssertVsTestModelError(result, error);
Assert.That(result.Messages.Count, Is.EqualTo(1));
TestResultMessage message = result.Messages.First();
Assert.That(message.Category, Is.EqualTo(TestResultMessage.StandardErrorCategory));
}
public void TestPassed(string name, string type, string method, double duration, string output)
{
VsTestResult result = MakeVsTestResult(type, method, duration, output, TestOutcome.Passed);
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
}
public void TestSkipped(string name, string type, string method, string reason)
{
VsTestResult result = MakeVsTestResult(type, method, 0.0, null, TestOutcome.Skipped);
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
}
/// <summary>
/// Maps a <see cref="Framework.Results.TestResult"/> to a <see cref="Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult"/>.
/// </summary>
/// <param name="testCase">An existing <see cref="TestCase"/></param>
/// <param name="result">The result to map</param>
/// <returns>A Visual Studio <see cref="Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult"/></returns>
public VSTestResult Map(TestCase testCase, PSTestResult result)
{
var vsResult = new VSTestResult(testCase)
{
Outcome = Map(result.Status),
Duration = result.Duration.HasValue ? result.Duration.Value : TimeSpan.Zero,
};
foreach (var artifact in result.Artifacts)
{
var attachmentSet = new AttachmentSet(artifact, "Attachment");
attachmentSet.Attachments.Add(new UriDataAttachment(artifact, string.Empty));
vsResult.Attachments.Add(attachmentSet);
}
var failedResult = result as FailedResult;
if (failedResult != null)
{
vsResult.ErrorMessage = failedResult.Reason.Message;
vsResult.ErrorStackTrace = failedResult.Reason.StackTrace;
}
return(vsResult);
}
public VSTestResult ConvertTestResult(NUnitTestResult result)
{
TestCase ourCase = GetCachedTestCase(result.Test.TestName.UniqueName);
if (ourCase == null) return null;
VSTestResult ourResult = new VSTestResult(ourCase)
{
DisplayName = ourCase.DisplayName,
Outcome = ResultStateToTestOutcome(result.ResultState),
Duration = TimeSpan.FromSeconds(result.Time)
};
// TODO: Remove this when NUnit provides a better duration
if (ourResult.Duration == TimeSpan.Zero && (ourResult.Outcome == TestOutcome.Passed || ourResult.Outcome == TestOutcome.Failed))
ourResult.Duration = TimeSpan.FromTicks(1);
ourResult.ComputerName = Environment.MachineName;
// TODO: Stuff we don't yet set
// StartTime - not in NUnit result
// EndTime - not in NUnit result
// Messages - could we add messages other than the error message? Where would they appear?
// Attachments - don't exist in NUnit
if (result.Message != null)
ourResult.ErrorMessage = GetErrorMessage(result);
if (!string.IsNullOrEmpty(result.StackTrace))
{
string stackTrace = StackTraceFilter.Filter(result.StackTrace);
ourResult.ErrorStackTrace = stackTrace;
}
return ourResult;
}
/// <summary>
/// Copies the result messages to unitTestResult
/// </summary>
/// <param name="unitTestResult">TRX TestResult</param>
/// <param name="testResult"> rock steady test result</param>
private void UpdateResultMessages(TrxObjectModel.TestResult unitTestResult, ObjectModel.TestResult testResult)
{
StringBuilder debugTrace = new StringBuilder();
StringBuilder stdErr = new StringBuilder();
StringBuilder stdOut = new StringBuilder();
foreach (TestResultMessage message in testResult.Messages)
{
if (TestResultMessage.AdditionalInfoCategory.Equals(message.Category, StringComparison.OrdinalIgnoreCase))
{
unitTestResult.AddTextMessage(message.Text);
}
else if (TestResultMessage.DebugTraceCategory.Equals(message.Category, StringComparison.OrdinalIgnoreCase))
{
debugTrace.AppendLine(message.Text);
}
else if (TestResultMessage.StandardErrorCategory.Equals(message.Category, StringComparison.OrdinalIgnoreCase))
{
stdErr.AppendLine(message.Text);
}
else if (TestResultMessage.StandardOutCategory.Equals(message.Category, StringComparison.OrdinalIgnoreCase))
{
stdOut.AppendLine(message.Text);
}
else
{
EqtTrace.Warning("The message category " + message.Category + " does not match any predefined category.");
}
}
unitTestResult.DebugTrace = debugTrace.ToString();
unitTestResult.StdErr = stdErr.ToString();
unitTestResult.StdOut = stdOut.ToString();
}
public void ConvertMemoryLeakToVSTestResult()
{
LogEntryMemoryLeak leak = new LogEntryMemoryLeak();
leak.LeakLineNumber = 32;
leak.LeakMemoryAllocationNumber = 836;
leak.LeakLeakedDataContents = " Data: <`- Leak... > 60 2D BD 00 4C 65 61 6B 2E 2E 2E 00 CD CD CD CD ";
leak.LeakSourceFilePath = @"C:\boostunittestsample.cpp";
leak.LeakSourceFileName = "boostunittestsample.cpp";
BoostTestResult testCaseResult = new BoostTestResultBuilder().
For(this.TestCase).
Passed().
Duration(1000).
Log(leak).
Build();
VSTestResult result = testCaseResult.AsVSTestResult(this.TestCase);
AssertVSTestModelProperties(result);
Assert.That(result.Outcome, Is.EqualTo(TestOutcome.Passed));
Assert.That(result.Duration, Is.EqualTo(Microseconds(1000)));
Assert.That(result.Messages.Count, Is.EqualTo(1));
TestResultMessage message = result.Messages.First();
Assert.That(message.Category, Is.EqualTo(TestResultMessage.StandardErrorCategory));
}
public void TestFinished(XmlNode resultNode)
{
TestResult ourResult = _testConverter.ConvertTestResult(resultNode);
_recorder.RecordEnd(ourResult.TestCase, ourResult.Outcome);
_recorder.RecordResult(ourResult);
}
private VSTestResult GetBasicResult(XmlNode resultNode, IEnumerable <XmlNode> outputNodes)
{
var vsTest = GetCachedTestCase(resultNode.GetAttribute("id"));
if (vsTest == null)
{
return(null);
}
string displayName = vsTest.FullyQualifiedName;
vsTest.FullyQualifiedName = System.Text.RegularExpressions.Regex.Replace(vsTest.FullyQualifiedName, @" ?\(.*?\)", string.Empty);
var vsResult = new VSTestResult(vsTest)
{
DisplayName = displayName,
Outcome = GetTestOutcome(resultNode),
Duration = TimeSpan.FromSeconds(resultNode.GetAttribute("duration", 0.0))
};
var startTime = resultNode.GetAttribute("start-time");
if (startTime != null)
{
vsResult.StartTime = DateTimeOffset.Parse(startTime, CultureInfo.InvariantCulture);
}
var endTime = resultNode.GetAttribute("end-time");
if (endTime != null)
{
vsResult.EndTime = DateTimeOffset.Parse(endTime, CultureInfo.InvariantCulture);
}
// TODO: Remove this when NUnit provides a better duration
if (vsResult.Duration == TimeSpan.Zero && (vsResult.Outcome == TestOutcome.Passed || vsResult.Outcome == TestOutcome.Failed))
{
vsResult.Duration = TimeSpan.FromTicks(1);
}
vsResult.ComputerName = Environment.MachineName;
FillResultFromOutputNodes(outputNodes, vsResult);
// Add stdOut messages from TestFinished element to vstest result
var outputNode = resultNode.SelectSingleNode("output");
if (outputNode != null)
{
vsResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, outputNode.InnerText));
}
var attachmentSet = ParseAttachments(resultNode);
if (attachmentSet.Attachments.Count > 0)
{
vsResult.Attachments.Add(attachmentSet);
}
return(vsResult);
}
public void RaiseTestResultShouldInvokeRegisteredEventHandlerIfTestResultEventArgsIsPassed()
{
EventWaitHandle waitHandle = new AutoResetEvent(false);
bool testResultReceived = false;
TestResultEventArgs eventArgs = null;
var result = new TestResult(new TestCase("This is a string.", new Uri("some://uri"), "DummySourceFileName"));
// Register for the test result event.
loggerEvents.TestResult += (sender, e) =>
{
testResultReceived = true;
eventArgs = e;
waitHandle.Set();
};
loggerEvents.EnableEvents();
// Send the test result event.
loggerEvents.RaiseTestResult(new TestResultEventArgs(result));
var waitSuccess = waitHandle.WaitOne(500);
Assert.IsTrue(waitSuccess, "Event must be raised within timeout.");
Assert.IsTrue(testResultReceived);
Assert.IsNotNull(eventArgs);
Assert.AreEqual(result, eventArgs.Result);
}
private static Mock <TestResultEventArgs> CreatePassTestResultEventArgsMock()
{
TestCase passTestCase = CreateTestCase("Pass1");
var passResult = new ObjectModel.TestResult(passTestCase);
return(new Mock <TestResultEventArgs>(passResult));
}
public void TestResultHandlerKeepingTheTrackOfTotalTests()
{
ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1");
ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2");
ObjectModel.TestCase failTestCase1 = CreateTestCase("Fail1");
ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1");
ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1);
passResult1.Outcome = ObjectModel.TestOutcome.Passed;
ObjectModel.TestResult passResult2 = new ObjectModel.TestResult(passTestCase2);
passResult2.Outcome = ObjectModel.TestOutcome.Passed;
ObjectModel.TestResult failResult1 = new ObjectModel.TestResult(failTestCase1);
failResult1.Outcome = ObjectModel.TestOutcome.Failed;
ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1);
skipResult1.Outcome = ObjectModel.TestOutcome.Skipped;
Mock <TestResultEventArgs> pass1 = new Mock <TestResultEventArgs>(passResult1);
Mock <TestResultEventArgs> pass2 = new Mock <TestResultEventArgs>(passResult2);
Mock <TestResultEventArgs> fail1 = new Mock <TestResultEventArgs>(failResult1);
Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1);
this.testableTrxLogger.TestResultHandler(new object(), pass1.Object);
this.testableTrxLogger.TestResultHandler(new object(), pass2.Object);
this.testableTrxLogger.TestResultHandler(new object(), fail1.Object);
this.testableTrxLogger.TestResultHandler(new object(), skip1.Object);
Assert.AreEqual(this.testableTrxLogger.TotalTestCount, 4, "Passed Tests");
}
private void ExecuteTestRun(TestCollection testCollection, IFrameworkHandle frameworkHandle, Dictionary <Test, TestCase> testMapping)
{
try
{
_testRun = testCollection.CreateTestRun();
_testRun.TestCompleted += result =>
{
var vsResult = new VisualStudioTestResult(testMapping[result.Test])
{
Outcome = OutcomeFromResult(result.Result),
Duration = result.Duration,
ErrorMessage = result.Output,
ErrorStackTrace = GetCombinedStackTrace(result.Exception)
};
vsResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, result.Output));
vsResult.ErrorStackTrace = GetCombinedStackTrace(result.Exception);
frameworkHandle.RecordResult(vsResult);
};
_testRun.ExecuteAsync().GetAwaiter().GetResult();
}
catch (Exception ex)
{
File.WriteAllText("C:/Users/seamillo/Desktop/jazsharp.testadapter.log", ex.ToString());
throw;
}
}
private VsTestResult MakeVsTestResult(TestOutcome outcome, ITestCase testCase, string displayName, double executionTime = 0.0, string output = null)
{
var vsTestCase = testCases[testCase];
var fqTestMethodName = String.Format("{0}.{1}", testCase.TestMethod.TestClass.Class.Name, testCase.TestMethod.Method.Name);
var vsDisplayName = settings.GetDisplayName(displayName, testCase.TestMethod.Method.Name, fqTestMethodName);
var result = new VsTestResult(vsTestCase)
{
#if !WINDOWS_PHONE_APP
ComputerName = Environment.MachineName,
#endif
DisplayName = vsDisplayName,
Duration = TimeSpan.FromSeconds(executionTime),
Outcome = outcome,
};
// Work around VS considering a test "not run" when the duration is 0
if (result.Duration.TotalMilliseconds == 0)
{
result.Duration = TimeSpan.FromMilliseconds(1);
}
if (!String.IsNullOrEmpty(output))
{
result.Messages.Add(new VsTestResultMessage(VsTestResultMessage.StandardOutCategory, output));
}
return(result);
}
public void TestResultHandlerShouldAddSingleTestEntryForOrderedTest()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
ObjectModel.TestCase testCase2 = CreateTestCase("TestCase2");
ObjectModel.TestCase testCase3 = CreateTestCase("TestCase3");
Guid parentExecutionId = Guid.NewGuid();
ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.TestTypeProperty, TrxLoggerConstants.OrderedTestTypeGuid);
ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase2);
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase3);
result3.Outcome = ObjectModel.TestOutcome.Failed;
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1);
Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2);
Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object);
Assert.AreEqual(this.testableTrxLogger.TestEntryCount, 1, "TestResultHandler is adding multiple test entries for ordered test.");
}
public void GetHashCodeShouldReturnAnUniqueHash()
{
var result = new TestResult(new TestCase());
var resultInfo = new TestResultInfo(result, string.Empty, string.Empty, string.Empty);
Assert.AreNotEqual(new TestResult(new TestCase()).GetHashCode(), resultInfo.GetHashCode());
}
public void EqualsShouldReturnFalseForNonTestResultInfoObject()
{
var result = new TestResult(new TestCase());
var resultInfo = new TestResultInfo(result, string.Empty, string.Empty, string.Empty);
Assert.IsFalse(resultInfo.Equals(new ()));
}
public void TestResultHandlerShouldAddSingleTestElementForDataDrivenTests()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
Guid parentExecutionId = Guid.NewGuid();
ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId);
ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase1);
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase1);
result3.Outcome = ObjectModel.TestOutcome.Failed;
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1);
Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2);
Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object);
Assert.AreEqual(this.testableTrxLogger.UnitTestElementCount, 1, "TestResultHandler is adding multiple test elements for data driven tests.");
}
public void TestResultHandlerShouldAddHierarchicalResultsIfParentTestResultIsPresent()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
Guid parentExecutionId = Guid.NewGuid();
ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId);
ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase1);
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase1);
result3.Outcome = ObjectModel.TestOutcome.Failed;
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1);
Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2);
Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object);
Assert.AreEqual(this.testableTrxLogger.TestResultCount, 1, "TestResultHandler is not creating hierarchical results when parent result is present.");
Assert.AreEqual(this.testableTrxLogger.TotalTestCount, 3, "TestResultHandler is not adding all inner results in parent test result.");
}
/// <summary>
/// Returns the QToolsCommon.TestResult object created from rockSteady TestResult.
/// </summary>
/// <param name="rockSteadyTestResult"> rock steady test result</param>
/// <param name="testElement"> testElement of that test</param>
/// <param name="testOutcome"> Test outcome </param>
/// <param name="testRun"> test run object </param>
/// <returns> TestResult object </returns>
private static TrxObjectModel.UnitTestResult GetQToolsTestResultFromTestResult(
ObjectModel.TestResult rockSteadyTestResult,
TrxObjectModel.UnitTestElement testElement,
TrxObjectModel.TestOutcome testOutcome,
TrxObjectModel.TestRun testRun)
{
UnitTestResult testResult = new UnitTestResult(Environment.MachineName, testRun.Id, testElement, testOutcome);
if (rockSteadyTestResult.ErrorMessage != null)
{
testResult.ErrorMessage = rockSteadyTestResult.ErrorMessage;
}
if (rockSteadyTestResult.ErrorStackTrace != null)
{
testResult.ErrorStackTrace = rockSteadyTestResult.ErrorStackTrace;
}
// set start and end times
if (rockSteadyTestResult.EndTime != null)
{
testResult.EndTime = rockSteadyTestResult.EndTime.UtcDateTime;
}
if (rockSteadyTestResult.StartTime != null)
{
testResult.StartTime = rockSteadyTestResult.StartTime.UtcDateTime;
}
if (rockSteadyTestResult.Duration != null)
{
testResult.Duration = rockSteadyTestResult.Duration;
}
return(testResult);
}
private static VSResult ConvertToVsResult(TestCase @case, UnitTest unitTest, SUResult sunitResult)
{
var result = new VSResult(@case)
{
DisplayName = unitTest.Name,
Outcome = sunitResult.Kind.ToTestOutcome()
};
switch (sunitResult)
{
case UnexpectedExceptionResult errorResult:
string heading = $"Unexpected {errorResult.Exception.GetType().Name}";
string message = IndentLines(errorResult.Exception.Message, " ");
result.ErrorMessage = $"{heading}\n{message}";
result.ErrorStackTrace = errorResult.Exception.StackTrace;
break;
case RanSuccessfullyResult ranResult when !ranResult.Result.Passed:
result.ErrorMessage = ranResult.Result.ToString();
break;
case RanSuccessfullyResult ranResult when ranResult.Result.Passed:
break;
default:
result.ErrorMessage = $"{sunitResult.Kind}: {sunitResult}";
break;
}
return(result);
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
var testCaseLookup = tests
.ToDictionary(test => test.ToUnitTest(), test => test);
cts = new CancellationTokenSource();
try
{
var resultObservable = TestRunner.RunTests(testCaseLookup.Select(kvp => kvp.Key));
resultObservable.Do(sunitResult =>
{
UnitTest unitTest = sunitResult.UnitTest;
TestCase testCase = testCaseLookup[unitTest];
VSResult vsResult = ConvertToVsResult(testCase, unitTest, sunitResult);
frameworkHandle.RecordResult(vsResult);
}).ToTask()
.Wait(cts.Token);
}
catch (Exception error)
{
frameworkHandle.SendMessage(TestMessageLevel.Error, $"Unexpected {error.GetType().FullName}");
frameworkHandle.SendMessage(TestMessageLevel.Error, error.Message);
frameworkHandle.SendMessage(TestMessageLevel.Error, error.StackTrace);
}
}
VsTestResult MakeVsTestResult(TestOutcome outcome, ITestCase testCase, string displayName, double executionTime = 0.0, string output = null)
{
var vsTestCase = FindTestCase(testCase);
if (vsTestCase == null)
{
return(null);
}
var result = new VsTestResult(vsTestCase)
{
#if !WINDOWS_UAP
ComputerName = Environment.MachineName,
#endif
DisplayName = displayName,
Duration = TimeSpan.FromSeconds(executionTime),
Outcome = outcome,
};
// Work around VS considering a test "not run" when the duration is 0
if (result.Duration.TotalMilliseconds == 0)
{
result.Duration = TimeSpan.FromMilliseconds(1);
}
if (!string.IsNullOrEmpty(output))
{
result.Messages.Add(new VsTestResultMessage(VsTestResultMessage.StandardOutCategory, output));
}
return(result);
}
private VsTestResult MakeVsTestResult(ITestResultMessage testResult, TestOutcome outcome)
{
var testCase = testCases[testResult.TestCase];
var result = new VsTestResult(testCase)
{
ComputerName = Environment.MachineName,
DisplayName = testResult.TestDisplayName,
Duration = TimeSpan.FromSeconds((double)testResult.ExecutionTime),
Outcome = outcome,
};
// Work around VS considering a test "not run" when the duration is 0
if (result.Duration.TotalMilliseconds == 0)
{
result.Duration = TimeSpan.FromMilliseconds(1);
}
if (!String.IsNullOrEmpty(testResult.Output))
{
result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testResult.Output));
}
return(result);
}
public void RunTestsShouldIterateThroughAllExecutors()
{
var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location;
var executorUriExtensionMap = new List <Tuple <Uri, string> >
{
new Tuple <Uri, string>(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation),
new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), assemblyLocation)
};
// Setup mocks.
this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); };
this.runTestsInstance.InvokeExecutorCallback =
(executor, executorUriExtensionTuple, runContext, frameworkHandle) =>
{
var testCase = new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll");
var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase);
this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult);
};
this.runTestsInstance.RunTests();
var expectedUris = new string[] { BadBaseRunTestsExecutorUri.ToLower(), BaseRunTestsExecutorUri.ToLower() };
CollectionAssert.AreEqual(expectedUris, this.runTestsInstance.GetExecutorUrisThatRanTests.ToArray());
}
public void TestResultHandlerShouldAddHierarchicalResultsForOrderedTest()
{
ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1");
ObjectModel.TestCase testCase2 = CreateTestCase("TestCase2");
ObjectModel.TestCase testCase3 = CreateTestCase("TestCase3");
Guid parentExecutionId = Guid.NewGuid();
ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId);
result1.SetPropertyValue <Guid>(TrxLoggerConstants.TestTypeProperty, TrxLoggerConstants.OrderedTestTypeGuid);
ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase2);
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase3);
result3.Outcome = ObjectModel.TestOutcome.Failed;
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid());
result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId);
Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1);
Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2);
Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object);
this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object);
Assert.AreEqual(1, this.testableTrxLogger.TestResultCount, "TestResultHandler is not creating hierarchical results for ordered test.");
Assert.AreEqual(3, this.testableTrxLogger.TotalTestCount, "TestResultHandler is not adding all inner results in ordered test.");
}
private Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult GetTestResult(int index)
{
var tc = new TestCase("Test" + index, new Uri("executor://dummy"), "DummySourceFileName");
var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(tc);
testResult.TestCase.Id = Guid.NewGuid();
return(testResult);
}
/// <summary>
/// Gets parent execution id of test result.
/// </summary>
/// <param name="testResult"></param>
/// <returns>Parent execution id.</returns>
public Guid GetParentExecutionId(ObjectModel.TestResult testResult)
{
TestProperty parentExecutionIdProperty = testResult.Properties.FirstOrDefault(
property => property.Id.Equals(Constants.ParentExecutionIdPropertyIdentifier));
return(parentExecutionIdProperty == null ?
Guid.Empty :
testResult.GetPropertyValue(parentExecutionIdProperty, Guid.Empty));
}
public void RecordResultShouldUpdateTestRunCache()
{
var testCase = new TestCase("A.C.M", new Uri("executor://dummy"), "A");
var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase);
var testRecorder = new TestExecutionRecorder(null, this.testableTestRunCache);
testRecorder.RecordResult(testResult);
Assert.IsTrue(this.testableTestRunCache.TestResultList.Contains(testResult));
}
/// <summary>
/// Converts a Boost.Test.Result.TestResult model into an equivalent
/// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model.
/// </summary>
/// <param name="result">The Boost.Test.Result.TestResult model to convert.</param>
/// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param>
/// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns>
public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test)
{
Utility.Code.Require(result, "result");
Utility.Code.Require(test, "test");
VSTestResult vsResult = new VSTestResult(test);
vsResult.ComputerName = Environment.MachineName;
vsResult.Outcome = GetTestOutcome(result.Result);
// Boost.Test.Result.TestResult.Duration is in microseconds
// 1 millisecond = 10,000 ticks
// => 1 microsecond = 10 ticks
// Reference: https://msdn.microsoft.com/en-us/library/zz841zbz(v=vs.110).aspx
long ticks = (long) Math.Min((result.Duration * 10), long.MaxValue);
// Clamp tick count to 1 in case Boost duration is listed as 0
vsResult.Duration = new TimeSpan(Math.Max(ticks, 1));
if (result.LogEntries.Count > 0)
{
foreach (TestResultMessage message in GetTestMessages(result))
{
vsResult.Messages.Add(message);
}
// Test using the TestOutcome type since elements from the
// Boost Result type may be collapsed into a particular value
if (vsResult.Outcome == VSTestOutcome.Failed)
{
LogEntry error = GetLastError(result);
if (error != null)
{
vsResult.ErrorMessage = GetErrorMessage(result);
vsResult.ErrorStackTrace = ((error.Source == null) ? null : error.Source.ToString());
}
}
}
return vsResult;
}
public TestResult BuildTestResult(TestData testData, TestStepRun testStepRun, TestCase testCase)
{
var testResult = new TestResult(testCase)
{
DisplayName = testData.Name,
StartTime = testStepRun.StartTime,
EndTime = testStepRun.EndTime,
Duration = testStepRun.Result.Duration,
Outcome = GetOutcome(testStepRun.Result.Outcome.Status),
};
var failuresStream = testStepRun.TestLog.GetStream(MarkupStreamNames.Failures);
if (failuresStream != null)
{
testResult.ErrorMessage = failuresStream.ToString();
failuresStream.Body.AcceptContents(new StackTraceHunter(s => testResult.ErrorStackTrace = s));
}
return testResult;
}
public void TestCompleted(ITest test, Guitar.Lib.TestResult result)
{
TestOutcome outcome = TestOutcome.None;
switch (result.Outcome)
{
case TestStatus.Passed:
outcome = TestOutcome.Passed;
break;
case TestStatus.Failed:
outcome = TestOutcome.Failed;
break;
case TestStatus.Ignored:
outcome = TestOutcome.Skipped;
break;
}
TestCase testToStart = _expectedTests[createDictionaryKey(test)];
_frameworkHandle.RecordEnd(testToStart, outcome);
TestResult testresult = new TestResult(testToStart) {ErrorMessage = result.Message, Outcome = outcome};
_frameworkHandle.RecordResult(testresult);
}
public VSTestResult ConvertTestResult(XmlNode resultNode)
{
TestCase ourCase = GetCachedTestCase(resultNode.GetAttribute("id"));
if (ourCase == null) return null;
VSTestResult ourResult = new VSTestResult(ourCase)
{
DisplayName = ourCase.DisplayName,
Outcome = GetTestOutcome(resultNode),
Duration = TimeSpan.FromSeconds(resultNode.GetAttribute("duration", 0.0))
};
var startTime = resultNode.GetAttribute("start-time");
if (startTime != null)
ourResult.StartTime = DateTimeOffset.Parse(startTime);
var endTime = resultNode.GetAttribute("end-time");
if (endTime != null)
ourResult.EndTime = DateTimeOffset.Parse(endTime);
// TODO: Remove this when NUnit provides a better duration
if (ourResult.Duration == TimeSpan.Zero && (ourResult.Outcome == TestOutcome.Passed || ourResult.Outcome == TestOutcome.Failed))
ourResult.Duration = TimeSpan.FromTicks(1);
ourResult.ComputerName = Environment.MachineName;
ourResult.ErrorMessage = GetErrorMessage(resultNode);
XmlNode stackTraceNode = resultNode.SelectSingleNode("failure/stack-trace");
if (stackTraceNode != null)
ourResult.ErrorStackTrace = stackTraceNode.InnerText;
XmlNode outputNode = resultNode.SelectSingleNode("output");
if (outputNode != null)
ourResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, outputNode.InnerText));
return ourResult;
}
/// <summary>
/// Converts a Boost.Test.Result.TestResult model into an equivalent
/// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model.
/// </summary>
/// <param name="result">The Boost.Test.Result.TestResult model to convert.</param>
/// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param>
/// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns>
public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test)
{
Utility.Code.Require(result, "result");
Utility.Code.Require(test, "test");
VSTestResult vsResult = new VSTestResult(test);
vsResult.ComputerName = Environment.MachineName;
vsResult.Outcome = GetTestOutcome(result.Result);
// Boost.Test.Result.TestResult.Duration is in microseconds
vsResult.Duration = TimeSpan.FromMilliseconds(result.Duration / 1000);
if (result.LogEntries.Count > 0)
{
foreach (TestResultMessage message in GetTestMessages(result))
{
vsResult.Messages.Add(message);
}
// Test using the TestOutcome type since elements from the
// Boost Result type may be collapsed into a particular value
if (vsResult.Outcome == VSTestOutcome.Failed)
{
LogEntry error = GetLastError(result);
if (error != null)
{
vsResult.ErrorMessage = GetErrorMessage(result);
vsResult.ErrorStackTrace = ((error.Source == null) ? null : error.Source.ToString());
}
}
}
return vsResult;
}
private VsTestResult MakeVsTestResult(string type, string method, double duration, string output, TestOutcome outcome)
{
VsTestResult result = new VsTestResult(GetTestCase(type, method)) {
Duration = TimeSpan.FromSeconds(duration),
ComputerName = Environment.MachineName,
Outcome = outcome,
};
if (!String.IsNullOrWhiteSpace(output))
result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, output));
return result;
}
private VsTestResult MakeVsTestResult(string name, string type, string method, double duration, string output, TestOutcome outcome)
{
string fullyQualifiedName = GetFullyQualifiedName(type, method);
TestCase testCase = GetTestCase(fullyQualifiedName);
VsTestResult result = new VsTestResult(testCase)
{
ComputerName = Environment.MachineName,
DisplayName = GetTestResultDisplayName(testCase.DisplayName, name, fullyQualifiedName),
Duration = TimeSpan.FromSeconds(duration),
Outcome = outcome,
};
// Work around VS considering a test "not run" when the duration is 0
if (result.Duration.TotalMilliseconds == 0)
result.Duration = TimeSpan.FromMilliseconds(1);
if (!String.IsNullOrEmpty(output))
result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, output));
return result;
}
/// <summary>
/// Converts a Boost.Test.Result.TestResult model into an equivalent
/// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model.
/// </summary>
/// <param name="result">The Boost.Test.Result.TestResult model to convert.</param>
/// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param>
/// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns>
public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test)
{
Utility.Code.Require(result, "result");
Utility.Code.Require(test, "test");
VSTestResult vsResult = new VSTestResult(test);
vsResult.ComputerName = Environment.MachineName;
vsResult.Outcome = GetTestOutcome(result.Result);
// Boost.Test.Result.TestResult.Duration is in microseconds
// 1 millisecond = 10,000 ticks
// => 1 microsecond = 10 ticks
// Reference: https://msdn.microsoft.com/en-us/library/zz841zbz(v=vs.110).aspx
long ticks = (long) Math.Min((result.Duration * 10), long.MaxValue);
// Clamp tick count to 1 in case Boost duration is listed as 0
vsResult.Duration = new TimeSpan(Math.Max(ticks, 1));
if (result.LogEntries.Count > 0)
{
foreach (TestResultMessage message in GetTestMessages(result))
{
vsResult.Messages.Add(message);
}
// Test using the TestOutcome type since elements from the
// Boost Result type may be collapsed into a particular value
if (vsResult.Outcome == VSTestOutcome.Failed)
{
LogEntry error = GetLastError(result);
if (error != null)
{
vsResult.ErrorMessage = GetErrorMessage(result);
if (error.Source != null)
{
//String format for a hyper linkable Stack Trace
//Reference: NUnit3 Test Adapter.
vsResult.ErrorStackTrace = string.Format(CultureInfo.InvariantCulture, "at {0}() in {1}:line {2}", vsResult.TestCase.DisplayName, ConvertSlashes(error.Source.File), error.Source.LineNumber);
}
}
}
}
return vsResult;
}
/// <summary>
/// Asserts that a log entry resulting in a test failure is correctly noted
/// in the Visual Studio TestResult.
/// </summary>
/// <param name="result">The Visual Studio TestResult to test</param>
/// <param name="entry">The log entry detailing a test case error</param>
private void AssertVsTestModelError(VSTestResult result, LogEntry entry)
{
Assert.That(result.ErrorMessage, Is.EqualTo(entry.Detail));
Assert.That(result.ErrorStackTrace, Is.EqualTo(entry.Source.ToString()));
}
private void VerifyTestResult(VSTestResult ourResult)
{
Assert.NotNull(ourResult, "TestResult not set");
VerifyTestCase(ourResult.TestCase);
Assert.AreEqual(Environment.MachineName, ourResult.ComputerName);
Assert.AreEqual(TestOutcome.Passed, ourResult.Outcome);
Assert.AreEqual("It passed!", ourResult.ErrorMessage);
Assert.AreEqual(TimeSpan.FromSeconds(1.234), ourResult.Duration);
}
/// <summary>
/// Asserts that a log entry resulting in a test failure is correctly noted
/// in the Visual Studio TestResult.
/// </summary>
/// <param name="result">The Visual Studio TestResult to test</param>
/// <param name="entry">The log entry detailing a test case error</param>
private void AssertVsTestModelError(VSTestResult result, LogEntry entry)
{
Assert.That(result.ErrorMessage, Is.EqualTo(entry.Detail));
}
/// <summary>
/// Asserts general Visual Studio TestResult properties for the default TestCase.
/// </summary>
/// <param name="result">The Visual Studio TestResult to test</param>
private void AssertVSTestModelProperties(VSTestResult result)
{
AssertVSTestModelProperties(result, this.TestCase);
}
/// <summary>
/// Asserts general Visual Studio TestResult properties for the provided TestCase.
/// </summary>
/// <param name="result">The Visual Studio TestResult to test</param>
/// <param name="test">The Visual Studio TestCase for which the result is based on</param>
private void AssertVSTestModelProperties(VSTestResult result, VSTestCase test)
{
Assert.That(result.TestCase, Is.EqualTo(test));
Assert.That(result.ComputerName, Is.EqualTo(Environment.MachineName));
}
/// <summary>
/// Generates a default TestResult for a timeout exception.
/// </summary>
/// <param name="test">The test which failed due to a timeout.</param>
/// <param name="ex">The exception related to this timeout.</param>
/// <returns>A timed-out, failed TestResult related to the provided test.</returns>
private static VSTestResult GenerateTimeoutResult(VSTestCase test, Boost.Runner.TimeoutException ex)
{
VSTestResult result = new VSTestResult(test);
result.ComputerName = Environment.MachineName;
result.Outcome = TestOutcome.Failed;
result.Duration = TimeSpan.FromMilliseconds(ex.Timeout);
result.ErrorMessage = "Timeout exceeded. Test ran for more than " + ex.Timeout + " ms.";
if (!string.IsNullOrEmpty(test.CodeFilePath))
{
result.ErrorStackTrace = new SourceFileInfo(test.CodeFilePath, test.LineNumber).ToString();
}
return result;
}
/// <summary>
/// Generates a default TestResult for a 'test not found' exception.
/// </summary>
/// <param name="test">The test which failed due to a timeout.</param>
/// <returns>A timed-out, failed TestResult related to the provided test.</returns>
private static VSTestResult GenerateNotFoundResult(VSTestCase test)
{
VSTestResult result = new VSTestResult(test);
result.ComputerName = Environment.MachineName;
result.Outcome = TestOutcome.Skipped;
result.ErrorMessage = GetNotFoundErrorMessage(test);
return result;
}
