public void ConvertExceptionToVSTestResult() { LogEntryException exception = new LogEntryException("C string: some error", new SourceFileInfo("unknown location", 0)); exception.LastCheckpoint = new SourceFileInfo("boostunittestsample.cpp", 13); exception.CheckpointDetail = "Going to throw an exception"; BoostTestResult testCaseResult = new BoostTestResultBuilder(). For(this.TestCase). Aborted(). Duration(0). Log(exception). Build(); VSTestResult result = testCaseResult.AsVSTestResult(this.TestCase); AssertVSTestModelProperties(result); Assert.That(result.Outcome, Is.EqualTo(TestOutcome.Failed)); // A 0 duration should list as 1 tick to avoid UI issues in the test adapter Assert.That(result.Duration, Is.EqualTo(TimeSpan.FromTicks(1))); AssertVsTestModelError(result, exception); Assert.That(result.Messages.Count, Is.EqualTo(1)); TestResultMessage message = result.Messages.First(); Assert.That(message.Category, Is.EqualTo(TestResultMessage.StandardErrorCategory)); }
public void OutcomeOfRunWillBeCompletedIfNoTestsFails() { ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1"); ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2"); ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1); passResult1.Outcome = ObjectModel.TestOutcome.Passed; ObjectModel.TestResult passResult2 = new ObjectModel.TestResult(passTestCase2); passResult2.Outcome = ObjectModel.TestOutcome.Passed; ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1); skipResult1.Outcome = ObjectModel.TestOutcome.Skipped; Mock <TestResultEventArgs> pass1 = new Mock <TestResultEventArgs>(passResult1); Mock <TestResultEventArgs> pass2 = new Mock <TestResultEventArgs>(passResult2); Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1); this.testableTrxLogger.TestResultHandler(new object(), pass1.Object); this.testableTrxLogger.TestResultHandler(new object(), pass2.Object); this.testableTrxLogger.TestResultHandler(new object(), skip1.Object); var testRunCompleteEventArgs = CreateTestRunCompleteEventArgs(); this.testableTrxLogger.TestRunCompleteHandler(new object(), testRunCompleteEventArgs); Assert.AreEqual(TrxLoggerObjectModel.TestOutcome.Completed, this.testableTrxLogger.TestResultOutcome); }
public void ConvertFailToVSTestResult() { LogEntryError error = new LogEntryError("Error: 1 != 2", new SourceFileInfo("file.cpp", 10)); BoostTestResult testCaseResult = new BoostTestResultBuilder(). For(this.TestCase). Failed(). Duration(2500). Log(error). Build(); VSTestResult result = testCaseResult.AsVSTestResult(this.TestCase); AssertVSTestModelProperties(result); Assert.That(result.Outcome, Is.EqualTo(TestOutcome.Failed)); Assert.That(result.Duration, Is.EqualTo(Microseconds(2500))); AssertVsTestModelError(result, error); Assert.That(result.Messages.Count, Is.EqualTo(1)); TestResultMessage message = result.Messages.First(); Assert.That(message.Category, Is.EqualTo(TestResultMessage.StandardErrorCategory)); }
public void TestPassed(string name, string type, string method, double duration, string output) { VsTestResult result = MakeVsTestResult(type, method, duration, output, TestOutcome.Passed); recorder.RecordEnd(result.TestCase, result.Outcome); recorder.RecordResult(result); }
public void TestSkipped(string name, string type, string method, string reason) { VsTestResult result = MakeVsTestResult(type, method, 0.0, null, TestOutcome.Skipped); recorder.RecordEnd(result.TestCase, result.Outcome); recorder.RecordResult(result); }
/// <summary> /// Maps a <see cref="Framework.Results.TestResult"/> to a <see cref="Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult"/>. /// </summary> /// <param name="testCase">An existing <see cref="TestCase"/></param> /// <param name="result">The result to map</param> /// <returns>A Visual Studio <see cref="Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult"/></returns> public VSTestResult Map(TestCase testCase, PSTestResult result) { var vsResult = new VSTestResult(testCase) { Outcome = Map(result.Status), Duration = result.Duration.HasValue ? result.Duration.Value : TimeSpan.Zero, }; foreach (var artifact in result.Artifacts) { var attachmentSet = new AttachmentSet(artifact, "Attachment"); attachmentSet.Attachments.Add(new UriDataAttachment(artifact, string.Empty)); vsResult.Attachments.Add(attachmentSet); } var failedResult = result as FailedResult; if (failedResult != null) { vsResult.ErrorMessage = failedResult.Reason.Message; vsResult.ErrorStackTrace = failedResult.Reason.StackTrace; } return(vsResult); }
public VSTestResult ConvertTestResult(NUnitTestResult result) { TestCase ourCase = GetCachedTestCase(result.Test.TestName.UniqueName); if (ourCase == null) return null; VSTestResult ourResult = new VSTestResult(ourCase) { DisplayName = ourCase.DisplayName, Outcome = ResultStateToTestOutcome(result.ResultState), Duration = TimeSpan.FromSeconds(result.Time) }; // TODO: Remove this when NUnit provides a better duration if (ourResult.Duration == TimeSpan.Zero && (ourResult.Outcome == TestOutcome.Passed || ourResult.Outcome == TestOutcome.Failed)) ourResult.Duration = TimeSpan.FromTicks(1); ourResult.ComputerName = Environment.MachineName; // TODO: Stuff we don't yet set // StartTime - not in NUnit result // EndTime - not in NUnit result // Messages - could we add messages other than the error message? Where would they appear? // Attachments - don't exist in NUnit if (result.Message != null) ourResult.ErrorMessage = GetErrorMessage(result); if (!string.IsNullOrEmpty(result.StackTrace)) { string stackTrace = StackTraceFilter.Filter(result.StackTrace); ourResult.ErrorStackTrace = stackTrace; } return ourResult; }
/// <summary> /// Copies the result messages to unitTestResult /// </summary> /// <param name="unitTestResult">TRX TestResult</param> /// <param name="testResult"> rock steady test result</param> private void UpdateResultMessages(TrxObjectModel.TestResult unitTestResult, ObjectModel.TestResult testResult) { StringBuilder debugTrace = new StringBuilder(); StringBuilder stdErr = new StringBuilder(); StringBuilder stdOut = new StringBuilder(); foreach (TestResultMessage message in testResult.Messages) { if (TestResultMessage.AdditionalInfoCategory.Equals(message.Category, StringComparison.OrdinalIgnoreCase)) { unitTestResult.AddTextMessage(message.Text); } else if (TestResultMessage.DebugTraceCategory.Equals(message.Category, StringComparison.OrdinalIgnoreCase)) { debugTrace.AppendLine(message.Text); } else if (TestResultMessage.StandardErrorCategory.Equals(message.Category, StringComparison.OrdinalIgnoreCase)) { stdErr.AppendLine(message.Text); } else if (TestResultMessage.StandardOutCategory.Equals(message.Category, StringComparison.OrdinalIgnoreCase)) { stdOut.AppendLine(message.Text); } else { EqtTrace.Warning("The message category " + message.Category + " does not match any predefined category."); } } unitTestResult.DebugTrace = debugTrace.ToString(); unitTestResult.StdErr = stdErr.ToString(); unitTestResult.StdOut = stdOut.ToString(); }
public void ConvertMemoryLeakToVSTestResult() { LogEntryMemoryLeak leak = new LogEntryMemoryLeak(); leak.LeakLineNumber = 32; leak.LeakMemoryAllocationNumber = 836; leak.LeakLeakedDataContents = " Data: <`- Leak... > 60 2D BD 00 4C 65 61 6B 2E 2E 2E 00 CD CD CD CD "; leak.LeakSourceFilePath = @"C:\boostunittestsample.cpp"; leak.LeakSourceFileName = "boostunittestsample.cpp"; BoostTestResult testCaseResult = new BoostTestResultBuilder(). For(this.TestCase). Passed(). Duration(1000). Log(leak). Build(); VSTestResult result = testCaseResult.AsVSTestResult(this.TestCase); AssertVSTestModelProperties(result); Assert.That(result.Outcome, Is.EqualTo(TestOutcome.Passed)); Assert.That(result.Duration, Is.EqualTo(Microseconds(1000))); Assert.That(result.Messages.Count, Is.EqualTo(1)); TestResultMessage message = result.Messages.First(); Assert.That(message.Category, Is.EqualTo(TestResultMessage.StandardErrorCategory)); }
public void TestFinished(XmlNode resultNode) { TestResult ourResult = _testConverter.ConvertTestResult(resultNode); _recorder.RecordEnd(ourResult.TestCase, ourResult.Outcome); _recorder.RecordResult(ourResult); }
private VSTestResult GetBasicResult(XmlNode resultNode, IEnumerable <XmlNode> outputNodes) { var vsTest = GetCachedTestCase(resultNode.GetAttribute("id")); if (vsTest == null) { return(null); } string displayName = vsTest.FullyQualifiedName; vsTest.FullyQualifiedName = System.Text.RegularExpressions.Regex.Replace(vsTest.FullyQualifiedName, @" ?\(.*?\)", string.Empty); var vsResult = new VSTestResult(vsTest) { DisplayName = displayName, Outcome = GetTestOutcome(resultNode), Duration = TimeSpan.FromSeconds(resultNode.GetAttribute("duration", 0.0)) }; var startTime = resultNode.GetAttribute("start-time"); if (startTime != null) { vsResult.StartTime = DateTimeOffset.Parse(startTime, CultureInfo.InvariantCulture); } var endTime = resultNode.GetAttribute("end-time"); if (endTime != null) { vsResult.EndTime = DateTimeOffset.Parse(endTime, CultureInfo.InvariantCulture); } // TODO: Remove this when NUnit provides a better duration if (vsResult.Duration == TimeSpan.Zero && (vsResult.Outcome == TestOutcome.Passed || vsResult.Outcome == TestOutcome.Failed)) { vsResult.Duration = TimeSpan.FromTicks(1); } vsResult.ComputerName = Environment.MachineName; FillResultFromOutputNodes(outputNodes, vsResult); // Add stdOut messages from TestFinished element to vstest result var outputNode = resultNode.SelectSingleNode("output"); if (outputNode != null) { vsResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, outputNode.InnerText)); } var attachmentSet = ParseAttachments(resultNode); if (attachmentSet.Attachments.Count > 0) { vsResult.Attachments.Add(attachmentSet); } return(vsResult); }
public void RaiseTestResultShouldInvokeRegisteredEventHandlerIfTestResultEventArgsIsPassed() { EventWaitHandle waitHandle = new AutoResetEvent(false); bool testResultReceived = false; TestResultEventArgs eventArgs = null; var result = new TestResult(new TestCase("This is a string.", new Uri("some://uri"), "DummySourceFileName")); // Register for the test result event. loggerEvents.TestResult += (sender, e) => { testResultReceived = true; eventArgs = e; waitHandle.Set(); }; loggerEvents.EnableEvents(); // Send the test result event. loggerEvents.RaiseTestResult(new TestResultEventArgs(result)); var waitSuccess = waitHandle.WaitOne(500); Assert.IsTrue(waitSuccess, "Event must be raised within timeout."); Assert.IsTrue(testResultReceived); Assert.IsNotNull(eventArgs); Assert.AreEqual(result, eventArgs.Result); }
private static Mock <TestResultEventArgs> CreatePassTestResultEventArgsMock() { TestCase passTestCase = CreateTestCase("Pass1"); var passResult = new ObjectModel.TestResult(passTestCase); return(new Mock <TestResultEventArgs>(passResult)); }
public void TestResultHandlerKeepingTheTrackOfTotalTests() { ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1"); ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2"); ObjectModel.TestCase failTestCase1 = CreateTestCase("Fail1"); ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1); passResult1.Outcome = ObjectModel.TestOutcome.Passed; ObjectModel.TestResult passResult2 = new ObjectModel.TestResult(passTestCase2); passResult2.Outcome = ObjectModel.TestOutcome.Passed; ObjectModel.TestResult failResult1 = new ObjectModel.TestResult(failTestCase1); failResult1.Outcome = ObjectModel.TestOutcome.Failed; ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1); skipResult1.Outcome = ObjectModel.TestOutcome.Skipped; Mock <TestResultEventArgs> pass1 = new Mock <TestResultEventArgs>(passResult1); Mock <TestResultEventArgs> pass2 = new Mock <TestResultEventArgs>(passResult2); Mock <TestResultEventArgs> fail1 = new Mock <TestResultEventArgs>(failResult1); Mock <TestResultEventArgs> skip1 = new Mock <TestResultEventArgs>(skipResult1); this.testableTrxLogger.TestResultHandler(new object(), pass1.Object); this.testableTrxLogger.TestResultHandler(new object(), pass2.Object); this.testableTrxLogger.TestResultHandler(new object(), fail1.Object); this.testableTrxLogger.TestResultHandler(new object(), skip1.Object); Assert.AreEqual(this.testableTrxLogger.TotalTestCount, 4, "Passed Tests"); }
private void ExecuteTestRun(TestCollection testCollection, IFrameworkHandle frameworkHandle, Dictionary <Test, TestCase> testMapping) { try { _testRun = testCollection.CreateTestRun(); _testRun.TestCompleted += result => { var vsResult = new VisualStudioTestResult(testMapping[result.Test]) { Outcome = OutcomeFromResult(result.Result), Duration = result.Duration, ErrorMessage = result.Output, ErrorStackTrace = GetCombinedStackTrace(result.Exception) }; vsResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, result.Output)); vsResult.ErrorStackTrace = GetCombinedStackTrace(result.Exception); frameworkHandle.RecordResult(vsResult); }; _testRun.ExecuteAsync().GetAwaiter().GetResult(); } catch (Exception ex) { File.WriteAllText("C:/Users/seamillo/Desktop/jazsharp.testadapter.log", ex.ToString()); throw; } }
private VsTestResult MakeVsTestResult(TestOutcome outcome, ITestCase testCase, string displayName, double executionTime = 0.0, string output = null) { var vsTestCase = testCases[testCase]; var fqTestMethodName = String.Format("{0}.{1}", testCase.TestMethod.TestClass.Class.Name, testCase.TestMethod.Method.Name); var vsDisplayName = settings.GetDisplayName(displayName, testCase.TestMethod.Method.Name, fqTestMethodName); var result = new VsTestResult(vsTestCase) { #if !WINDOWS_PHONE_APP ComputerName = Environment.MachineName, #endif DisplayName = vsDisplayName, Duration = TimeSpan.FromSeconds(executionTime), Outcome = outcome, }; // Work around VS considering a test "not run" when the duration is 0 if (result.Duration.TotalMilliseconds == 0) { result.Duration = TimeSpan.FromMilliseconds(1); } if (!String.IsNullOrEmpty(output)) { result.Messages.Add(new VsTestResultMessage(VsTestResultMessage.StandardOutCategory, output)); } return(result); }
public void TestResultHandlerShouldAddSingleTestEntryForOrderedTest() { ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); ObjectModel.TestCase testCase2 = CreateTestCase("TestCase2"); ObjectModel.TestCase testCase3 = CreateTestCase("TestCase3"); Guid parentExecutionId = Guid.NewGuid(); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId); result1.SetPropertyValue <Guid>(TrxLoggerConstants.TestTypeProperty, TrxLoggerConstants.OrderedTestTypeGuid); ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase2); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase3); result3.Outcome = ObjectModel.TestOutcome.Failed; result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1); Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2); Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object); Assert.AreEqual(this.testableTrxLogger.TestEntryCount, 1, "TestResultHandler is adding multiple test entries for ordered test."); }
public void GetHashCodeShouldReturnAnUniqueHash() { var result = new TestResult(new TestCase()); var resultInfo = new TestResultInfo(result, string.Empty, string.Empty, string.Empty); Assert.AreNotEqual(new TestResult(new TestCase()).GetHashCode(), resultInfo.GetHashCode()); }
public void EqualsShouldReturnFalseForNonTestResultInfoObject() { var result = new TestResult(new TestCase()); var resultInfo = new TestResultInfo(result, string.Empty, string.Empty, string.Empty); Assert.IsFalse(resultInfo.Equals(new ())); }
public void TestResultHandlerShouldAddSingleTestElementForDataDrivenTests() { ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); Guid parentExecutionId = Guid.NewGuid(); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId); ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase1); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase1); result3.Outcome = ObjectModel.TestOutcome.Failed; result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1); Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2); Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object); Assert.AreEqual(this.testableTrxLogger.UnitTestElementCount, 1, "TestResultHandler is adding multiple test elements for data driven tests."); }
public void TestResultHandlerShouldAddHierarchicalResultsIfParentTestResultIsPresent() { ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); Guid parentExecutionId = Guid.NewGuid(); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId); ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase1); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase1); result3.Outcome = ObjectModel.TestOutcome.Failed; result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1); Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2); Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object); Assert.AreEqual(this.testableTrxLogger.TestResultCount, 1, "TestResultHandler is not creating hierarchical results when parent result is present."); Assert.AreEqual(this.testableTrxLogger.TotalTestCount, 3, "TestResultHandler is not adding all inner results in parent test result."); }
/// <summary> /// Returns the QToolsCommon.TestResult object created from rockSteady TestResult. /// </summary> /// <param name="rockSteadyTestResult"> rock steady test result</param> /// <param name="testElement"> testElement of that test</param> /// <param name="testOutcome"> Test outcome </param> /// <param name="testRun"> test run object </param> /// <returns> TestResult object </returns> private static TrxObjectModel.UnitTestResult GetQToolsTestResultFromTestResult( ObjectModel.TestResult rockSteadyTestResult, TrxObjectModel.UnitTestElement testElement, TrxObjectModel.TestOutcome testOutcome, TrxObjectModel.TestRun testRun) { UnitTestResult testResult = new UnitTestResult(Environment.MachineName, testRun.Id, testElement, testOutcome); if (rockSteadyTestResult.ErrorMessage != null) { testResult.ErrorMessage = rockSteadyTestResult.ErrorMessage; } if (rockSteadyTestResult.ErrorStackTrace != null) { testResult.ErrorStackTrace = rockSteadyTestResult.ErrorStackTrace; } // set start and end times if (rockSteadyTestResult.EndTime != null) { testResult.EndTime = rockSteadyTestResult.EndTime.UtcDateTime; } if (rockSteadyTestResult.StartTime != null) { testResult.StartTime = rockSteadyTestResult.StartTime.UtcDateTime; } if (rockSteadyTestResult.Duration != null) { testResult.Duration = rockSteadyTestResult.Duration; } return(testResult); }
private static VSResult ConvertToVsResult(TestCase @case, UnitTest unitTest, SUResult sunitResult) { var result = new VSResult(@case) { DisplayName = unitTest.Name, Outcome = sunitResult.Kind.ToTestOutcome() }; switch (sunitResult) { case UnexpectedExceptionResult errorResult: string heading = $"Unexpected {errorResult.Exception.GetType().Name}"; string message = IndentLines(errorResult.Exception.Message, " "); result.ErrorMessage = $"{heading}\n{message}"; result.ErrorStackTrace = errorResult.Exception.StackTrace; break; case RanSuccessfullyResult ranResult when !ranResult.Result.Passed: result.ErrorMessage = ranResult.Result.ToString(); break; case RanSuccessfullyResult ranResult when ranResult.Result.Passed: break; default: result.ErrorMessage = $"{sunitResult.Kind}: {sunitResult}"; break; } return(result); }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { var testCaseLookup = tests .ToDictionary(test => test.ToUnitTest(), test => test); cts = new CancellationTokenSource(); try { var resultObservable = TestRunner.RunTests(testCaseLookup.Select(kvp => kvp.Key)); resultObservable.Do(sunitResult => { UnitTest unitTest = sunitResult.UnitTest; TestCase testCase = testCaseLookup[unitTest]; VSResult vsResult = ConvertToVsResult(testCase, unitTest, sunitResult); frameworkHandle.RecordResult(vsResult); }).ToTask() .Wait(cts.Token); } catch (Exception error) { frameworkHandle.SendMessage(TestMessageLevel.Error, $"Unexpected {error.GetType().FullName}"); frameworkHandle.SendMessage(TestMessageLevel.Error, error.Message); frameworkHandle.SendMessage(TestMessageLevel.Error, error.StackTrace); } }
VsTestResult MakeVsTestResult(TestOutcome outcome, ITestCase testCase, string displayName, double executionTime = 0.0, string output = null) { var vsTestCase = FindTestCase(testCase); if (vsTestCase == null) { return(null); } var result = new VsTestResult(vsTestCase) { #if !WINDOWS_UAP ComputerName = Environment.MachineName, #endif DisplayName = displayName, Duration = TimeSpan.FromSeconds(executionTime), Outcome = outcome, }; // Work around VS considering a test "not run" when the duration is 0 if (result.Duration.TotalMilliseconds == 0) { result.Duration = TimeSpan.FromMilliseconds(1); } if (!string.IsNullOrEmpty(output)) { result.Messages.Add(new VsTestResultMessage(VsTestResultMessage.StandardOutCategory, output)); } return(result); }
private VsTestResult MakeVsTestResult(ITestResultMessage testResult, TestOutcome outcome) { var testCase = testCases[testResult.TestCase]; var result = new VsTestResult(testCase) { ComputerName = Environment.MachineName, DisplayName = testResult.TestDisplayName, Duration = TimeSpan.FromSeconds((double)testResult.ExecutionTime), Outcome = outcome, }; // Work around VS considering a test "not run" when the duration is 0 if (result.Duration.TotalMilliseconds == 0) { result.Duration = TimeSpan.FromMilliseconds(1); } if (!String.IsNullOrEmpty(testResult.Output)) { result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testResult.Output)); } return(result); }
public void RunTestsShouldIterateThroughAllExecutors() { var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; var executorUriExtensionMap = new List <Tuple <Uri, string> > { new Tuple <Uri, string>(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation), new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), assemblyLocation) }; // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); }; this.runTestsInstance.InvokeExecutorCallback = (executor, executorUriExtensionTuple, runContext, frameworkHandle) => { var testCase = new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll"); var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult); }; this.runTestsInstance.RunTests(); var expectedUris = new string[] { BadBaseRunTestsExecutorUri.ToLower(), BaseRunTestsExecutorUri.ToLower() }; CollectionAssert.AreEqual(expectedUris, this.runTestsInstance.GetExecutorUrisThatRanTests.ToArray()); }
public void TestResultHandlerShouldAddHierarchicalResultsForOrderedTest() { ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); ObjectModel.TestCase testCase2 = CreateTestCase("TestCase2"); ObjectModel.TestCase testCase3 = CreateTestCase("TestCase3"); Guid parentExecutionId = Guid.NewGuid(); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, parentExecutionId); result1.SetPropertyValue <Guid>(TrxLoggerConstants.TestTypeProperty, TrxLoggerConstants.OrderedTestTypeGuid); ObjectModel.TestResult result2 = new ObjectModel.TestResult(testCase2); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result2.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); ObjectModel.TestResult result3 = new ObjectModel.TestResult(testCase3); result3.Outcome = ObjectModel.TestOutcome.Failed; result3.SetPropertyValue <Guid>(TrxLoggerConstants.ExecutionIdProperty, Guid.NewGuid()); result3.SetPropertyValue <Guid>(TrxLoggerConstants.ParentExecIdProperty, parentExecutionId); Mock <TestResultEventArgs> resultEventArg1 = new Mock <TestResultEventArgs>(result1); Mock <TestResultEventArgs> resultEventArg2 = new Mock <TestResultEventArgs>(result2); Mock <TestResultEventArgs> resultEventArg3 = new Mock <TestResultEventArgs>(result3); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg1.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg2.Object); this.testableTrxLogger.TestResultHandler(new object(), resultEventArg3.Object); Assert.AreEqual(1, this.testableTrxLogger.TestResultCount, "TestResultHandler is not creating hierarchical results for ordered test."); Assert.AreEqual(3, this.testableTrxLogger.TotalTestCount, "TestResultHandler is not adding all inner results in ordered test."); }
private Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult GetTestResult(int index) { var tc = new TestCase("Test" + index, new Uri("executor://dummy"), "DummySourceFileName"); var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(tc); testResult.TestCase.Id = Guid.NewGuid(); return(testResult); }
/// <summary> /// Gets parent execution id of test result. /// </summary> /// <param name="testResult"></param> /// <returns>Parent execution id.</returns> public Guid GetParentExecutionId(ObjectModel.TestResult testResult) { TestProperty parentExecutionIdProperty = testResult.Properties.FirstOrDefault( property => property.Id.Equals(Constants.ParentExecutionIdPropertyIdentifier)); return(parentExecutionIdProperty == null ? Guid.Empty : testResult.GetPropertyValue(parentExecutionIdProperty, Guid.Empty)); }
public void RecordResultShouldUpdateTestRunCache() { var testCase = new TestCase("A.C.M", new Uri("executor://dummy"), "A"); var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); var testRecorder = new TestExecutionRecorder(null, this.testableTestRunCache); testRecorder.RecordResult(testResult); Assert.IsTrue(this.testableTestRunCache.TestResultList.Contains(testResult)); }
/// <summary> /// Converts a Boost.Test.Result.TestResult model into an equivalent /// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model. /// </summary> /// <param name="result">The Boost.Test.Result.TestResult model to convert.</param> /// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param> /// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns> public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test) { Utility.Code.Require(result, "result"); Utility.Code.Require(test, "test"); VSTestResult vsResult = new VSTestResult(test); vsResult.ComputerName = Environment.MachineName; vsResult.Outcome = GetTestOutcome(result.Result); // Boost.Test.Result.TestResult.Duration is in microseconds // 1 millisecond = 10,000 ticks // => 1 microsecond = 10 ticks // Reference: https://msdn.microsoft.com/en-us/library/zz841zbz(v=vs.110).aspx long ticks = (long) Math.Min((result.Duration * 10), long.MaxValue); // Clamp tick count to 1 in case Boost duration is listed as 0 vsResult.Duration = new TimeSpan(Math.Max(ticks, 1)); if (result.LogEntries.Count > 0) { foreach (TestResultMessage message in GetTestMessages(result)) { vsResult.Messages.Add(message); } // Test using the TestOutcome type since elements from the // Boost Result type may be collapsed into a particular value if (vsResult.Outcome == VSTestOutcome.Failed) { LogEntry error = GetLastError(result); if (error != null) { vsResult.ErrorMessage = GetErrorMessage(result); vsResult.ErrorStackTrace = ((error.Source == null) ? null : error.Source.ToString()); } } } return vsResult; }
public TestResult BuildTestResult(TestData testData, TestStepRun testStepRun, TestCase testCase) { var testResult = new TestResult(testCase) { DisplayName = testData.Name, StartTime = testStepRun.StartTime, EndTime = testStepRun.EndTime, Duration = testStepRun.Result.Duration, Outcome = GetOutcome(testStepRun.Result.Outcome.Status), }; var failuresStream = testStepRun.TestLog.GetStream(MarkupStreamNames.Failures); if (failuresStream != null) { testResult.ErrorMessage = failuresStream.ToString(); failuresStream.Body.AcceptContents(new StackTraceHunter(s => testResult.ErrorStackTrace = s)); } return testResult; }
public void TestCompleted(ITest test, Guitar.Lib.TestResult result) { TestOutcome outcome = TestOutcome.None; switch (result.Outcome) { case TestStatus.Passed: outcome = TestOutcome.Passed; break; case TestStatus.Failed: outcome = TestOutcome.Failed; break; case TestStatus.Ignored: outcome = TestOutcome.Skipped; break; } TestCase testToStart = _expectedTests[createDictionaryKey(test)]; _frameworkHandle.RecordEnd(testToStart, outcome); TestResult testresult = new TestResult(testToStart) {ErrorMessage = result.Message, Outcome = outcome}; _frameworkHandle.RecordResult(testresult); }
public VSTestResult ConvertTestResult(XmlNode resultNode) { TestCase ourCase = GetCachedTestCase(resultNode.GetAttribute("id")); if (ourCase == null) return null; VSTestResult ourResult = new VSTestResult(ourCase) { DisplayName = ourCase.DisplayName, Outcome = GetTestOutcome(resultNode), Duration = TimeSpan.FromSeconds(resultNode.GetAttribute("duration", 0.0)) }; var startTime = resultNode.GetAttribute("start-time"); if (startTime != null) ourResult.StartTime = DateTimeOffset.Parse(startTime); var endTime = resultNode.GetAttribute("end-time"); if (endTime != null) ourResult.EndTime = DateTimeOffset.Parse(endTime); // TODO: Remove this when NUnit provides a better duration if (ourResult.Duration == TimeSpan.Zero && (ourResult.Outcome == TestOutcome.Passed || ourResult.Outcome == TestOutcome.Failed)) ourResult.Duration = TimeSpan.FromTicks(1); ourResult.ComputerName = Environment.MachineName; ourResult.ErrorMessage = GetErrorMessage(resultNode); XmlNode stackTraceNode = resultNode.SelectSingleNode("failure/stack-trace"); if (stackTraceNode != null) ourResult.ErrorStackTrace = stackTraceNode.InnerText; XmlNode outputNode = resultNode.SelectSingleNode("output"); if (outputNode != null) ourResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, outputNode.InnerText)); return ourResult; }
/// <summary> /// Converts a Boost.Test.Result.TestResult model into an equivalent /// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model. /// </summary> /// <param name="result">The Boost.Test.Result.TestResult model to convert.</param> /// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param> /// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns> public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test) { Utility.Code.Require(result, "result"); Utility.Code.Require(test, "test"); VSTestResult vsResult = new VSTestResult(test); vsResult.ComputerName = Environment.MachineName; vsResult.Outcome = GetTestOutcome(result.Result); // Boost.Test.Result.TestResult.Duration is in microseconds vsResult.Duration = TimeSpan.FromMilliseconds(result.Duration / 1000); if (result.LogEntries.Count > 0) { foreach (TestResultMessage message in GetTestMessages(result)) { vsResult.Messages.Add(message); } // Test using the TestOutcome type since elements from the // Boost Result type may be collapsed into a particular value if (vsResult.Outcome == VSTestOutcome.Failed) { LogEntry error = GetLastError(result); if (error != null) { vsResult.ErrorMessage = GetErrorMessage(result); vsResult.ErrorStackTrace = ((error.Source == null) ? null : error.Source.ToString()); } } } return vsResult; }
private VsTestResult MakeVsTestResult(string type, string method, double duration, string output, TestOutcome outcome) { VsTestResult result = new VsTestResult(GetTestCase(type, method)) { Duration = TimeSpan.FromSeconds(duration), ComputerName = Environment.MachineName, Outcome = outcome, }; if (!String.IsNullOrWhiteSpace(output)) result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, output)); return result; }
private VsTestResult MakeVsTestResult(string name, string type, string method, double duration, string output, TestOutcome outcome) { string fullyQualifiedName = GetFullyQualifiedName(type, method); TestCase testCase = GetTestCase(fullyQualifiedName); VsTestResult result = new VsTestResult(testCase) { ComputerName = Environment.MachineName, DisplayName = GetTestResultDisplayName(testCase.DisplayName, name, fullyQualifiedName), Duration = TimeSpan.FromSeconds(duration), Outcome = outcome, }; // Work around VS considering a test "not run" when the duration is 0 if (result.Duration.TotalMilliseconds == 0) result.Duration = TimeSpan.FromMilliseconds(1); if (!String.IsNullOrEmpty(output)) result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, output)); return result; }
/// <summary> /// Converts a Boost.Test.Result.TestResult model into an equivalent /// Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult model. /// </summary> /// <param name="result">The Boost.Test.Result.TestResult model to convert.</param> /// <param name="test">The Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase model which is related to the result.</param> /// <returns>The Boost.Test.Result.TestResult model converted into its Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult counterpart.</returns> public static VSTestResult AsVSTestResult(this BoostTestAdapter.Boost.Results.TestResult result, VSTestCase test) { Utility.Code.Require(result, "result"); Utility.Code.Require(test, "test"); VSTestResult vsResult = new VSTestResult(test); vsResult.ComputerName = Environment.MachineName; vsResult.Outcome = GetTestOutcome(result.Result); // Boost.Test.Result.TestResult.Duration is in microseconds // 1 millisecond = 10,000 ticks // => 1 microsecond = 10 ticks // Reference: https://msdn.microsoft.com/en-us/library/zz841zbz(v=vs.110).aspx long ticks = (long) Math.Min((result.Duration * 10), long.MaxValue); // Clamp tick count to 1 in case Boost duration is listed as 0 vsResult.Duration = new TimeSpan(Math.Max(ticks, 1)); if (result.LogEntries.Count > 0) { foreach (TestResultMessage message in GetTestMessages(result)) { vsResult.Messages.Add(message); } // Test using the TestOutcome type since elements from the // Boost Result type may be collapsed into a particular value if (vsResult.Outcome == VSTestOutcome.Failed) { LogEntry error = GetLastError(result); if (error != null) { vsResult.ErrorMessage = GetErrorMessage(result); if (error.Source != null) { //String format for a hyper linkable Stack Trace //Reference: NUnit3 Test Adapter. vsResult.ErrorStackTrace = string.Format(CultureInfo.InvariantCulture, "at {0}() in {1}:line {2}", vsResult.TestCase.DisplayName, ConvertSlashes(error.Source.File), error.Source.LineNumber); } } } } return vsResult; }
/// <summary> /// Asserts that a log entry resulting in a test failure is correctly noted /// in the Visual Studio TestResult. /// </summary> /// <param name="result">The Visual Studio TestResult to test</param> /// <param name="entry">The log entry detailing a test case error</param> private void AssertVsTestModelError(VSTestResult result, LogEntry entry) { Assert.That(result.ErrorMessage, Is.EqualTo(entry.Detail)); Assert.That(result.ErrorStackTrace, Is.EqualTo(entry.Source.ToString())); }
private void VerifyTestResult(VSTestResult ourResult) { Assert.NotNull(ourResult, "TestResult not set"); VerifyTestCase(ourResult.TestCase); Assert.AreEqual(Environment.MachineName, ourResult.ComputerName); Assert.AreEqual(TestOutcome.Passed, ourResult.Outcome); Assert.AreEqual("It passed!", ourResult.ErrorMessage); Assert.AreEqual(TimeSpan.FromSeconds(1.234), ourResult.Duration); }
/// <summary> /// Asserts that a log entry resulting in a test failure is correctly noted /// in the Visual Studio TestResult. /// </summary> /// <param name="result">The Visual Studio TestResult to test</param> /// <param name="entry">The log entry detailing a test case error</param> private void AssertVsTestModelError(VSTestResult result, LogEntry entry) { Assert.That(result.ErrorMessage, Is.EqualTo(entry.Detail)); }
/// <summary> /// Asserts general Visual Studio TestResult properties for the default TestCase. /// </summary> /// <param name="result">The Visual Studio TestResult to test</param> private void AssertVSTestModelProperties(VSTestResult result) { AssertVSTestModelProperties(result, this.TestCase); }
/// <summary> /// Asserts general Visual Studio TestResult properties for the provided TestCase. /// </summary> /// <param name="result">The Visual Studio TestResult to test</param> /// <param name="test">The Visual Studio TestCase for which the result is based on</param> private void AssertVSTestModelProperties(VSTestResult result, VSTestCase test) { Assert.That(result.TestCase, Is.EqualTo(test)); Assert.That(result.ComputerName, Is.EqualTo(Environment.MachineName)); }
/// <summary> /// Generates a default TestResult for a timeout exception. /// </summary> /// <param name="test">The test which failed due to a timeout.</param> /// <param name="ex">The exception related to this timeout.</param> /// <returns>A timed-out, failed TestResult related to the provided test.</returns> private static VSTestResult GenerateTimeoutResult(VSTestCase test, Boost.Runner.TimeoutException ex) { VSTestResult result = new VSTestResult(test); result.ComputerName = Environment.MachineName; result.Outcome = TestOutcome.Failed; result.Duration = TimeSpan.FromMilliseconds(ex.Timeout); result.ErrorMessage = "Timeout exceeded. Test ran for more than " + ex.Timeout + " ms."; if (!string.IsNullOrEmpty(test.CodeFilePath)) { result.ErrorStackTrace = new SourceFileInfo(test.CodeFilePath, test.LineNumber).ToString(); } return result; }
/// <summary> /// Generates a default TestResult for a 'test not found' exception. /// </summary> /// <param name="test">The test which failed due to a timeout.</param> /// <returns>A timed-out, failed TestResult related to the provided test.</returns> private static VSTestResult GenerateNotFoundResult(VSTestCase test) { VSTestResult result = new VSTestResult(test); result.ComputerName = Environment.MachineName; result.Outcome = TestOutcome.Skipped; result.ErrorMessage = GetNotFoundErrorMessage(test); return result; }