// internal static bool AttachTestResultCode() // { // bool result = false; // // // // return result; // } internal static void AddTestResultTextDetail(ITestResultDetailCmdletBaseDataObject cmdlet, object detail) { ITestResultDetail testResultDetail = new TestResultDetail(); testResultDetail.AddTestResultDetail( TestResultDetailTypes.Comment, detail.ToString()); CurrentTestResult.Details.Add(testResultDetail); testResultDetail.DetailStatus = cmdlet.TestResultStatus; switch (cmdlet.TestResultStatus) { case TestStatuses.Failed: if (TestStatuses.KnownIssue != CurrentTestResult.enStatus) CurrentTestResult.enStatus = TestStatuses.Failed; break; case TestStatuses.Passed: if (TestStatuses.KnownIssue != CurrentTestResult.enStatus && TestStatuses.Failed != CurrentTestResult.enStatus) CurrentTestResult.enStatus = TestStatuses.Passed; break; case TestStatuses.NotRun: // nothing to do break; case TestStatuses.KnownIssue: CurrentTestResult.enStatus = TestStatuses.KnownIssue; break; default: // cmdlet.WriteVerbose(cmdlet, "TestResultStatus = ????"); // cmdlet.WriteVerbose(cmdlet, cmdlet.TestResultStatus.ToString()); break; } OnTmxNewTestResultDetailAdded(testResultDetail, null); if (cmdlet.Finished) { TmxHelper.TestCaseStarted = DateTime.Now; // 20130621 // TestData.CurrentTestScenario.TestResults.Add(new TestResult(TestData.CurrentTestScenario.Id, TestData.CurrentTestSuite.Id)); // TestData.CurrentTestScenario.TestResults[TestData.CurrentTestScenario.TestResults.Count - 1] = // TestData.CurrentTestResult; // 20130621 ITestResult newTestResult = new TestResult(CurrentTestScenario.Id, CurrentTestSuite.Id); if (CurrentTestResult == CurrentTestScenario.TestResults[CurrentTestScenario.TestResults.Count - 1]) { // autogeneration + Close-TmxTestResult CurrentTestScenario.TestResults.Add(newTestResult); CurrentTestResult = CurrentTestScenario.TestResults[CurrentTestScenario.TestResults.Count - 1]; } else { // Set-TmxCurrentTestResult + any closing CurrentTestScenario.TestResults.Add(CurrentTestResult); CurrentTestResult = newTestResult; } } }
// 20160116 internal static void AddTestResult(string closingTestResultName, // previousTestResultName string closingTestResultId, // previousTestResultId bool? passed, bool isKnownIssue, bool generateNextResult, // 20160116 // InvocationInfo myInvocation, // ErrorRecord error, Exception error, string testResultDescription, TestResultOrigins origin, bool skipAutomatic) { //TestData.InitCurrentTestScenario(); bool generated = TestResultOrigins.Automatic == origin; ITestResult currentTestResult; if (null != CurrentTestResult) { currentTestResult = CurrentTestResult; } else { // 20141204 // currentTestResult = // new TestResult( // TestData.CurrentTestScenario.Id, // TestData.CurrentTestSuite.Id); currentTestResult = new TestResult( CurrentTestScenario.Id, CurrentTestSuite.Id) { SuiteUniqueId = CurrentTestSuite.UniqueId, ScenarioUniqueId = CurrentTestScenario.UniqueId }; } if (string.IsNullOrEmpty(currentTestResult.Name)) { // if (closingTestResultName != null && // closingTestResultName != string.Empty && // closingTestResultName.Length > 0 && // TestData.CurrentTestResult != null && // closingTestResultName != TestData.CurrentTestResult.Name) { if (!string.IsNullOrEmpty(closingTestResultName) && ((CurrentTestResult != null && closingTestResultName != CurrentTestResult.Name) || null == CurrentTestResult)) { currentTestResult.Name = closingTestResultName; } else { currentTestResult.Name = Tmx_Core_Resources.TestData_AddTestResult_generated_test_result_name; } } else { // the current test result is a result that was preset // nothing to do } // setting test result's origin //if (generated) // currentTestResult.SetOrigin(TestResultOrigins.Automatic); //else // currentTestResult.SetOrigin(origin); currentTestResult.SetOrigin(generated ? TestResultOrigins.Automatic : origin); if (string.IsNullOrEmpty(currentTestResult.Id)) { if (!string.IsNullOrEmpty(closingTestResultId) && ((null != CurrentTestResult && closingTestResultId != CurrentTestResult.Id) || null == CurrentTestResult)) currentTestResult.Id = closingTestResultId; else currentTestResult.Id = GetTestResultId(); } else { // there already was the Id // nothing to do } if (passed != null) { if ((bool)passed) { currentTestResult.enStatus = TestStatuses.Passed; } else { currentTestResult.enStatus = TestStatuses.Failed; } if (isKnownIssue) { currentTestResult.enStatus = TestStatuses.KnownIssue; } } else { //currentTestResult.enStatus = TestResultStatuses.NotTested; // if there were no errors during the test case execution // it is marked as passed bool noErrors = true; if (null == currentTestResult.Error && TestStatuses.Failed != currentTestResult.enStatus && TestStatuses.KnownIssue != currentTestResult.enStatus) { //foreach (ITestResultDetail detail in currentTestResult.Details) { // if (null == ((TestResultDetail)detail).ErrorDetail) { // noErrors = false; // break; // } //} if (currentTestResult.Details.Any(detail => null == ((TestResultDetail) detail).ErrorDetail)) noErrors = false; } if (noErrors && null != passed) { currentTestResult.enStatus = TestStatuses.Passed; } } if (!string.IsNullOrEmpty(testResultDescription)) { currentTestResult.Description = testResultDescription; } if (generated) currentTestResult.SetGenerated(); if (TmxHelper.TestCaseStarted == DateTime.MinValue) TmxHelper.TestCaseStarted = DateTime.Now; currentTestResult.SetNow(); currentTestResult.SetTimeSpent( (currentTestResult.Timestamp - TmxHelper.TestCaseStarted).TotalSeconds); CurrentTestResult = currentTestResult; try { CurrentTestResult.PlatformUniqueId = CurrentTestScenario.PlatformUniqueId; CurrentTestResult.PlatformId = CurrentTestScenario.PlatformId; // 20141204 CurrentTestResult.SuiteUniqueId = CurrentTestSuite.UniqueId; CurrentTestResult.ScenarioUniqueId = CurrentTestScenario.UniqueId; } catch {} if (null != CurrentTestScenario.TestResults[CurrentTestScenario.TestResults.Count - 1] && TestResultOrigins.Logical == CurrentTestScenario.TestResults[CurrentTestScenario.TestResults.Count - 1].Origin && CurrentTestScenario.TestResults[CurrentTestScenario.TestResults.Count - 1] != CurrentTestResult && !string.IsNullOrEmpty(CurrentTestScenario.TestResults[CurrentTestScenario.TestResults.Count - 1].Name)) { TmxHelper.TestCaseStarted = DateTime.Now; // 20141204 // TestData.CurrentTestScenario.TestResults.Add(new TestResult(TestData.CurrentTestScenario.Id, TestData.CurrentTestSuite.Id)); CurrentTestScenario.TestResults.Add(new TestResult(CurrentTestScenario.Id, CurrentTestSuite.Id) { SuiteUniqueId = CurrentTestSuite.UniqueId, ScenarioUniqueId = CurrentTestScenario.UniqueId }); } /* if (null != TestData.CurrentTestScenario.TestResults[TestData.CurrentTestScenario.TestResults.Count - 1] && TestResultOrigins.Logical == TestData.CurrentTestScenario.TestResults[TestData.CurrentTestScenario.TestResults.Count - 1].Origin && TestData.CurrentTestScenario.TestResults[TestData.CurrentTestScenario.TestResults.Count - 1] != TestData.CurrentTestResult && null != TestData.CurrentTestScenario.TestResults[TestData.CurrentTestScenario.TestResults.Count - 1].Name && 0 < TestData.CurrentTestScenario.TestResults[TestData.CurrentTestScenario.TestResults.Count - 1].Name.Length) { TmxHelper.TestCaseStarted = System.DateTime.Now; TestData.CurrentTestScenario.TestResults.Add(new TestResult(TestData.CurrentTestScenario.Id, TestData.CurrentTestSuite.Id)); } */ CurrentTestScenario.TestResults[CurrentTestScenario.TestResults.Count - 1] = CurrentTestResult; #region Test Result's PowerShell data // 20160116 /* if (myInvocation != null) { TestData.CurrentTestResult.SetScriptName(TmxHelper.GetScriptName(myInvocation)); TestData.CurrentTestResult.SetLineNumber(TmxHelper.GetScriptLineNumber(myInvocation)); TestData.CurrentTestResult.SetPosition(TmxHelper.GetPipelinePosition(myInvocation)); // 20130329 try { if (((bool)passed && Preferences.LogScriptName_Passed) || (!(bool)passed && Preferences.LogScriptName_Failed)) { TestData.CurrentTestResult.Code += "script name: " + TestData.CurrentTestResult.ScriptName; } } catch {} try { if (((bool)passed && Preferences.LogLineNumber_Passed) || (!(bool)passed && Preferences.LogLineNumber_Failed)) { TestData.CurrentTestResult.Code += "\r\nline number: " + TestData.CurrentTestResult.LineNumber.ToString(); } } catch {} try { if (((bool)passed && Preferences.LogCode_Passed) || (!(bool)passed && Preferences.LogCode_Failed)) { TestData.CurrentTestResult.Code += "\r\ncode:\r\n" + myInvocation.Line; } } catch {} } if (error != null) { // 20150116 // TestData.CurrentTestResult.SetError(error); TestData.CurrentTestResult.SetError(error.Exception); } */ #endregion Test Result's PowerShell data var sourceTestResult = CurrentTestResult; SetScenarioStatus(skipAutomatic); SetSuiteStatus(skipAutomatic); if (generateNextResult) { // write current time TmxHelper.TestCaseStarted = DateTime.Now; CurrentTestScenario.TestResults.Add( new TestResult( CurrentTestScenario.Id, CurrentTestScenario.SuiteId) { SuiteUniqueId = CurrentTestScenario.SuiteUniqueId, ScenarioUniqueId = CurrentTestScenario.UniqueId }); CurrentTestResult = CurrentTestScenario.TestResults[CurrentTestScenario.TestResults.Count - 1]; } else { // write zero time TmxHelper.TestCaseStarted = DateTime.MinValue; CurrentTestResult = null; } if (Preferences.Storage) { using (var session = StorageHelper.SessionFactory.OpenSession()) { session.Save(CurrentTestResult); } } OnTmxNewTestResultClosed(sourceTestResult, null); }
/// <summary> /// Called when an individual test has finished. /// </summary> /// <param name="result"> /// The result. /// </param> public void TestFinished(TestResult result) { Console.Write(result.IsFailure || result.IsError ? 'F' : '.'); }
/// <summary> /// Called when the suite finished running. /// </summary> /// <param name="result"> /// The result of the test run. /// </param> public void SuiteFinished(TestResult result) { this.results.Add(result); }
/// <summary> /// Called when a run is finished with a result. /// </summary> /// <param name="result"> /// The result of the test run. /// </param> public void RunFinished(TestResult result) { }
/// <summary> /// Runs all of the tests defined in the specified assembly, specifically not running /// any of the tests in the calling type. /// </summary> /// <param name="assemblyToTest"> /// The assembly to be tested. /// </param> /// <param name="callingType"> /// No tests will be run that are defined in this type. /// </param> /// <returns> /// The result of running the suite of tests. /// </returns> private TestResult RunTestsInParallelImpl(Assembly assemblyToTest, Type callingType) { // NUnit requires this initialization step before any tests can be run CoreExtensions.Host.InitializeService(); this.Listener = new ParallelListener(); this.Filter = new IgnoreFilter(this.Listener); List<Test> concurrentTestFixtures; try { concurrentTestFixtures = assemblyToTest.GetTypes() .Where(type => callingType != type && TestFixtureBuilder.CanBuildFrom(type)) .Select(TestFixtureBuilder.BuildFrom) .ToList(); } catch (ReflectionTypeLoadException ex) { Console.WriteLine("ReflectionTypeLoadException caught..."); foreach (Exception loaderException in ex.LoaderExceptions) { Console.WriteLine("Loader Exception --------------------"); Console.WriteLine("Message:{0}", loaderException.Message); if (loaderException is FileNotFoundException) { Console.WriteLine((loaderException as FileNotFoundException).FusionLog); } Console.WriteLine("StackTrace: {0}", loaderException.StackTrace); } throw; } var stopwatch = new Stopwatch(); stopwatch.Start(); var currentDirectory = Path.GetDirectoryName(assemblyToTest.Location); Parallel.ForEach(concurrentTestFixtures, testFixture => { Environment.CurrentDirectory = currentDirectory; TestContext.Save(); testFixture.Run(this.Listener, this.Filter); }); stopwatch.Stop(); // Find out what the failures were var testResults = this.AllResults(this.Listener.Results) .Where(x => x.Results == null) .ToList(); var failuresAndErrors = testResults .Where(result => (result.IsFailure || result.IsError)) .ToList(); // Report the errors if there are any Console.WriteLine(); foreach (TestResult failure in failuresAndErrors) { Console.WriteLine("------------------------------------------------"); Console.WriteLine(failure.Test.TestName + " failed"); Console.WriteLine(failure.Message); Console.WriteLine(failure.StackTrace); } Console.WriteLine("================================================="); var totalErrors = testResults.Count(x => x.IsError); var totalFailures = testResults.Count(x => x.IsFailure); var totalInconclusive = testResults.Count(x => x.ResultState == ResultState.Inconclusive); var totalIgnored = testResults.Count(x => x.ResultState == ResultState.Ignored); var totalSkipped = testResults.Count(x => x.ResultState == ResultState.Skipped); var totalNotRunnable = testResults.Count(x => x.ResultState == ResultState.NotRunnable); var totalNotRun = totalInconclusive + totalIgnored + totalSkipped + totalNotRunnable; var totalRun = testResults.Count - totalNotRun; Console.WriteLine( String.Format("Tests run: {0}, Errors: {1}, Failures: {2}, Inconclusive: {3}, Time: {4} seconds", totalRun, totalErrors, totalFailures, totalInconclusive, stopwatch.Elapsed.TotalSeconds)); Console.WriteLine(String.Format(" Not run: {0}, Invalid: {1}, Ignored: {2}, Skipped: {3}", totalNotRun, failuresAndErrors.Count, totalIgnored, totalSkipped)); Console.WriteLine("================================================="); // Build up the results for return var finalResult = new TestResult(new TestName()); foreach (var result in this.Listener.Results) { finalResult.AddResult(result); } return finalResult; }