public TrxObjectModel.TestOutcome ToOutcome(ObjectModel.TestOutcome rockSteadyOutcome)
{
TrxObjectModel.TestOutcome outcome = TrxObjectModel.TestOutcome.Failed;
switch (rockSteadyOutcome)
{
case ObjectModel.TestOutcome.Failed:
outcome = TrxObjectModel.TestOutcome.Failed;
break;
case ObjectModel.TestOutcome.Passed:
outcome = TrxObjectModel.TestOutcome.Passed;
break;
case ObjectModel.TestOutcome.Skipped:
case ObjectModel.TestOutcome.None:
case ObjectModel.TestOutcome.NotFound:
outcome = TrxObjectModel.TestOutcome.NotExecuted;
break;
default:
Debug.Fail("Unexpected Outcome.");
break;
}
return(outcome);
}
/// <summary>
/// Creates test result
/// </summary>
/// <param name="executionId"></param>
/// <param name="parentExecutionId"></param>
/// <param name="testType"></param>
/// <param name="testElement"></param>
/// <param name="parentTestElement"></param>
/// <param name="parentTestResult"></param>
/// <param name="rocksteadyTestResult"></param>
/// <returns>Trx test result</returns>
private ITestResult CreateTestResult(Guid executionId, Guid parentExecutionId, TestType testType,
ITestElement testElement, ITestElement parentTestElement, ITestResult parentTestResult, ObjectModel.TestResult rocksteadyTestResult)
{
// Create test result
TrxLoggerObjectModel.TestOutcome testOutcome = this.converter.ToOutcome(rocksteadyTestResult.Outcome);
var testResult = this.converter.ToTestResult(testElement.Id.Id, executionId, parentExecutionId, testElement.Name,
this.testResultsDirPath, testType, testElement.CategoryId, testOutcome, this.testRun, rocksteadyTestResult);
// Normal result scenario
if (parentTestResult == null)
{
this.results.TryAdd(executionId, testResult);
return(testResult);
}
// Ordered test inner result scenario
if (parentTestElement != null && parentTestElement.TestType.Equals(TrxLoggerConstants.OrderedTestType))
{
(parentTestResult as TestResultAggregation).InnerResults.Add(testResult);
this.innerResults.TryAdd(executionId, testResult);
return(testResult);
}
// Data driven inner result scenario
if (parentTestElement != null && parentTestElement.TestType.Equals(TrxLoggerConstants.UnitTestType))
{
(parentTestResult as TestResultAggregation).InnerResults.Add(testResult);
testResult.DataRowInfo = (parentTestResult as TestResultAggregation).InnerResults.Count;
testResult.ResultType = TrxLoggerConstants.InnerDataDrivenResultType;
parentTestResult.ResultType = TrxLoggerConstants.ParentDataDrivenResultType;
return(testResult);
}
return(testResult);
}
/// <summary>
/// Called when a test message is received.
/// </summary>
/// <param name="sender">
/// The sender.
/// </param>
/// <param name="e">
/// Event args
/// </param>
internal void TestMessageHandler(object sender, TestRunMessageEventArgs e)
{
ValidateArg.NotNull(sender, "sender");
ValidateArg.NotNull(e, "e");
RunInfo runMessage;
switch (e.Level)
{
case TestMessageLevel.Informational:
this.AddRunLevelInformationalMessage(e.Message);
break;
case TestMessageLevel.Warning:
runMessage = new RunInfo(e.Message, null, Environment.MachineName, TrxLoggerObjectModel.TestOutcome.Warning);
this.runLevelErrorsAndWarnings.Add(runMessage);
break;
case TestMessageLevel.Error:
this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Failed;
runMessage = new RunInfo(e.Message, null, Environment.MachineName, TrxLoggerObjectModel.TestOutcome.Error);
this.runLevelErrorsAndWarnings.Add(runMessage);
break;
default:
Debug.Fail("TrxLogger.TestMessageHandler: The test message level is unrecognized: {0}", e.Level.ToString());
break;
}
}
/// <summary>
/// Returns the QToolsCommon.TestResult object created from rockSteady TestResult.
/// </summary>
/// <param name="rockSteadyTestResult"> rock steady test result</param>
/// <param name="testElement"> testElement of that test</param>
/// <param name="testOutcome"> Test outcome </param>
/// <param name="testRun"> test run object </param>
/// <returns> TestResult object </returns>
private static TrxObjectModel.UnitTestResult GetQToolsTestResultFromTestResult(
ObjectModel.TestResult rockSteadyTestResult,
TrxObjectModel.UnitTestElement testElement,
TrxObjectModel.TestOutcome testOutcome,
TrxObjectModel.TestRun testRun)
{
UnitTestResult testResult = new UnitTestResult(Environment.MachineName, testRun.Id, testElement, testOutcome);
if (rockSteadyTestResult.ErrorMessage != null)
{
testResult.ErrorMessage = rockSteadyTestResult.ErrorMessage;
}
if (rockSteadyTestResult.ErrorStackTrace != null)
{
testResult.ErrorStackTrace = rockSteadyTestResult.ErrorStackTrace;
}
// set start and end times
if (rockSteadyTestResult.EndTime != null)
{
testResult.EndTime = rockSteadyTestResult.EndTime.UtcDateTime;
}
if (rockSteadyTestResult.StartTime != null)
{
testResult.StartTime = rockSteadyTestResult.StartTime.UtcDateTime;
}
if (rockSteadyTestResult.Duration != null)
{
testResult.Duration = rockSteadyTestResult.Duration;
}
return(testResult);
}
/// <summary>
/// Called when a test result is received.
/// </summary>
/// <param name="sender">
/// The sender.
/// </param>
/// <param name="e">
/// The eventArgs.
/// </param>
internal void TestResultHandler(object sender, TestResultEventArgs e)
{
// Create test run
if (this.testRun == null)
{
CreateTestRun();
}
// Convert skipped test to a log entry as that is the behavior of mstest.
if (e.Result.Outcome == ObjectModel.TestOutcome.Skipped)
{
this.HandleSkippedTest(e.Result);
}
var testType = this.converter.GetTestType(e.Result);
var executionId = this.converter.GetExecutionId(e.Result);
// Setting parent properties like parent result, parent test element, parent execution id.
var parentExecutionId = this.converter.GetParentExecutionId(e.Result);
var parentTestResult = GetTestResult(parentExecutionId);
var parentTestElement = (parentTestResult != null) ? GetTestElement(parentTestResult.Id.TestId) : null;
// Switch to flat test results in case any parent related information is missing.
if (parentTestResult == null || parentTestElement == null || parentExecutionId == Guid.Empty)
{
parentTestResult = null;
parentTestElement = null;
parentExecutionId = Guid.Empty;
}
// Create trx test element from rocksteady test case
var testElement = GetOrCreateTestElement(executionId, parentExecutionId, testType, parentTestElement, e.Result);
// Update test links. Test Links are updated in case of Ordered test.
UpdateTestLinks(testElement, parentTestElement);
// Convert the rocksteady result to trx test result
var testResult = CreateTestResult(executionId, parentExecutionId, testType, testElement, parentTestElement, parentTestResult, e.Result);
// Update test entries
UpdateTestEntries(executionId, parentExecutionId, testElement, parentTestElement);
// Set various counts (passed tests, failed tests, total tests)
this.totalTests++;
if (testResult.Outcome == TrxLoggerObjectModel.TestOutcome.Failed)
{
this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Failed;
this.failTests++;
}
else if (testResult.Outcome == TrxLoggerObjectModel.TestOutcome.Passed)
{
this.passTests++;
}
}
private TrxLoggerObjectModel.TestOutcome changeTestOutcomeIfNecessary(TrxLoggerObjectModel.TestOutcome outcome)
{
// If no tests discovered/executed and TreatNoTestsAsError was set to True
// We will return ResultSummary as Failed
// Note : we only send the value of TreatNoTestsAsError if it is "True"
if (totalTests == 0 && parametersDictionary.ContainsKey(ObjectModelConstants.TreatNoTestsAsError))
{
outcome = TrxLoggerObjectModel.TestOutcome.Failed;
}
return(outcome);
}
public void SaveCounters(XmlElement xml, string location, int[] counters)
{
xml = (XmlElement)LocationToXmlNode(xml, location);
for (int i = 0; i < counters.Length; i++)
{
TrxObjectModel.TestOutcome outcome = (TrxObjectModel.TestOutcome)i;
string attributeName = outcome.ToString();
attributeName = attributeName.Substring(0, 1).ToLowerInvariant() + attributeName.Substring(1);
xml.SetAttribute(attributeName, counters[i].ToString(CultureInfo.InvariantCulture));
}
}
/// <summary>
/// Create test result.
/// Currently trx supports only UnitTest and OrderedTest. All tests except OrderedTest all converted to unit test result.
/// </summary>
/// <param name="runId"></param>
/// <param name="testId"></param>
/// <param name="executionId"></param>
/// <param name="parentExecutionId"></param>
/// <param name="resultName"></param>
/// <param name="outcome"></param>
/// <param name="testType"></param>
/// <param name="testCategoryId"></param>
/// <returns>Trx test result</returns>
private TrxObjectModel.TestResult CreateTestResult(
Guid runId,
Guid testId,
Guid executionId,
Guid parentExecutionId,
string resultName,
TrxObjectModel.TestOutcome outcome,
TestType testType,
TestListCategoryId testCategoryId)
{
return(testType.Equals(Constants.OrderedTestType) ?
new TestResultAggregation(runId, testId, executionId, parentExecutionId, resultName, Environment.MachineName, outcome, testType, testCategoryId, trxFileHelper) :
new UnitTestResult(runId, testId, executionId, parentExecutionId, resultName, Environment.MachineName, outcome, testType, testCategoryId, trxFileHelper));
}
/// <summary>
/// Converts the rockSteady result to unit test result
/// </summary>
/// <param name="testId"></param>
/// <param name="executionId"></param>
/// <param name="parentExecutionId"></param>
/// <param name="testName"></param>
/// <param name="trxFileDirectory"></param>
/// <param name="testType"></param>
/// <param name="testCategoryId"></param>
/// <param name="testOutcome"></param>
/// <param name="testRun"></param>
/// <param name="rockSteadyTestResult"></param>
/// <returns>Trx test result object</returns>
public ITestResult ToTestResult(
Guid testId,
Guid executionId,
Guid parentExecutionId,
string testName,
string trxFileDirectory,
TestType testType,
TestListCategoryId testCategoryId,
TrxObjectModel.TestOutcome testOutcome,
TestRun testRun,
ObjectModel.TestResult rockSteadyTestResult)
{
var resultName = !string.IsNullOrEmpty(rockSteadyTestResult.DisplayName) ? rockSteadyTestResult.DisplayName : testName;
var testResult = CreateTestResult(testRun.Id, testId, executionId, parentExecutionId, resultName, testOutcome, testType, testCategoryId);
if (rockSteadyTestResult.ErrorMessage != null)
{
testResult.ErrorMessage = rockSteadyTestResult.ErrorMessage;
}
if (rockSteadyTestResult.ErrorStackTrace != null)
{
testResult.ErrorStackTrace = rockSteadyTestResult.ErrorStackTrace;
}
if (rockSteadyTestResult.EndTime != null)
{
testResult.EndTime = rockSteadyTestResult.EndTime.UtcDateTime;
}
if (rockSteadyTestResult.StartTime != null)
{
testResult.StartTime = rockSteadyTestResult.StartTime.UtcDateTime;
}
if (rockSteadyTestResult.Duration != null)
{
testResult.Duration = rockSteadyTestResult.Duration;
}
// Clear existing messages and store rocksteady result messages.
testResult.TextMessages = null;
UpdateResultMessages(testResult, rockSteadyTestResult);
// Save result attachments to target location.
UpdateTestResultAttachments(rockSteadyTestResult, testResult, testRun, trxFileDirectory, true);
return(testResult);
}
/// <summary>
/// Converts the rockSteady result to unit test result
/// </summary>
/// <param name="rockSteadyTestResult"> rock steady test result</param>
/// <param name="testElement"> testElement of that test</param>
/// <param name="testOutcome"> Test outcome </param>
/// <param name="testRun"> test run object </param>
/// <param name="trxFileDirectory"> TRX file directory</param>
/// <returns> TestResult object </returns>
internal static TrxObjectModel.UnitTestResult ToUnitTestResult(
ObjectModel.TestResult rockSteadyTestResult,
TrxObjectModel.UnitTestElement testElement,
TrxObjectModel.TestOutcome testOutcome,
TrxObjectModel.TestRun testRun,
string trxFileDirectory)
{
TrxObjectModel.UnitTestResult qtoolsResult = GetQToolsTestResultFromTestResult(rockSteadyTestResult, testElement, testOutcome, testRun);
// Clear exsting messages and store rocksteady result messages.
qtoolsResult.TextMessages = null;
UpdateResultMessages(qtoolsResult, rockSteadyTestResult);
// Save result attachments to target location.
UpdateTestResultAttachments(rockSteadyTestResult, qtoolsResult, testRun, trxFileDirectory, true);
return(qtoolsResult);
}
/// <summary>
/// Called when a test run is completed.
/// </summary>
/// <param name="sender">
/// The sender.
/// </param>
/// <param name="e">
/// Test run complete events arguments.
/// </param>
internal void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e)
{
// Create test run
// If abort occurs there is no call to TestResultHandler which results in testRun not created.
// This happens when some test aborts in the first batch of execution.
if (this.testRun == null)
{
CreateTestRun();
}
XmlPersistence helper = new XmlPersistence();
XmlTestStoreParameters parameters = XmlTestStoreParameters.GetParameters();
XmlElement rootElement = helper.CreateRootElement("TestRun");
// Save runId/username/creation time etc.
this.testRun.Finished = DateTime.UtcNow;
helper.SaveSingleFields(rootElement, this.testRun, parameters);
// Save test settings
helper.SaveObject(this.testRun.RunConfiguration, rootElement, "TestSettings", parameters);
// Save test results
helper.SaveIEnumerable(this.results.Values, rootElement, "Results", ".", null, parameters);
// Save test definitions
helper.SaveIEnumerable(this.testElements.Values, rootElement, "TestDefinitions", ".", null, parameters);
// Save test entries
helper.SaveIEnumerable(this.entries.Values, rootElement, "TestEntries", ".", "TestEntry", parameters);
// Save default categories
List <TestListCategory> categories = new List <TestListCategory>();
categories.Add(TestListCategory.UncategorizedResults);
categories.Add(TestListCategory.AllResults);
helper.SaveList(categories, rootElement, "TestLists", ".", "TestList", parameters);
// Save summary
if (this.testRunOutcome == TrxLoggerObjectModel.TestOutcome.Passed)
{
this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Completed;
}
testRunOutcome = changeTestOutcomeIfNecessary(testRunOutcome);
List <string> errorMessages = new List <string>();
List <CollectorDataEntry> collectorEntries = this.converter.ToCollectionEntries(e.AttachmentSets, this.testRun, this.testResultsDirPath);
IList <string> resultFiles = this.converter.ToResultFiles(e.AttachmentSets, this.testRun, this.testResultsDirPath, errorMessages);
if (errorMessages.Count > 0)
{
// Got some errors while attaching files, report them and set the outcome of testrun to be Error...
this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Error;
foreach (string msg in errorMessages)
{
RunInfo runMessage = new RunInfo(msg, null, Environment.MachineName, TrxLoggerObjectModel.TestOutcome.Error);
this.runLevelErrorsAndWarnings.Add(runMessage);
}
}
TestRunSummary runSummary = new TestRunSummary(
this.totalTests,
this.passTests + this.failTests,
this.passTests,
this.failTests,
this.testRunOutcome,
this.runLevelErrorsAndWarnings,
this.runLevelStdOut.ToString(),
resultFiles,
collectorEntries);
helper.SaveObject(runSummary, rootElement, "ResultSummary", parameters);
this.ReserveTrxFilePath();
this.PopulateTrxFile(this.trxFilePath, rootElement);
}
/// <summary>
/// Called when a test result is received.
/// </summary>
/// <param name="sender">
/// The sender.
/// </param>
/// <param name="e">
/// The eventArgs.
/// </param>
internal void TestResultHandler(object sender, ObjectModel.Logging.TestResultEventArgs e)
{
if (this.testRun == null)
{
Guid runId = Guid.NewGuid();
this.testRun = new TestRun(runId);
// We cannot rely on the StartTime for the first test result
// In case of parallel, first test result is the fastest test and not the one which started first.
// Setting Started to DateTime.Now in Intialize will make sure we include the startup cost, which was being ignored earlier.
// This is in parity with the way we set this.testRun.Finished
this.testRun.Started = this.testRunStartTime;
// Save default test settings
string runDeploymentRoot = FileHelper.ReplaceInvalidFileNameChars(this.testRun.Name);
TestRunConfiguration testrunConfig = new TestRunConfiguration("default");
testrunConfig.RunDeploymentRootDirectory = runDeploymentRoot;
this.testRun.RunConfiguration = testrunConfig;
}
// Convert skipped test to a log entry as that is the behaviour of mstest.
if (e.Result.Outcome == ObjectModel.TestOutcome.Skipped)
{
this.HandleSkippedTest(e.Result);
}
// Create MSTest test element from rocksteady test case
UnitTestElement testElement = Converter.ToUnitTestElement(e.Result);
// Conver the rocksteady result to MSTest result
TrxLoggerObjectModel.TestOutcome testOutcome = Converter.ToOutcome(e.Result.Outcome);
TrxLoggerObjectModel.UnitTestResult testResult = Converter.ToUnitTestResult(e.Result, testElement, testOutcome, this.testRun, this.testResultsDirPath);
// Set various counts (passtests, failed tests, total tests)
this.totalTests++;
if (testResult.Outcome == TrxLoggerObjectModel.TestOutcome.Failed)
{
this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Failed;
this.failTests++;
}
else if (testResult.Outcome == TrxLoggerObjectModel.TestOutcome.Passed)
{
this.passTests++;
}
// Add results to in-memory lists that are saved to the xml at completion.
this.results.Add(testResult);
if (!this.testElements.Contains(testElement))
{
this.testElements.Add(testElement);
}
// create a test entry
TestEntry te = new TestEntry(testElement.Id, TestListCategory.UncategorizedResults.Id);
te.ExecId = testElement.ExecutionId;
this.entries.Add(te);
}
