internal TestResultEntity ProcessRegressionTestAsserts(TestResultEntity testResult)
{
// See if there's any regression test info defined
var expResult = metadata.TestCase.ExpectedRegressionTestResult;
if (expResult != null)
{
XElement xtestResults = testResult.DataElement;
TestResultType[] expResultTypes = expResult.ResultTypes;
if (expResultTypes != null && expResultTypes.Length != 0)
{
// Need to hold the temporary value because the xtestResults.SetAttributeValue
// will modify this and the error message won't make sense.
var resultType = testResult.ResultType;
if (expResultTypes.Contains(resultType))
{
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.Passed);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, String.Format("Expected regression result type assertion passed."));
}
else
{
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.RegressionAssertFailure);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, String.Format("Expected result type of any of the following [{0}] but got [{1}]."
, String.Join(", ", expResult.ResultTypes), resultType));
}
}
}
return(testResult);
}
private TreeNode GetBucketNode(TreeNode parentNode, TestResultEntity result)
{
string bucketName = GetBucketName(result);
if (bucketName == null)
{
return(parentNode);
}
TreeNode bucketNode = parentNode.Nodes
.Cast <TreeNode>()
.Where(n => n.Text == bucketName)
.FirstOrDefault();
if (bucketNode == null)
{
bucketNode = new TreeNode(bucketName)
{
ForeColor = parentNode.ForeColor
};
parentNode.Nodes.Add(bucketNode);
}
return(bucketNode);
}
private static TestResultEntity RunTestCaseGetResult(string testCaseFilePath)
{
XElement xtestResults;
var engine = new TestCaseRunnerEngine();
if (!engine.LoadTestCase(testCaseFilePath))
{
// Report these errors as is since we can't load the test case
xtestResults = TestResultUtil.CreateErrorXTestResult(engine.Error, engine.ErrorEx);
return(Model.Instance.EntityBuilder.EnsureEntityBound <TestResultEntity>(xtestResults));
}
xtestResults = engine.RunTestCase();
if (xtestResults == null)
{
xtestResults = TestResultUtil.CreateErrorXTestResult("RunTestCaseGetResult returned a null xml element.");
}
TestResultEntity testResult = Model.Instance.EntityBuilder.EnsureEntityBound <TestResultEntity>(xtestResults);
testResult = engine.PreProcessResults(testResult);
testResult = engine.ProcessExpectedResultAssertions(testResult);
// Lastly, do the assertions for regression tests if the TestCase defines it.
testResult = engine.ProcessRegressionTestAsserts(testResult);
return(testResult);
}
private static int RunTestCase(string testCaseFilePath)
{
TestResultEntity result = RunTestCaseGetResult(testCaseFilePath);
// Print result to std err no matter what
Console.Error.WriteLine();
Console.Error.WriteLine("MCUT Test Case Result: {0}", result.ResultType);
Console.Error.WriteLine(result.Message);
// Write the result to the output file
try
{
using (XmlWriter xwriter = XmlWriter.Create(resultFilePath, new XmlWriterSettings()
{
Indent = true
}))
{
result.DataElement.WriteTo(xwriter);
}
return(result.ExitCode);
}
catch (IOException ex)
{
Console.Error.WriteLine("Error writing to result xml file: " + ex.ToString());
return((int)TestResultType.Error);
}
catch (Exception ex)
{
Console.Error.WriteLine("Error with test result xml: " + ex.ToString());
return((int)TestResultType.Error);
}
}
//internal static IEnumerable<ChessResultEntity> GetChessResults(this TestRunEntity source)
//{
// return source.DataElement
// .Elements(XTestResultNames.TestResult)
// .Elements(XChessNames.ChessResults)
// .Elements(XChessNames.Result)
// .SelectEntities<ChessResultEntity>()
// ;
//}
internal static IEnumerable <ChessResultEntity> GetChessResults(this TestResultEntity source)
{
return(source.DataElement
.Elements(XChessNames.ChessResults)
.Elements(XChessNames.Result)
.SelectEntities <ChessResultEntity>()
);
}
internal TestResultTreeNode(TestResultEntity result)
{
this.result = result;
// Bind here:
Text = String.Format("({0}) {1}", result.OwningTestRun.TaskID, result.OwningTestRun.DisplayName);
//ForeColor = result.ResultType.GetDisplayColor();
ToolTipText = String.Format("{0}", result.Message);
}
private void RemoveResultNode(TestResultEntity result)
{
TestResultTreeNode node = result.DataElement.Annotation <TestResultTreeNode>();
if (node != null && node.TreeView != null)
{
System.Diagnostics.Debug.Assert(node.TreeView == treeView1);
_treeNeedsRebuilding = node.Parent.Nodes.Count == 1; // Only need to refresh if by removng us, it'll mean the parent is empty
node.Remove();
result.DataElement.RemoveAnnotations <TestResultTreeNode>();
}
}
public TestResultEntity PreProcessResults(TestCaseMetadata metadata, TestResultEntity testResult)
{
var expChessResult = metadata.TestCase.EntityOfType <ExpectedChessTestResultEntity>();
if (expChessResult != null)
{
XElement xtestResults = testResult.DataElement;
try
{
if (expChessResult.ExitCode.HasValue)
{
Assert.AreEqual(expChessResult.ExitCode, testResult.ChessExitCode, "Expected Result: ChessExitCode");
}
XElement xfinalStats = testResult.DataElement
.Elements(XChessNames.ChessResults)
.Elements(XChessNames.FinalStats)
.SingleOrDefault();
AssertFinalStatValue(xfinalStats, expChessResult.SchedulesRan, XChessNames.ASchedulesRan, "Schedules Ran");
AssertFinalStatValue(xfinalStats, expChessResult.LastThreadCount, XChessNames.ALastThreadCount, "Last Thread Count");
AssertFinalStatValue(xfinalStats, expChessResult.LastExecSteps, XChessNames.ALastExecSteps, "LastExecSteps");
AssertFinalStatValue(xfinalStats, expChessResult.LastHBExecSteps, XChessNames.ALastHBExecSteps, "Last HB Exec Steps");
// If we got here, than all the assertions passed
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.Passed);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, "Passed expected MChess test result assertions.");
}
catch (AssertFailedException ex)
{
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.ResultAssertFailure);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, ex.Message);
}
catch (AssertInconclusiveException ex)
{
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.ResultInconclusive);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, ex.Message);
}
catch (Exception ex)
{
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.Error);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, "Expected Chess Result Assertion: " + ex.Message);
}
}
return(testResult);
}
private void AddResultNode(TestResultEntity result)
{
TestResultTreeNode node = result.DataElement.Annotation <TestResultTreeNode>();
System.Diagnostics.Debug.Assert(node == null);
node = new TestResultTreeNode(result);
var resultTypeNode = _resultTypeNodes[result.ResultType];
result.DataElement.AddAnnotation(node);
// Allow for sub-grouping if the result type wants it
TreeNode bucketNode = GetBucketNode(resultTypeNode, result);
bucketNode.Nodes.Add(node);
_treeNeedsRebuilding = true;
}
internal TestResultEntity ProcessExpectedResultAssertions(TestResultEntity testResult)
{
ExpectedTestResultEntity expResult = metadata.TestCase.ExpectedTestResult;
if (expResult != null)
{
XElement xtestResults = testResult.DataElement;
try
{
// First, do the common result assertions
// If the expResult is specified, the ResultType is required by the xml schema
Assert.AreEqual(expResult.ResultType, testResult.ResultType, "Result.ResultType");
if (expResult.Message != null)
{
Assert.AreEqual(expResult.Message, testResult.Message, "Result.Message");
}
// If we got here, than all the assertions passed
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.Passed);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, "Passed expected result assertions.");
}
catch (AssertFailedException ex)
{
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.ResultAssertFailure);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, ex.Message);
}
catch (AssertInconclusiveException ex)
{
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.ResultInconclusive);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, ex.Message);
}
catch (Exception ex)
{
xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.Error);
xtestResults.SetElementValue(XTestResultNames.ResultMessage, "Expected Result Assertion: " + ex.Message);
}
}
return(testResult);
}
private string GetBucketName(TestResultEntity result)
{
switch (result.ResultType)
{
case TestResultType.Passed:
case TestResultType.DataRace:
case TestResultType.Deadlock:
case TestResultType.Livelock:
return(null); // Do not bucket
case TestResultType.Error:
case TestResultType.Exception:
var exError = result.Error as ExceptionErrorEntity;
if (exError != null && exError.IsAggregateException && exError.InnerErrorsCount == 1)
{
// Get the inner error (should be an exception error)
exError = exError.EntityOfType <ExceptionErrorEntity>();
if (exError != null)
{
return(exError.Message);
}
}
// Otherwise, just use the default of the result's message.
return(result.Message);
case TestResultType.AssertFailure:
case TestResultType.ResultAssertFailure:
return(result.Message);
case TestResultType.Inconclusive:
case TestResultType.ResultInconclusive:
return(result.Message);
default:
throw new NotImplementedException("TestResultType is not implemented: " + result.ResultType);
}
}
internal TestResultEntity PreProcessResults(TestResultEntity testResult)
{
return(runner.PreProcessResults(metadata, testResult));
}
public TestResultEntity PreProcessResults(TestCaseMetadata metadata, TestResultEntity testResult)
{
// Nothing to do
return(testResult);
}
/// <summary>
/// Gets the ChessResults that are non-informational only. i.e. not warnings or messages.
/// </summary>
/// <param name="source"></param>
/// <returns></returns>
internal static IEnumerable <ChessResultEntity> GetNonInformationalChessResults(this TestResultEntity source)
{
return(source.DataElement
.Elements(XChessNames.ChessResults)
.Elements(XChessNames.Result)
.SelectEntities <ChessResultEntity>()
.Where(cr => cr.ResultType != MChessResultType.Notification && cr.ResultType != MChessResultType.Warning)
);
}
