internal void IncrementCounter(TestVerdict testVerdict)
{
switch (testVerdict)
{
case TestVerdict.Pass:
_passed++;
break;
case TestVerdict.Fail:
_failed++;
break;
case TestVerdict.Error:
_errored++;
break;
case TestVerdict.Inconclusive:
_inconclusive++;
break;
case TestVerdict.DidNotExecute:
_didnotexecute++;
break;
default:
break;
}
}
public static string GetValue(TestVerdict tv)
{
switch (tv)
{
case TestVerdict.Pass:
return("PASS");
case TestVerdict.XPass:
return("XPASS");
case TestVerdict.Fail:
return("FAIL");
case TestVerdict.XFail:
return("XFAIL");
case TestVerdict.Unresolved:
return("UNRESOLVED");
case TestVerdict.Untested:
return("UNTESTED");
case TestVerdict.Unsupported:
return("Unsupported");
default:
return(string.Empty);
}
}
/// <summary>
/// Constructor with name, verdict and comments for a runtime test check validation.
/// </summary>
/// <param name="description">Name of test check.</param>
/// <param name="verdict">Verdict of test check.</param>
/// <param name="comment">Optional comment of the test check.</param>
internal TestCheck(string description, TestVerdict verdict, string comment)
{
_virtualUser = Thread.CurrentThread.Name;
_description = description;
_verdict = verdict;
_comment = comment;
_source = getSource();
}
public TestResult(string expectedResult, string actualResult, TestVerdict expectedVerdict
, TestVerdict actualVerdict)
{
this.expectedResult = expectedResult;
this.actualResult = actualResult;
this.expectedVerdict = expectedVerdict;
this.actualVerdict = actualVerdict;
}
public TestResult(string expectedResult, string actualResult, TestVerdict expectedVerdict
, TestVerdict actualVerdict)
{
this.expectedResult = expectedResult;
this.actualResult = actualResult;
this.expectedVerdict = expectedVerdict;
this.actualVerdict = actualVerdict;
}
public virtual bool MayProceed(TestResult result)
{
TestVerdict vd = result.Verdict;
if (vd == TestVerdict.Pass || vd == TestVerdict.XPass || vd == TestVerdict.XFail)
{
return(true);
}
return(false);
}
public TestStep(TestStep originalTestStep, TestCase parentTestCase)
: base(originalTestStep, parentTestCase)
{
this._expectedBehaviour = originalTestStep._expectedBehaviour;
this._testType = originalTestStep._testType;
this._expectedTestVerdict = originalTestStep._expectedTestVerdict;
this._alwaysExecute = originalTestStep._alwaysExecute;
this._iterations = originalTestStep._iterations;
this._testAutomationDefinition = new TestAutomationDefinition(originalTestStep.TestAutomationDefinition);
}
public TestStep(string title, TestCase parent)
: base(title, parent)
{
_testAutomationDefinition = new Core.TestAutomationDefinition();
_testType = Core.TestType.Manual;
_expectedTestVerdict = TestVerdict.Pass;
_alwaysExecute = false;
_status = Core.Status.Incomplete;
_iterations = 1;
TestScriptObject.OnPrepareToSave += TestScriptObject_OnPrepareToSave;
}
private ResultStatus convertToTestRailStatus(TestVerdict testVerdict)
{
ResultStatus resultStatus = ResultStatus.Untested;
if (testVerdict == TestVerdict.Pass)
{
resultStatus = ResultStatus.Passed;
}
else if (testVerdict == TestVerdict.Fail)
{
resultStatus = ResultStatus.Failed;
}
return(resultStatus);
}
public TestClassBase()
{
_testVerdict = Core.TestVerdict.Pass;
_testCheckCollection = new TestCheckCollection();
TestCheck.OnTestCheck += TestCheck_OnTestCheck;
_testWarningCollection = new TestWarningCollection();
TestWarning.OnTestWarning += TestWarning_OnTestWarning;
_testDataCollection = new TestDataCollection();
TestCheckFailuresOnly = true;
MaxFailedTestChecks = _defaultMaxTestCheckFailures;
}
public virtual TestVerdict ItemComposeVerdict(TestVerdict exp_verdict,
TestVerdict verdict)
{
try
{
TestVerdict realVerdict = this.TestComposeVerdict(exp_verdict, verdict
);
this.WriteLine("EXP ITEM RESULT\t: " + TestVerdictString.GetValue(exp_verdict));
this.WriteLine("ITEM RESULT\t: " + TestVerdictString.GetValue(verdict));
this.WriteLine("ITEM VERDICT\t: " + TestVerdictString.GetValue(realVerdict));
return realVerdict;
}
catch (System.IO.IOException)
{
return TestVerdict.Untested;
}
}
public virtual TestVerdict ItemComposeVerdict(TestVerdict exp_verdict,
TestVerdict verdict)
{
try
{
TestVerdict realVerdict = this.TestComposeVerdict(exp_verdict, verdict
);
this.WriteLine("EXP ITEM RESULT\t: " + TestVerdictString.GetValue(exp_verdict));
this.WriteLine("ITEM RESULT\t: " + TestVerdictString.GetValue(verdict));
this.WriteLine("ITEM VERDICT\t: " + TestVerdictString.GetValue(realVerdict));
return(realVerdict);
}
catch (System.IO.IOException)
{
return(TestVerdict.Untested);
}
}
public virtual TestVerdict TestResult(string exp_result, string result,
TestVerdict exp_verdict, TestVerdict verdict)
{
try
{
TestVerdict realVerdict = this.TestComposeVerdict(exp_verdict, verdict
);
this.WriteLine("EXPECTED RESULT\t: " + exp_result);
this.WriteLine("RESULT\t\t: " + result);
this.WriteLine("EXPECTED VERDICT: " + TestVerdictString.GetValue(exp_verdict));
this.WriteLine("TEST VERDICT\t: " + TestVerdictString.GetValue(verdict));
this.WriteLine("VERDICT\t\t: " + TestVerdictString.GetValue(realVerdict));
return(realVerdict);
}
catch (System.IO.IOException)
{
return(TestVerdict.Untested);
}
}
private TestVerdict updateTestVerdict(TestVerdict currentVerdict, TestVerdict processorVerdict)
{
switch (currentVerdict)
{
case TestVerdict.Unknown:
currentVerdict = processorVerdict;
break;
case TestVerdict.DidNotExecute:
currentVerdict = processorVerdict;
break;
case TestVerdict.Error:
break;
case TestVerdict.Fail:
if (processorVerdict == Core.TestVerdict.Error)
{
currentVerdict = processorVerdict;
}
break;
case TestVerdict.Inconclusive:
if (processorVerdict == Core.TestVerdict.Fail || processorVerdict == Core.TestVerdict.Error)
{
currentVerdict = processorVerdict;
}
break;
case TestVerdict.Pass:
if (processorVerdict == Core.TestVerdict.Fail || processorVerdict == Core.TestVerdict.Error)
{
currentVerdict = processorVerdict;
}
break;
default:
break;
}
return(currentVerdict);
}
private TestVerdict getProducerStepResult(TestStep testStep, TestCaseResult testCaseResult)
{
TestVerdict testVerdict = TestVerdict.Unknown;
TestScriptResult dependencySupplier = null;
if (testStep.DependsOn.HasValue && testStep.DependsOn != Guid.Empty)
{
foreach (var testStepResult in testCaseResult.TestStepResults)
{
if (testStepResult.ReferenceID == testStep.DependsOn)
{
dependencySupplier = testStepResult;
testVerdict = dependencySupplier.TestVerdict;
break;
}
}
}
return(testVerdict);
}
public static string GetValue(TestVerdict tv)
{
switch (tv)
{
case TestVerdict.Pass:
return "PASS";
case TestVerdict.XPass:
return "XPASS";
case TestVerdict.Fail:
return "FAIL";
case TestVerdict.XFail:
return "XFAIL";
case TestVerdict.Unresolved:
return "UNRESOLVED";
case TestVerdict.Untested:
return "UNTESTED";
case TestVerdict.Unsupported:
return "Unsupported";
default:
return string.Empty;
}
}
private TestVerdict RunTest(TestCase tc)
{
lock (this)
{
framework.TestStart(tc.TestcaseId, tc.Component, tc.Function);
if (tc.Title != null)
{
framework.TestTitle(tc.Title);
}
if (tc.Purpose != null)
{
framework.TestPurpose(tc.Purpose);
}
if (tc.Input != null)
{
framework.TestInput(tc.Input);
}
TestResult result = tc.Execute(framework);
TestVerdict verdict = framework.TestResult(result.ExpectedResult,
result.Result, result.ExpectedVerdict, result.Verdict);
framework.TestFinish();
return(verdict);
}
}
public virtual TestVerdict TestResult(double exp_result, double result,
TestVerdict exp_verdict, TestVerdict verdict)
{
return TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict);
}
public virtual TestVerdict TestResult(float exp_result, float result,
TestVerdict exp_verdict, TestVerdict verdict)
{
return(TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict));
}
public virtual TestVerdict TestResult(double exp_result, double result,
TestVerdict exp_verdict, TestVerdict verdict)
{
return(TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict));
}
public virtual TestVerdict[] RunTests()
{
if (testCases.Count > 0)
{
TestVerdict[] results = new TestVerdict[testCases.Count];
TestCase tc;
object obj;
startTime = DateTime.Now;
for (int i = 0; i < testCases.Count; i++)
{
obj = testCases[i];
if (obj is TestCase)
{
tc = (TestCase)testCases[i];
DateTime start = DateTime.Now;
results[i] = this.RunTest(tc);
DateTime stop = DateTime.Now;
int milliseconds = (int)Math.Round((stop - start).TotalMilliseconds);
testWriter.Write(GetDateString(start) + "\t| " +
GetDateString(stop) + "\t| " + milliseconds);
if (milliseconds > 100000)
{
testWriter.Write("\t| ");
}
else
{
testWriter.Write("\t\t| ");
}
string result = TestVerdictString.GetValue(results[i]);
if (result.Length > 5)
{
testWriter.Write(result + "\t| ");
}
else
{
testWriter.Write(result + "\t\t| ");
}
testWriter.WriteLine(tc.TestcaseId);
}
else
{
if (obj is TestSuite.MileStone)
{
TestSuite.MileStone tp = (TestSuite.MileStone)obj;
tp.SetTime();
}
}
}
try
{
string result = this.GetReport();
if (result != null)
{
DateTime cal = DateTime.Now;
StreamWriter writer = new StreamWriter("performance.log", true);
string userName = Environment.UserName;
string osName = Environment.OSVersion.Platform.ToString();
string osArch = "Running On Other";
Type t = Type.GetType("Mono.Runtime");
if (t != null)
{
osArch = "Running On Mono";
}
string osVersion = Environment.OSVersion.VersionString;
writer.WriteLine(starLine);
writer.WriteLine("TEST TIME: " + GetDateString(startTime) + " - " + GetDateString(cal));
writer.WriteLine("USER : "******"PLATFORM : " + osName + " " + osVersion + " " + osArch);
writer.WriteLine(result);
writer.Flush();
writer.Close();
}
}
catch (System.IO.IOException)
{
}
return results;
}
return null;
}
public virtual TestVerdict TestResult(float exp_result, float result,
TestVerdict exp_verdict, TestVerdict verdict)
{
return TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict);
}
public TestStepResult(TestVerdict testVerdict)
: this(testVerdict, null)
{
}
public TestStepResult(TestVerdict testVerdict, string testMesssage)
: base()
{
_testVerdict = testVerdict;
_testMessage = testMesssage;
}
public virtual TestVerdict TestResult(long exp_result, long result,
TestVerdict exp_verdict, TestVerdict verdict)
{
return TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict);
}
internal void SetTestVerdict(TestVerdict testVerdict)
{
_testVerdict = testVerdict;
}
private TestVerdict TestComposeVerdict(TestVerdict
expVerdict, TestVerdict verdict)
{
TestVerdict result;
if (expVerdict.Equals(TestVerdict.Pass))
{
if ((verdict.Equals(TestVerdict.Pass)) || (verdict.Equals(TestVerdict.Fail)))
{
result = verdict;
}
else
{
result = TestVerdict.Unresolved;
}
}
else
{
if (expVerdict.Equals(TestVerdict.Fail))
{
if (verdict.Equals(TestVerdict.Pass))
{
result = TestVerdict.XPass;
}
else
{
if (verdict.Equals(TestVerdict.Fail))
{
result = TestVerdict.XFail;
}
else
{
result = TestVerdict.Unresolved;
}
}
}
else
{
if (expVerdict.Equals(TestVerdict.Unresolved))
{
result = TestVerdict.Unresolved;
}
else
{
if (expVerdict.Equals(TestVerdict.Untested))
{
if (!(verdict.Equals(TestVerdict.Untested)))
{
result = TestVerdict.Unresolved;
}
else
{
result = TestVerdict.Untested;
}
}
else
{
if (expVerdict.Equals(TestVerdict.Unsupported))
{
if (!(verdict.Equals(TestVerdict.Unsupported)))
{
result = TestVerdict.Unresolved;
}
else
{
result = TestVerdict.Unsupported;
}
}
else
{
result = TestVerdict.Unresolved;
}
}
}
}
}
return result;
}
private TestVerdict TestComposeVerdict(TestVerdict
expVerdict, TestVerdict verdict)
{
TestVerdict result;
if (expVerdict.Equals(TestVerdict.Pass))
{
if ((verdict.Equals(TestVerdict.Pass)) || (verdict.Equals(TestVerdict.Fail)))
{
result = verdict;
}
else
{
result = TestVerdict.Unresolved;
}
}
else
{
if (expVerdict.Equals(TestVerdict.Fail))
{
if (verdict.Equals(TestVerdict.Pass))
{
result = TestVerdict.XPass;
}
else
{
if (verdict.Equals(TestVerdict.Fail))
{
result = TestVerdict.XFail;
}
else
{
result = TestVerdict.Unresolved;
}
}
}
else
{
if (expVerdict.Equals(TestVerdict.Unresolved))
{
result = TestVerdict.Unresolved;
}
else
{
if (expVerdict.Equals(TestVerdict.Untested))
{
if (!(verdict.Equals(TestVerdict.Untested)))
{
result = TestVerdict.Unresolved;
}
else
{
result = TestVerdict.Untested;
}
}
else
{
if (expVerdict.Equals(TestVerdict.Unsupported))
{
if (!(verdict.Equals(TestVerdict.Unsupported)))
{
result = TestVerdict.Unresolved;
}
else
{
result = TestVerdict.Unsupported;
}
}
else
{
result = TestVerdict.Unresolved;
}
}
}
}
}
return(result);
}
private TestVerdict determineVerdict(TestVerdict expected, TestVerdict actual)
{
return(expected.Equals(actual) ? TestVerdict.Pass : TestVerdict.Fail);
}
public virtual TestVerdict[] RunTests()
{
if (testCases.Count > 0)
{
TestVerdict[] results = new TestVerdict[testCases.Count];
TestCase tc;
object obj;
startTime = DateTime.Now;
for (int i = 0; i < testCases.Count; i++)
{
obj = testCases[i];
if (obj is TestCase)
{
tc = (TestCase)testCases[i];
DateTime start = DateTime.Now;
results[i] = this.RunTest(tc);
DateTime stop = DateTime.Now;
int milliseconds = (int)Math.Round((stop - start).TotalMilliseconds);
testWriter.Write(GetDateString(start) + "\t| " +
GetDateString(stop) + "\t| " + milliseconds);
if (milliseconds > 100000)
{
testWriter.Write("\t| ");
}
else
{
testWriter.Write("\t\t| ");
}
string result = TestVerdictString.GetValue(results[i]);
if (result.Length > 5)
{
testWriter.Write(result + "\t| ");
}
else
{
testWriter.Write(result + "\t\t| ");
}
testWriter.WriteLine(tc.TestcaseId);
}
else
{
if (obj is TestSuite.MileStone)
{
TestSuite.MileStone tp = (TestSuite.MileStone)obj;
tp.SetTime();
}
}
}
try
{
string result = this.GetReport();
if (result != null)
{
DateTime cal = DateTime.Now;
StreamWriter writer = new StreamWriter("performance.log", true);
string userName = Environment.UserName;
string osName = Environment.OSVersion.Platform.ToString();
string osArch = "Running On Other";
Type t = Type.GetType("Mono.Runtime");
if (t != null)
{
osArch = "Running On Mono";
}
string osVersion = Environment.OSVersion.VersionString;
writer.WriteLine(starLine);
writer.WriteLine("TEST TIME: " + GetDateString(startTime) + " - " + GetDateString(cal));
writer.WriteLine("USER : "******"PLATFORM : " + osName + " " + osVersion + " " + osArch);
writer.WriteLine(result);
writer.Flush();
writer.Close();
}
}
catch (System.IO.IOException)
{
}
return(results);
}
return(null);
}
public virtual TestVerdict TestResult(string exp_result, string result,
TestVerdict exp_verdict, TestVerdict verdict)
{
try
{
TestVerdict realVerdict = this.TestComposeVerdict(exp_verdict, verdict
);
this.WriteLine("EXPECTED RESULT\t: " + exp_result);
this.WriteLine("RESULT\t\t: " + result);
this.WriteLine("EXPECTED VERDICT: " + TestVerdictString.GetValue(exp_verdict));
this.WriteLine("TEST VERDICT\t: " + TestVerdictString.GetValue(verdict));
this.WriteLine("VERDICT\t\t: " + TestVerdictString.GetValue(realVerdict));
return realVerdict;
}
catch (System.IO.IOException)
{
return TestVerdict.Untested;
}
}
