internal void IncrementCounter(TestVerdict testVerdict) { switch (testVerdict) { case TestVerdict.Pass: _passed++; break; case TestVerdict.Fail: _failed++; break; case TestVerdict.Error: _errored++; break; case TestVerdict.Inconclusive: _inconclusive++; break; case TestVerdict.DidNotExecute: _didnotexecute++; break; default: break; } }
public static string GetValue(TestVerdict tv) { switch (tv) { case TestVerdict.Pass: return("PASS"); case TestVerdict.XPass: return("XPASS"); case TestVerdict.Fail: return("FAIL"); case TestVerdict.XFail: return("XFAIL"); case TestVerdict.Unresolved: return("UNRESOLVED"); case TestVerdict.Untested: return("UNTESTED"); case TestVerdict.Unsupported: return("Unsupported"); default: return(string.Empty); } }
/// <summary> /// Constructor with name, verdict and comments for a runtime test check validation. /// </summary> /// <param name="description">Name of test check.</param> /// <param name="verdict">Verdict of test check.</param> /// <param name="comment">Optional comment of the test check.</param> internal TestCheck(string description, TestVerdict verdict, string comment) { _virtualUser = Thread.CurrentThread.Name; _description = description; _verdict = verdict; _comment = comment; _source = getSource(); }
public TestResult(string expectedResult, string actualResult, TestVerdict expectedVerdict , TestVerdict actualVerdict) { this.expectedResult = expectedResult; this.actualResult = actualResult; this.expectedVerdict = expectedVerdict; this.actualVerdict = actualVerdict; }
public virtual bool MayProceed(TestResult result) { TestVerdict vd = result.Verdict; if (vd == TestVerdict.Pass || vd == TestVerdict.XPass || vd == TestVerdict.XFail) { return(true); } return(false); }
public TestStep(TestStep originalTestStep, TestCase parentTestCase) : base(originalTestStep, parentTestCase) { this._expectedBehaviour = originalTestStep._expectedBehaviour; this._testType = originalTestStep._testType; this._expectedTestVerdict = originalTestStep._expectedTestVerdict; this._alwaysExecute = originalTestStep._alwaysExecute; this._iterations = originalTestStep._iterations; this._testAutomationDefinition = new TestAutomationDefinition(originalTestStep.TestAutomationDefinition); }
public TestStep(string title, TestCase parent) : base(title, parent) { _testAutomationDefinition = new Core.TestAutomationDefinition(); _testType = Core.TestType.Manual; _expectedTestVerdict = TestVerdict.Pass; _alwaysExecute = false; _status = Core.Status.Incomplete; _iterations = 1; TestScriptObject.OnPrepareToSave += TestScriptObject_OnPrepareToSave; }
private ResultStatus convertToTestRailStatus(TestVerdict testVerdict) { ResultStatus resultStatus = ResultStatus.Untested; if (testVerdict == TestVerdict.Pass) { resultStatus = ResultStatus.Passed; } else if (testVerdict == TestVerdict.Fail) { resultStatus = ResultStatus.Failed; } return(resultStatus); }
public TestClassBase() { _testVerdict = Core.TestVerdict.Pass; _testCheckCollection = new TestCheckCollection(); TestCheck.OnTestCheck += TestCheck_OnTestCheck; _testWarningCollection = new TestWarningCollection(); TestWarning.OnTestWarning += TestWarning_OnTestWarning; _testDataCollection = new TestDataCollection(); TestCheckFailuresOnly = true; MaxFailedTestChecks = _defaultMaxTestCheckFailures; }
public virtual TestVerdict ItemComposeVerdict(TestVerdict exp_verdict, TestVerdict verdict) { try { TestVerdict realVerdict = this.TestComposeVerdict(exp_verdict, verdict ); this.WriteLine("EXP ITEM RESULT\t: " + TestVerdictString.GetValue(exp_verdict)); this.WriteLine("ITEM RESULT\t: " + TestVerdictString.GetValue(verdict)); this.WriteLine("ITEM VERDICT\t: " + TestVerdictString.GetValue(realVerdict)); return realVerdict; } catch (System.IO.IOException) { return TestVerdict.Untested; } }
public virtual TestVerdict ItemComposeVerdict(TestVerdict exp_verdict, TestVerdict verdict) { try { TestVerdict realVerdict = this.TestComposeVerdict(exp_verdict, verdict ); this.WriteLine("EXP ITEM RESULT\t: " + TestVerdictString.GetValue(exp_verdict)); this.WriteLine("ITEM RESULT\t: " + TestVerdictString.GetValue(verdict)); this.WriteLine("ITEM VERDICT\t: " + TestVerdictString.GetValue(realVerdict)); return(realVerdict); } catch (System.IO.IOException) { return(TestVerdict.Untested); } }
public virtual TestVerdict TestResult(string exp_result, string result, TestVerdict exp_verdict, TestVerdict verdict) { try { TestVerdict realVerdict = this.TestComposeVerdict(exp_verdict, verdict ); this.WriteLine("EXPECTED RESULT\t: " + exp_result); this.WriteLine("RESULT\t\t: " + result); this.WriteLine("EXPECTED VERDICT: " + TestVerdictString.GetValue(exp_verdict)); this.WriteLine("TEST VERDICT\t: " + TestVerdictString.GetValue(verdict)); this.WriteLine("VERDICT\t\t: " + TestVerdictString.GetValue(realVerdict)); return(realVerdict); } catch (System.IO.IOException) { return(TestVerdict.Untested); } }
private TestVerdict updateTestVerdict(TestVerdict currentVerdict, TestVerdict processorVerdict) { switch (currentVerdict) { case TestVerdict.Unknown: currentVerdict = processorVerdict; break; case TestVerdict.DidNotExecute: currentVerdict = processorVerdict; break; case TestVerdict.Error: break; case TestVerdict.Fail: if (processorVerdict == Core.TestVerdict.Error) { currentVerdict = processorVerdict; } break; case TestVerdict.Inconclusive: if (processorVerdict == Core.TestVerdict.Fail || processorVerdict == Core.TestVerdict.Error) { currentVerdict = processorVerdict; } break; case TestVerdict.Pass: if (processorVerdict == Core.TestVerdict.Fail || processorVerdict == Core.TestVerdict.Error) { currentVerdict = processorVerdict; } break; default: break; } return(currentVerdict); }
private TestVerdict getProducerStepResult(TestStep testStep, TestCaseResult testCaseResult) { TestVerdict testVerdict = TestVerdict.Unknown; TestScriptResult dependencySupplier = null; if (testStep.DependsOn.HasValue && testStep.DependsOn != Guid.Empty) { foreach (var testStepResult in testCaseResult.TestStepResults) { if (testStepResult.ReferenceID == testStep.DependsOn) { dependencySupplier = testStepResult; testVerdict = dependencySupplier.TestVerdict; break; } } } return(testVerdict); }
public static string GetValue(TestVerdict tv) { switch (tv) { case TestVerdict.Pass: return "PASS"; case TestVerdict.XPass: return "XPASS"; case TestVerdict.Fail: return "FAIL"; case TestVerdict.XFail: return "XFAIL"; case TestVerdict.Unresolved: return "UNRESOLVED"; case TestVerdict.Untested: return "UNTESTED"; case TestVerdict.Unsupported: return "Unsupported"; default: return string.Empty; } }
private TestVerdict RunTest(TestCase tc) { lock (this) { framework.TestStart(tc.TestcaseId, tc.Component, tc.Function); if (tc.Title != null) { framework.TestTitle(tc.Title); } if (tc.Purpose != null) { framework.TestPurpose(tc.Purpose); } if (tc.Input != null) { framework.TestInput(tc.Input); } TestResult result = tc.Execute(framework); TestVerdict verdict = framework.TestResult(result.ExpectedResult, result.Result, result.ExpectedVerdict, result.Verdict); framework.TestFinish(); return(verdict); } }
public virtual TestVerdict TestResult(double exp_result, double result, TestVerdict exp_verdict, TestVerdict verdict) { return TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict); }
public virtual TestVerdict TestResult(float exp_result, float result, TestVerdict exp_verdict, TestVerdict verdict) { return(TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict)); }
public virtual TestVerdict TestResult(double exp_result, double result, TestVerdict exp_verdict, TestVerdict verdict) { return(TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict)); }
public virtual TestVerdict[] RunTests() { if (testCases.Count > 0) { TestVerdict[] results = new TestVerdict[testCases.Count]; TestCase tc; object obj; startTime = DateTime.Now; for (int i = 0; i < testCases.Count; i++) { obj = testCases[i]; if (obj is TestCase) { tc = (TestCase)testCases[i]; DateTime start = DateTime.Now; results[i] = this.RunTest(tc); DateTime stop = DateTime.Now; int milliseconds = (int)Math.Round((stop - start).TotalMilliseconds); testWriter.Write(GetDateString(start) + "\t| " + GetDateString(stop) + "\t| " + milliseconds); if (milliseconds > 100000) { testWriter.Write("\t| "); } else { testWriter.Write("\t\t| "); } string result = TestVerdictString.GetValue(results[i]); if (result.Length > 5) { testWriter.Write(result + "\t| "); } else { testWriter.Write(result + "\t\t| "); } testWriter.WriteLine(tc.TestcaseId); } else { if (obj is TestSuite.MileStone) { TestSuite.MileStone tp = (TestSuite.MileStone)obj; tp.SetTime(); } } } try { string result = this.GetReport(); if (result != null) { DateTime cal = DateTime.Now; StreamWriter writer = new StreamWriter("performance.log", true); string userName = Environment.UserName; string osName = Environment.OSVersion.Platform.ToString(); string osArch = "Running On Other"; Type t = Type.GetType("Mono.Runtime"); if (t != null) { osArch = "Running On Mono"; } string osVersion = Environment.OSVersion.VersionString; writer.WriteLine(starLine); writer.WriteLine("TEST TIME: " + GetDateString(startTime) + " - " + GetDateString(cal)); writer.WriteLine("USER : "******"PLATFORM : " + osName + " " + osVersion + " " + osArch); writer.WriteLine(result); writer.Flush(); writer.Close(); } } catch (System.IO.IOException) { } return results; } return null; }
public virtual TestVerdict TestResult(float exp_result, float result, TestVerdict exp_verdict, TestVerdict verdict) { return TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict); }
public TestStepResult(TestVerdict testVerdict) : this(testVerdict, null) { }
public TestStepResult(TestVerdict testVerdict, string testMesssage) : base() { _testVerdict = testVerdict; _testMessage = testMesssage; }
public virtual TestVerdict TestResult(long exp_result, long result, TestVerdict exp_verdict, TestVerdict verdict) { return TestResult(exp_result.ToString(), result.ToString(), exp_verdict, verdict); }
internal void SetTestVerdict(TestVerdict testVerdict) { _testVerdict = testVerdict; }
private TestVerdict TestComposeVerdict(TestVerdict expVerdict, TestVerdict verdict) { TestVerdict result; if (expVerdict.Equals(TestVerdict.Pass)) { if ((verdict.Equals(TestVerdict.Pass)) || (verdict.Equals(TestVerdict.Fail))) { result = verdict; } else { result = TestVerdict.Unresolved; } } else { if (expVerdict.Equals(TestVerdict.Fail)) { if (verdict.Equals(TestVerdict.Pass)) { result = TestVerdict.XPass; } else { if (verdict.Equals(TestVerdict.Fail)) { result = TestVerdict.XFail; } else { result = TestVerdict.Unresolved; } } } else { if (expVerdict.Equals(TestVerdict.Unresolved)) { result = TestVerdict.Unresolved; } else { if (expVerdict.Equals(TestVerdict.Untested)) { if (!(verdict.Equals(TestVerdict.Untested))) { result = TestVerdict.Unresolved; } else { result = TestVerdict.Untested; } } else { if (expVerdict.Equals(TestVerdict.Unsupported)) { if (!(verdict.Equals(TestVerdict.Unsupported))) { result = TestVerdict.Unresolved; } else { result = TestVerdict.Unsupported; } } else { result = TestVerdict.Unresolved; } } } } } return result; }
private TestVerdict TestComposeVerdict(TestVerdict expVerdict, TestVerdict verdict) { TestVerdict result; if (expVerdict.Equals(TestVerdict.Pass)) { if ((verdict.Equals(TestVerdict.Pass)) || (verdict.Equals(TestVerdict.Fail))) { result = verdict; } else { result = TestVerdict.Unresolved; } } else { if (expVerdict.Equals(TestVerdict.Fail)) { if (verdict.Equals(TestVerdict.Pass)) { result = TestVerdict.XPass; } else { if (verdict.Equals(TestVerdict.Fail)) { result = TestVerdict.XFail; } else { result = TestVerdict.Unresolved; } } } else { if (expVerdict.Equals(TestVerdict.Unresolved)) { result = TestVerdict.Unresolved; } else { if (expVerdict.Equals(TestVerdict.Untested)) { if (!(verdict.Equals(TestVerdict.Untested))) { result = TestVerdict.Unresolved; } else { result = TestVerdict.Untested; } } else { if (expVerdict.Equals(TestVerdict.Unsupported)) { if (!(verdict.Equals(TestVerdict.Unsupported))) { result = TestVerdict.Unresolved; } else { result = TestVerdict.Unsupported; } } else { result = TestVerdict.Unresolved; } } } } } return(result); }
private TestVerdict determineVerdict(TestVerdict expected, TestVerdict actual) { return(expected.Equals(actual) ? TestVerdict.Pass : TestVerdict.Fail); }
public virtual TestVerdict[] RunTests() { if (testCases.Count > 0) { TestVerdict[] results = new TestVerdict[testCases.Count]; TestCase tc; object obj; startTime = DateTime.Now; for (int i = 0; i < testCases.Count; i++) { obj = testCases[i]; if (obj is TestCase) { tc = (TestCase)testCases[i]; DateTime start = DateTime.Now; results[i] = this.RunTest(tc); DateTime stop = DateTime.Now; int milliseconds = (int)Math.Round((stop - start).TotalMilliseconds); testWriter.Write(GetDateString(start) + "\t| " + GetDateString(stop) + "\t| " + milliseconds); if (milliseconds > 100000) { testWriter.Write("\t| "); } else { testWriter.Write("\t\t| "); } string result = TestVerdictString.GetValue(results[i]); if (result.Length > 5) { testWriter.Write(result + "\t| "); } else { testWriter.Write(result + "\t\t| "); } testWriter.WriteLine(tc.TestcaseId); } else { if (obj is TestSuite.MileStone) { TestSuite.MileStone tp = (TestSuite.MileStone)obj; tp.SetTime(); } } } try { string result = this.GetReport(); if (result != null) { DateTime cal = DateTime.Now; StreamWriter writer = new StreamWriter("performance.log", true); string userName = Environment.UserName; string osName = Environment.OSVersion.Platform.ToString(); string osArch = "Running On Other"; Type t = Type.GetType("Mono.Runtime"); if (t != null) { osArch = "Running On Mono"; } string osVersion = Environment.OSVersion.VersionString; writer.WriteLine(starLine); writer.WriteLine("TEST TIME: " + GetDateString(startTime) + " - " + GetDateString(cal)); writer.WriteLine("USER : "******"PLATFORM : " + osName + " " + osVersion + " " + osArch); writer.WriteLine(result); writer.Flush(); writer.Close(); } } catch (System.IO.IOException) { } return(results); } return(null); }
public virtual TestVerdict TestResult(string exp_result, string result, TestVerdict exp_verdict, TestVerdict verdict) { try { TestVerdict realVerdict = this.TestComposeVerdict(exp_verdict, verdict ); this.WriteLine("EXPECTED RESULT\t: " + exp_result); this.WriteLine("RESULT\t\t: " + result); this.WriteLine("EXPECTED VERDICT: " + TestVerdictString.GetValue(exp_verdict)); this.WriteLine("TEST VERDICT\t: " + TestVerdictString.GetValue(verdict)); this.WriteLine("VERDICT\t\t: " + TestVerdictString.GetValue(realVerdict)); return realVerdict; } catch (System.IO.IOException) { return TestVerdict.Untested; } }