/// <summary>
/// Determines execution status.
/// </summary>
/// <param name="testStep">Current teststep under determination</param>
/// <param name="testCaseResult">Current test case result (used to resolved dependencies).</param>
/// <param name="explanation">Explanation of determination if approprieate.</param>
/// <returns>True to subsequently execute, false if not.</returns>
/// <remarks>
/// If all previous test step's passed, always execute unless current test step's execution is dependent on previous test step's passage.
/// However if a previous step failed but the test step is marked to always execute do so. If the parent
/// test case's OnTestStepFailure is set to "Continue", execute all active child test steps.
/// </remarks>
private bool determineExecutionStatus(TestStep testStep, TestCaseResult testCaseResult, out string explanation)
{
var executeTestStep = true;
explanation = string.Empty;
// Only execute active test steps.
if (testStep.Status == Core.Status.Active)
{
// If test case is currently in pass state, continue to execute.
if (testCaseResult.TestVerdict != TestVerdict.Fail)
{
executeTestStep = true; // Alittle redundant, but clarifies logic
}
else
{
// If continue, always execute, if not consider special cases.
if (OnTestStepFailure != OnFailure.Continue)
{
if (!testStep.AlwaysExecute)
{
if (testStep.HasDependency())
{
// If dependency resolve it (if found and pass, continue).
executeTestStep = resolveExecutionDependency(testStep, testCaseResult, out explanation);
}
else
{
executeTestStep = false;
explanation = "Previous step failed." +
" To always run, consider setting test step's \"Always Execute\" property to true.";
}
}
}
}
}
else
{
executeTestStep = false;
}
return(executeTestStep);
}
public TestCaseResultComposite(TestCase testCase, TestCaseResult testCaseResult)
: base(testCase, testCaseResult)
{
Defects = testCase.Defects;
KnownDefects = testCase.KnownDefects;
OnTestStepFailure = testCase.OnTestStepFailure;
Tags = testCase.Tags;
foreach (var testStepResult in testCaseResult.TestStepResults)
{
var testStep = testCase.TestSteps.Find(x => x.SystemID.CompareTo(testStepResult.ReferenceID) == 0);
if (!(testStep is null))
{
TestScripResultCollection.Add(new TestStepResultComposite(testStep as TestStep, testStepResult as TestStepResult));
}
}
}
private TestVerdict getProducerStepResult(TestStep testStep, TestCaseResult testCaseResult)
{
TestVerdict testVerdict = TestVerdict.Unknown;
TestScriptResult dependencySupplier = null;
if (testStep.DependsOn.HasValue && testStep.DependsOn != Guid.Empty)
{
foreach (var testStepResult in testCaseResult.TestStepResults)
{
if (testStepResult.ReferenceID == testStep.DependsOn)
{
dependencySupplier = testStepResult;
testVerdict = dependencySupplier.TestVerdict;
break;
}
}
}
return(testVerdict);
}
/// <summary>
/// Attempts to resolve test steps dependency reference provider's test verdict.
/// </summary>
/// <param name="testStep">Current test step</param>
/// <param name="testCaseResult">Current test case result.</param>
/// <param name="explanation">Comment explaining resolution results. </param>
/// <returns>True if test step can execute, false otherwise.</returns>
private bool resolveExecutionDependency(TestStep testStep, TestCaseResult testCaseResult, out string explanation)
{
bool executeTestStep = false;
explanation = string.Empty;
// If there is a dependency, resolve it.
if (testStep.DependsOn.HasValue && testStep.DependsOn != Guid.Empty)
{
TestScriptResult provider = null;
// Iterate through previous test step results, looking for dependency.
foreach (var testStepResult in testCaseResult.TestStepResults)
{
// If found, evaluate continue based on test verdict. If cannot resolve dependency, do not execute.
if (testStepResult.ReferenceID == testStep.DependsOn)
{
provider = testStepResult;
executeTestStep = provider.TestVerdict == TestVerdict.Pass ? true : false;
explanation = executeTestStep ? string.Empty : "The dependency referenced test step either did not pass or did not execute.";
break;
}
}
if (provider is null)
{
explanation = "Unable to find referenced test step, verify this test step's \"Depends On\" value is correct.";
}
}
else
{
executeTestStep = true;
}
return(executeTestStep);
}
public TestCaseResult Execute()
{
string currentUser = Thread.CurrentThread.Name;
FireExecutionBeginEvent(this, new TestCaseBeginExecutionArgs(currentUser));
TestCaseResult testCaseResult = new TestCaseResult();
testCaseResult.SetReferenceID(SystemID);
testCaseResult.SetVirtualUser(currentUser);
// Save copy of test property collection for restoration later (maintains scope)
TestPropertyCollection savedTestPropertyCollection = new TestPropertyCollection(Core.TestProperties.TestPropertyCollection);
AddRuntimeTestPropertiesToTestProperties();
_testClassDictionary = new Dictionary <int, TestClassBase>();
List <TestScriptObject> activeTestSteps = getActiveChildren();
testCaseResult.SetStartTime(DateTime.Now);
if (activeTestSteps.Count > 0)
{
// Iterate through each test step.
foreach (TestStep testStep in activeTestSteps)
{
// Only execute active test steps.
if (testStep.Status == Core.Status.Active)
{
// Stop if test case already failed (i.e., a previous test step has failed...UNLESS
// a) the test case is marked to continue on failure (continue regardless of previous failures.
// b) a specific test step is marked to always execute (regardless of previous failures).
if ((testCaseResult.TestVerdict != TestVerdict.Fail || OnTestStepFailure == OnFailure.Continue) || testStep.AlwaysExecute)
{
var producerResult = getProducerStepResult(testStep, testCaseResult);
var testStepResult = testStep.Execute(_testClassDictionary);
testCaseResult.AddResult(testStepResult);
testCaseResult.IncrementCounter(testStepResult.TestVerdict);
if (testStepResult.TestVerdict != TestVerdict.Pass)
{
testCaseResult.FinalizeVerdict();
}
}
else
{
var testStepResult = new TestStepResult(TestVerdict.DidNotExecute,
"Did not execute as a previous step failed the test case." +
" To always run, consider setting the \"Always Run\" property to true.");
testCaseResult.AddResult(testStepResult);
testCaseResult.IncrementCounter(testStepResult.TestVerdict);
testStep.FireExecutionCompleteEvent(testStep, testStepResult);
}
}
}
}
else
{
testCaseResult.SetTestVerdict(TestVerdict.DidNotExecute);
testCaseResult.SetTestMessage("The test case is set to \"Active\", however it does not contain active test steps.");
}
testCaseResult.SetEndTime(DateTime.Now);
testCaseResult.FinalizeVerdict();
// Restore preserved test property collection.
Core.TestProperties.SetTestPropertyCollection(savedTestPropertyCollection);
FireExecutionCompleteEvent(this, testCaseResult);
return(testCaseResult);
}
public static void SerializeToFile(TestCaseResult testCaseResult, string filePath)
{
TestArtifact.SerializeToFile(testCaseResult, filePath);
}
public TestCaseResult Execute()
{
string currentUser = Thread.CurrentThread.Name;
FireExecutionBeginEvent(this, new TestCaseBeginExecutionArgs(currentUser));
CheckForPossibleBreakpointMode();
TestCaseResult testCaseResult = new TestCaseResult();
testCaseResult.SetReferenceID(SystemID);
testCaseResult.SetVirtualUser(currentUser);
// Save copy of test property collection for restoration later (maintains scope)
TestPropertyCollection savedTestPropertyCollection = new TestPropertyCollection(Core.TestProperties.TestPropertyCollection);
AddRuntimeTestPropertiesToTestProperties();
_testClassDictionary = new Dictionary <int, TestClassBase>();
List <TestScriptObject> activeTestSteps = getActiveChildren();
testCaseResult.SetStartTime(DateTime.Now);
if (activeTestSteps.Count > 0)
{
// Iterate through each test step.
foreach (TestStep testStep in activeTestSteps)
{
var explanation = string.Empty;
// Determine is test steup should be run. If so, execute.
if (determineExecutionStatus(testStep, testCaseResult, out explanation))
{
var testStepResult = testStep.Execute(_testClassDictionary);
testCaseResult.AddResult(testStepResult);
testCaseResult.IncrementCounter(testStepResult.TestVerdict);
if (testStepResult.TestVerdict != TestVerdict.Pass)
{
testCaseResult.FinalizeVerdict();
}
}
else
{
var testStepResult = new TestStepResult(TestVerdict.DidNotExecute, explanation);
testCaseResult.AddResult(testStepResult);
testCaseResult.IncrementCounter(testStepResult.TestVerdict);
testStep.FireExecutionCompleteEvent(testStep, testStepResult);
}
}
}
else
{
testCaseResult.SetTestVerdict(TestVerdict.DidNotExecute);
testCaseResult.SetTestMessage("The test case is set to \"Active\", however it does not contain active test steps.");
}
testCaseResult.SetEndTime(DateTime.Now);
testCaseResult.FinalizeVerdict();
// Restore preserved test property collection.
Core.TestProperties.SetTestPropertyCollection(savedTestPropertyCollection);
FireExecutionCompleteEvent(this, testCaseResult);
return(testCaseResult);
}
