/// <summary> /// Determines execution status. /// </summary> /// <param name="testStep">Current teststep under determination</param> /// <param name="testCaseResult">Current test case result (used to resolved dependencies).</param> /// <param name="explanation">Explanation of determination if approprieate.</param> /// <returns>True to subsequently execute, false if not.</returns> /// <remarks> /// If all previous test step's passed, always execute unless current test step's execution is dependent on previous test step's passage. /// However if a previous step failed but the test step is marked to always execute do so. If the parent /// test case's OnTestStepFailure is set to "Continue", execute all active child test steps. /// </remarks> private bool determineExecutionStatus(TestStep testStep, TestCaseResult testCaseResult, out string explanation) { var executeTestStep = true; explanation = string.Empty; // Only execute active test steps. if (testStep.Status == Core.Status.Active) { // If test case is currently in pass state, continue to execute. if (testCaseResult.TestVerdict != TestVerdict.Fail) { executeTestStep = true; // Alittle redundant, but clarifies logic } else { // If continue, always execute, if not consider special cases. if (OnTestStepFailure != OnFailure.Continue) { if (!testStep.AlwaysExecute) { if (testStep.HasDependency()) { // If dependency resolve it (if found and pass, continue). executeTestStep = resolveExecutionDependency(testStep, testCaseResult, out explanation); } else { executeTestStep = false; explanation = "Previous step failed." + " To always run, consider setting test step's \"Always Execute\" property to true."; } } } } } else { executeTestStep = false; } return(executeTestStep); }
public TestCaseResultComposite(TestCase testCase, TestCaseResult testCaseResult) : base(testCase, testCaseResult) { Defects = testCase.Defects; KnownDefects = testCase.KnownDefects; OnTestStepFailure = testCase.OnTestStepFailure; Tags = testCase.Tags; foreach (var testStepResult in testCaseResult.TestStepResults) { var testStep = testCase.TestSteps.Find(x => x.SystemID.CompareTo(testStepResult.ReferenceID) == 0); if (!(testStep is null)) { TestScripResultCollection.Add(new TestStepResultComposite(testStep as TestStep, testStepResult as TestStepResult)); } } }
private TestVerdict getProducerStepResult(TestStep testStep, TestCaseResult testCaseResult) { TestVerdict testVerdict = TestVerdict.Unknown; TestScriptResult dependencySupplier = null; if (testStep.DependsOn.HasValue && testStep.DependsOn != Guid.Empty) { foreach (var testStepResult in testCaseResult.TestStepResults) { if (testStepResult.ReferenceID == testStep.DependsOn) { dependencySupplier = testStepResult; testVerdict = dependencySupplier.TestVerdict; break; } } } return(testVerdict); }
/// <summary> /// Attempts to resolve test steps dependency reference provider's test verdict. /// </summary> /// <param name="testStep">Current test step</param> /// <param name="testCaseResult">Current test case result.</param> /// <param name="explanation">Comment explaining resolution results. </param> /// <returns>True if test step can execute, false otherwise.</returns> private bool resolveExecutionDependency(TestStep testStep, TestCaseResult testCaseResult, out string explanation) { bool executeTestStep = false; explanation = string.Empty; // If there is a dependency, resolve it. if (testStep.DependsOn.HasValue && testStep.DependsOn != Guid.Empty) { TestScriptResult provider = null; // Iterate through previous test step results, looking for dependency. foreach (var testStepResult in testCaseResult.TestStepResults) { // If found, evaluate continue based on test verdict. If cannot resolve dependency, do not execute. if (testStepResult.ReferenceID == testStep.DependsOn) { provider = testStepResult; executeTestStep = provider.TestVerdict == TestVerdict.Pass ? true : false; explanation = executeTestStep ? string.Empty : "The dependency referenced test step either did not pass or did not execute."; break; } } if (provider is null) { explanation = "Unable to find referenced test step, verify this test step's \"Depends On\" value is correct."; } } else { executeTestStep = true; } return(executeTestStep); }
public TestCaseResult Execute() { string currentUser = Thread.CurrentThread.Name; FireExecutionBeginEvent(this, new TestCaseBeginExecutionArgs(currentUser)); TestCaseResult testCaseResult = new TestCaseResult(); testCaseResult.SetReferenceID(SystemID); testCaseResult.SetVirtualUser(currentUser); // Save copy of test property collection for restoration later (maintains scope) TestPropertyCollection savedTestPropertyCollection = new TestPropertyCollection(Core.TestProperties.TestPropertyCollection); AddRuntimeTestPropertiesToTestProperties(); _testClassDictionary = new Dictionary <int, TestClassBase>(); List <TestScriptObject> activeTestSteps = getActiveChildren(); testCaseResult.SetStartTime(DateTime.Now); if (activeTestSteps.Count > 0) { // Iterate through each test step. foreach (TestStep testStep in activeTestSteps) { // Only execute active test steps. if (testStep.Status == Core.Status.Active) { // Stop if test case already failed (i.e., a previous test step has failed...UNLESS // a) the test case is marked to continue on failure (continue regardless of previous failures. // b) a specific test step is marked to always execute (regardless of previous failures). if ((testCaseResult.TestVerdict != TestVerdict.Fail || OnTestStepFailure == OnFailure.Continue) || testStep.AlwaysExecute) { var producerResult = getProducerStepResult(testStep, testCaseResult); var testStepResult = testStep.Execute(_testClassDictionary); testCaseResult.AddResult(testStepResult); testCaseResult.IncrementCounter(testStepResult.TestVerdict); if (testStepResult.TestVerdict != TestVerdict.Pass) { testCaseResult.FinalizeVerdict(); } } else { var testStepResult = new TestStepResult(TestVerdict.DidNotExecute, "Did not execute as a previous step failed the test case." + " To always run, consider setting the \"Always Run\" property to true."); testCaseResult.AddResult(testStepResult); testCaseResult.IncrementCounter(testStepResult.TestVerdict); testStep.FireExecutionCompleteEvent(testStep, testStepResult); } } } } else { testCaseResult.SetTestVerdict(TestVerdict.DidNotExecute); testCaseResult.SetTestMessage("The test case is set to \"Active\", however it does not contain active test steps."); } testCaseResult.SetEndTime(DateTime.Now); testCaseResult.FinalizeVerdict(); // Restore preserved test property collection. Core.TestProperties.SetTestPropertyCollection(savedTestPropertyCollection); FireExecutionCompleteEvent(this, testCaseResult); return(testCaseResult); }
public static void SerializeToFile(TestCaseResult testCaseResult, string filePath) { TestArtifact.SerializeToFile(testCaseResult, filePath); }
public TestCaseResult Execute() { string currentUser = Thread.CurrentThread.Name; FireExecutionBeginEvent(this, new TestCaseBeginExecutionArgs(currentUser)); CheckForPossibleBreakpointMode(); TestCaseResult testCaseResult = new TestCaseResult(); testCaseResult.SetReferenceID(SystemID); testCaseResult.SetVirtualUser(currentUser); // Save copy of test property collection for restoration later (maintains scope) TestPropertyCollection savedTestPropertyCollection = new TestPropertyCollection(Core.TestProperties.TestPropertyCollection); AddRuntimeTestPropertiesToTestProperties(); _testClassDictionary = new Dictionary <int, TestClassBase>(); List <TestScriptObject> activeTestSteps = getActiveChildren(); testCaseResult.SetStartTime(DateTime.Now); if (activeTestSteps.Count > 0) { // Iterate through each test step. foreach (TestStep testStep in activeTestSteps) { var explanation = string.Empty; // Determine is test steup should be run. If so, execute. if (determineExecutionStatus(testStep, testCaseResult, out explanation)) { var testStepResult = testStep.Execute(_testClassDictionary); testCaseResult.AddResult(testStepResult); testCaseResult.IncrementCounter(testStepResult.TestVerdict); if (testStepResult.TestVerdict != TestVerdict.Pass) { testCaseResult.FinalizeVerdict(); } } else { var testStepResult = new TestStepResult(TestVerdict.DidNotExecute, explanation); testCaseResult.AddResult(testStepResult); testCaseResult.IncrementCounter(testStepResult.TestVerdict); testStep.FireExecutionCompleteEvent(testStep, testStepResult); } } } else { testCaseResult.SetTestVerdict(TestVerdict.DidNotExecute); testCaseResult.SetTestMessage("The test case is set to \"Active\", however it does not contain active test steps."); } testCaseResult.SetEndTime(DateTime.Now); testCaseResult.FinalizeVerdict(); // Restore preserved test property collection. Core.TestProperties.SetTestPropertyCollection(savedTestPropertyCollection); FireExecutionCompleteEvent(this, testCaseResult); return(testCaseResult); }