/// <inheritdoc /> protected override void DecorateTest(IPatternScope scope, ICodeElementInfo codeElement) { scope.TestBuilder.TestInstanceActions.RunTestInstanceBodyChain.Around(delegate(PatternTestInstanceState state, GallioFunc <PatternTestInstanceState, TestOutcome> inner) { TestOutcome outcome = TestOutcome.Passed; int passedCount = 0; for (int i = 0; i < numRepetitions; i++) { string name = String.Format("Repetition #{0}", i + 1); TestContext context = TestStep.RunStep(name, delegate { TestOutcome innerOutcome = inner(state); if (innerOutcome.Status != TestStatus.Passed) { throw new SilentTestException(innerOutcome); } }, null, false, codeElement); outcome = outcome.CombineWith(context.Outcome); if (context.Outcome.Status == TestStatus.Passed) { passedCount += 1; } } TestLog.WriteLine(String.Format("{0} of {1} repetitions passed.", passedCount, numRepetitions)); return(outcome); }); }
/// <inheritdoc /> protected override void DecorateTest(IPatternScope scope, ICodeElementInfo codeElement) { scope.TestBuilder.TestInstanceActions.RunTestInstanceBodyChain.Around( delegate(PatternTestInstanceState state, Gallio.Common.Func <PatternTestInstanceState, TestOutcome> inner) { TestOutcome outcome = TestOutcome.Skipped; if (_configValue == "Default") { TestLog.Warnings.WriteLine("RepeatForConfigValueAttribue did not find a key matching '" + _configKey + "' in the App.Config, using a default value of 1"); _numberOfRepetitions = 1; } else { _numberOfRepetitions = int.Parse(_configValue); } for (int i = 0; i < _numberOfRepetitions; i++) { string name = String.Format("Repetition #{0}", i + 1); TestContext context = TestStep.RunStep(name, delegate { TestOutcome innerOutcome = inner(state); if (innerOutcome.Status != TestStatus.Passed) { throw new SilentTestException(innerOutcome); } }, null, false, codeElement); outcome = context.Outcome; } return(outcome); }); }
/// <inheritdoc /> protected override void DecorateTest(IPatternScope scope, ICodeElementInfo codeElement) { scope.TestBuilder.TestInstanceActions.RunTestInstanceBodyChain.Around((state, inner) => { CultureInfo current = Thread.CurrentThread.CurrentCulture; TestOutcome outcome = TestOutcome.Passed; int passedCount = 0; try { foreach (string culture in cultures) { string name = String.Format("Culture {0}", culture); TestContext context = TestStep.RunStep(name, () => { CultureInfo cultureInfo; try { cultureInfo = new CultureInfo(culture); } catch (Exception exception) { TestContext.CurrentContext.LogWriter.Default.WriteException(exception, String.Format("An exception occurred while setting the culture to '{0}'.", culture)); throw new SilentTestException(TestOutcome.Error); } Thread.CurrentThread.CurrentCulture = cultureInfo; TestOutcome innerOutcome = inner(state); if (innerOutcome.Status != TestStatus.Passed) { throw new SilentTestException(innerOutcome); } }, null, false, codeElement); outcome = outcome.CombineWith(context.Outcome); if (context.Outcome.Status == TestStatus.Passed) { passedCount++; } } } finally { Thread.CurrentThread.CurrentCulture = current; } TestLog.WriteLine(String.Format("{0} of {1} cultures passed.", passedCount, cultures.Length)); return(outcome); }); }
public void DifferentTestContextsHaveDifferentAssertionContexts() { AssertionContext a = null, b = null; TestStep.RunStep("A", () => a = AssertionContext.CurrentContext); TestStep.RunStep("B", () => b = AssertionContext.CurrentContext); Assert.IsNotNull(a); Assert.IsNotNull(b); Assert.AreNotSame(a, b); }
/// <summary> /// Execute the script and verify it passed /// </summary> /// <param name="testName"></param> /// <returns></returns> public TestOutcome ExecuteScriptAndGetOutcome(string testName) { Action executeTest = delegate { Driver.Connect(host); Driver.ExecuteScript(scriptName, description); VerifySuccess(); AttachTestFiles(); }; return(TestStep.RunStep(testName, executeTest, new TimeSpan(0, 0, timeoutMin, 0), true, null).Outcome); }
public void RunStep_NameActionTimeoutIsTestCaseAndCodeElement_ExecutionWithIsTestCaseTrue() { bool ran = false; TestContext context = TestStep.RunStep("Abc", () => { ran = true; }, null, true, null); Assert.IsTrue(ran, "Verify that the step ran."); Assert.AreEqual(TestOutcome.Passed, context.Outcome, "Verify that the step has the expected outcome."); Assert.AreEqual("Abc", context.TestStep.Name); Assert.IsNull(context.TestStep.CodeElement); Assert.IsTrue(context.TestStep.IsTestCase); }
public void RunStep_NameAndAction_Execution() { bool ran = false; TestContext context = TestStep.RunStep("Abc", () => { ran = true; Assert.Inconclusive(); }); Assert.IsTrue(ran, "Verify that the step ran."); Assert.AreEqual(TestOutcome.Inconclusive, context.Outcome, "Verify that the step has the expected outcome."); Assert.AreEqual("Abc", context.TestStep.Name); Assert.AreEqual("RunStep_NameAndAction_Execution", context.TestStep.CodeElement.CodeReference.MemberName); Assert.IsFalse(context.TestStep.IsTestCase); }
public void RunStep_NameActionAndTimeout_Execution() { bool ran = false; TestContext context = TestStep.RunStep("Abc", () => { ran = true; Assert.TerminateSilently(TestOutcome.Skipped); }, null); Assert.IsTrue(ran, "Verify that the step ran."); Assert.AreEqual(TestOutcome.Skipped, context.Outcome, "Verify that the step has the expected outcome."); Assert.AreEqual("Abc", context.TestStep.Name); Assert.AreEqual("RunStep_NameActionAndTimeout_Execution", context.TestStep.CodeElement.CodeReference.MemberName); Assert.IsFalse(context.TestStep.IsTestCase); }
protected override TestOutcome RunDynamicTest(ICodeElementInfo declaringCodeElement, Action setUp, Action tearDown) { return(TestStep.RunStep(Name, () => { TestStep.AddMetadata(MetadataKeys.TestKind, Kind); foreach (var pair in Metadata.Pairs) { TestStep.AddMetadata(pair.Key, pair.Value); } try { TestContext.CurrentContext.LifecyclePhase = LifecyclePhases.SetUp; if (setUp != null) { setUp(); } try { OnSetupSelf(); TestContext.CurrentContext.LifecyclePhase = LifecyclePhases.Execute; OnExecuteSelf(); TestOutcome outcome = RunDynamicTests(GetChildren(), declaringCodeElement, OnSetupChild, OnTearDownChild); if (outcome != TestOutcome.Passed) { throw new SilentTestException(outcome); } } finally { TestContext.CurrentContext.LifecyclePhase = LifecyclePhases.TearDown; OnTearDownSelf(); } } finally { TestContext.CurrentContext.LifecyclePhase = LifecyclePhases.TearDown; if (tearDown != null) { tearDown(); } } }, Timeout, IsTestCase, CodeElement ?? declaringCodeElement).Outcome); }
public void RunStep_NameActionTimeoutIsTestCaseAndCodeElement_ExecutionWithCodeElement() { bool ran = false; TestContext context = TestStep.RunStep("Abc", () => { ran = true; Assert.TerminateSilently(TestOutcome.Skipped); }, null, false, Reflector.Wrap(typeof(TestStep))); Assert.IsTrue(ran, "Verify that the step ran."); Assert.AreEqual(TestOutcome.Skipped, context.Outcome, "Verify that the step has the expected outcome."); Assert.AreEqual("Abc", context.TestStep.Name); Assert.EndsWith(context.TestStep.CodeElement.CodeReference.TypeName, "TestStep"); Assert.IsNull(context.TestStep.CodeElement.CodeReference.MemberName); Assert.IsFalse(context.TestStep.IsTestCase); }
public void RunStep_NameActionAndTimeout_ExecutionWithUnexpiredTimeout() { bool ran = false; TestContext context = TestStep.RunStep("Abc", () => { ran = true; Thread.Sleep(100); Assert.TerminateSilently(TestOutcome.Pending); }, TimeSpan.FromSeconds(60)); Assert.IsTrue(ran, "Verify that the step ran."); Assert.AreEqual(TestOutcome.Pending, context.Outcome, "Verify that the step has the expected outcome."); Assert.AreEqual("Abc", context.TestStep.Name); Assert.AreEqual("RunStep_NameActionAndTimeout_ExecutionWithUnexpiredTimeout", context.TestStep.CodeElement.CodeReference.MemberName); Assert.IsFalse(context.TestStep.IsTestCase); }
public void RunStep_NameActionAndTimeout_ExecutionWithExpiredTimeout() { bool ran = false; bool shouldNotGetHere = false; TestContext context = TestStep.RunStep("Abc", () => { ran = true; Thread.Sleep(1000); shouldNotGetHere = true; Assert.TerminateSilently(TestOutcome.Skipped); }, TimeSpan.FromMilliseconds(100)); Assert.IsTrue(ran, "Verify that the step ran."); Assert.IsFalse(shouldNotGetHere, "Verify that the step aborted prematurely due to the timeout."); Assert.AreEqual(TestOutcome.Timeout, context.Outcome, "Verify that the step has the expected outcome."); Assert.AreEqual("Abc", context.TestStep.Name); Assert.AreEqual("RunStep_NameActionAndTimeout_ExecutionWithExpiredTimeout", context.TestStep.CodeElement.CodeReference.MemberName); Assert.IsFalse(context.TestStep.IsTestCase); }
/// <inheritdoc /> protected override void DecorateTest(IPatternScope scope, ICodeElementInfo codeElement) { scope.TestBuilder.TestInstanceActions.RunTestInstanceBodyChain.Around( delegate(PatternTestInstanceState state, Gallio.Common.Func <PatternTestInstanceState, TestOutcome> inner) { TestOutcome outcome = TestOutcome.Passed; int failureCount = 0; // we will try up to 'max' times to get a pass, if we do, then break out and don't run the test anymore for (int i = 0; i < _maxNumberOfAttempts; i++) { string name = String.Format("Repetition #{0}", i + 1); TestContext context = TestStep.RunStep(name, delegate { TestOutcome innerOutcome = inner(state); // if we get a fail, and we have used up the number of attempts allowed to get a pass, throw an error if (innerOutcome.Status != TestStatus.Passed) { throw new SilentTestException(innerOutcome); } }, null, false, codeElement); outcome = context.Outcome; // escape the loop if the test has passed, otherwise increment the failure count if (context.Outcome.Status == TestStatus.Passed) { break; } failureCount++; } //TestLog.WriteLine(String.Format( Log.Message(String.Format( failureCount == _maxNumberOfAttempts ? "Tried {0} times to get a pass test result but didn't get it" : "The test passed on attempt {1} out of {0}", _maxNumberOfAttempts, failureCount + 1)); return(outcome); }); }
/// <inheritdoc /> protected override void DecorateTest(IPatternScope scope, ICodeElementInfo codeElement) { scope.TestBuilder.TestInstanceActions.RunTestInstanceBodyChain.Around( delegate(PatternTestInstanceState state, Gallio.Common.Func <PatternTestInstanceState, TestOutcome> inner) { TestOutcome outcome = TestOutcome.Passed; //int failureCount = 0; // we will try up to 'max' times to get a pass, if we do, then break out and don't run the test anymore Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); for (int i = 0; stopwatch.Elapsed < TimeSpan.FromMinutes(_minutes); i++) { string name = String.Format("Repetition #{0}", i + 1); DiagnosticLog.WriteLine(name); TestContext context = TestStep.RunStep(name, delegate { TestOutcome innerOutcome = inner(state); //if we get a fail, and we have used up the number of attempts allowed to get a pass, throw an error if (innerOutcome.Status != TestStatus.Passed) { throw new SilentTestException(innerOutcome); } }, null, false, codeElement); outcome = context.Outcome; // escape the loop if the test has passed, otherwise increment the failure count //if (context.Outcome.Status == TestStatus.Passed) // break; // failureCount++; } Log.Message(String.Format("Ran the test for {0} minutes", _minutes)); //TestLog.WriteLine(String.Format("Ran the test for {0} minutes", _minutes)); return(outcome); }); }
public void RunStep_NameAndAction_ArgumentValidation() { Assert.Throws <ArgumentNullException>(() => TestStep.RunStep(null, () => { })); Assert.Throws <ArgumentNullException>(() => TestStep.RunStep("Abc", null)); }
public void RunStep_NameActionAndTimeout_ArgumentValidation() { Assert.Throws <ArgumentNullException>(() => TestStep.RunStep(null, () => { }, TimeSpan.FromSeconds(60))); Assert.Throws <ArgumentNullException>(() => TestStep.RunStep("Abc", null, TimeSpan.FromSeconds(60))); Assert.Throws <ArgumentOutOfRangeException>(() => TestStep.RunStep("Abc", () => { }, TimeSpan.FromSeconds(-1))); }
/// <inheritdoc /> protected override void DecorateTest(IPatternScope scope, ICodeElementInfo codeElement) { scope.TestBuilder.TestInstanceActions.RunTestInstanceBodyChain.Around((state, inner) => { TaskContainer container = new TaskContainer(); try { TestOutcome[] threadOutcomes = new TestOutcome[numThreads]; TestContext context = TestContext.CurrentContext; for (int i = 0; i < numThreads; i++) { int index = i; string name = String.Format("Threaded Repetition #{0}", index + 1); var task = new TestEnvironmentAwareThreadTask(name, delegate { TestContext threadContext = TestStep.RunStep(name, delegate { TestOutcome innerOutcome = inner(state); if (innerOutcome.Status != TestStatus.Passed) { throw new SilentTestException(innerOutcome); } }, null, false, codeElement); threadOutcomes[index] = threadContext.Outcome; }, null); task.Terminated += delegate { if (!task.Result.HasValue) { threadOutcomes[index] = TestOutcome.Error; context.LogWriter.Default.WriteException(task.Result.Exception, String.Format("An exception occurred while starting Threaded Repetition #{0}.", index)); } }; container.Watch(task); task.Start(); } container.JoinAll(null); TestOutcome outcome = TestOutcome.Passed; int passedCount = 0; foreach (TestOutcome threadOutcome in threadOutcomes) { outcome = outcome.CombineWith(threadOutcome); if (threadOutcome.Status == TestStatus.Passed) { passedCount += 1; } } context.LogWriter.Default.WriteLine(String.Format("{0} of {1} threaded repetitions passed.", passedCount, numThreads)); return(outcome); } finally { container.AbortAll(); container.JoinAll(null); } }); }
public void RunStep_NameActionTimeoutIsTestCaseAndCodeElement_ArgumentValidation() { Assert.Throws <ArgumentNullException>(() => TestStep.RunStep(null, () => { }, TimeSpan.FromSeconds(60), false, Reflector.Wrap(typeof(TestStepTest)))); Assert.Throws <ArgumentNullException>(() => TestStep.RunStep("Abc", null, TimeSpan.FromSeconds(60), false, Reflector.Wrap(typeof(TestStepTest)))); Assert.Throws <ArgumentOutOfRangeException>(() => TestStep.RunStep("Abc", () => { }, TimeSpan.FromSeconds(-1), false, Reflector.Wrap(typeof(TestStepTest)))); }
//private readonly int numRepetitions; // private int _maxNumberOfAttempts; /// <summary> /// Executes the test method repeatedly. /// </summary> /// <example> /// <code><![CDATA[ /// [Test] /// [Repeat(10)] /// public void Test() /// { /// // This test will be executed 10 times. /// } /// ]]></code> /// </example> /// <param name="numRepetitions">The number of times to repeat the test.</param> /// <exception cref="ArgumentOutOfRangeException">Thrown if <paramref name="numRepetitions"/> /// is less than 1.</exception> protected override void DecorateTest(IPatternScope scope, ICodeElementInfo codeElement) { scope.TestBuilder.TestInstanceActions.RunTestInstanceBodyChain.Around(delegate(PatternTestInstanceState state, Gallio.Common.Func <PatternTestInstanceState, TestOutcome> inner) { TestOutcome outcome = TestOutcome.Passed; int failureCount = 0; // we will try up to 'max' times to get a pass, if we do, then break out and don't run the test anymore for (int i = 0; i < FrameGlobals.repeatCount; i++) { string name = String.Format("Repetition #{0}", i + 1); TestContext context = TestStep.RunStep(name, delegate { TestOutcome innerOutcome = inner(state); // if we get a fail, and we have used up the number of attempts allowed to get a pass, throw an error if (innerOutcome.Status != TestStatus.Passed) { throw new SilentTestException(innerOutcome); } }, null, false, codeElement); outcome = context.Outcome; // escape the loop if the test has pa -0ssed, otherwise increment the failure count if (context.Outcome.Status == TestStatus.Passed) { break; } failureCount++; } TestLog.WriteLine(String.Format( failureCount == FrameGlobals.repeatCount ? "Tried {0} times to get a pass test result but didn't get it" : "The test passed on attempt {1} out of {0}", FrameGlobals.repeatCount, failureCount + 1)); Queue tcStatus2 = new Queue(); BaseTest.TestDetails tc2 = new BaseTest.TestDetails(); string parent = Gallio.Framework.TestContext.CurrentContext.Test.Parent.ToString(); tc2.Name = Gallio.Framework.TestContext.CurrentContext.Test.Name; if (outcome.Status == TestStatus.Passed) { tc2.Status = "Pass"; } else { tc2.Status = "Fail"; } if (BaseTest.thirdStatus.ContainsKey(parent + tc2.Name)) { tc2.Status = BaseTest.thirdStatus[parent + tc2.Name]; } if (BaseTest.testStatus.ContainsKey(parent)) { tcStatus2 = BaseTest.testStatus[parent]; tcStatus2.Enqueue(tc2); BaseTest.testStatus.Remove(parent); BaseTest.testStatus.Add(parent, tcStatus2); } else { tcStatus2.Enqueue(tc2); BaseTest.testStatus.Add(Gallio.Framework.TestContext.CurrentContext.Test.Parent.ToString(), tcStatus2); } return(outcome); }); }