private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor) { // NOTE: This method has been optimized to minimize the total stack depth of the action // by inlining blocks on the critical path that had previously been factored out. using (TestController testController = testControllerProvider(testCommand.Test)) { if (testController != null) { try { using (IProgressMonitor subProgressMonitor = progressMonitor.CreateSubProgressMonitor(testCommand.TestCount)) { // Calling RunImpl directly instead of Run to minimize stack depth // because we already know the arguments are valid. return(testController.RunImpl(testCommand, parentTestStep, options, subProgressMonitor)); } } catch (Exception ex) { ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep); context.LogWriter.Failures.WriteException(ex, "Fatal Exception in test controller"); return(context.FinishStep(TestOutcome.Error, null)); } } } // Enter the scope of the test and recurse until we find a controller. progressMonitor.SetStatus(testCommand.Test.FullName); ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand monitor in testCommand.Children) { if (progressMonitor.IsCanceled) { break; } TestResult childResult = RunTest(monitor, testContext.TestStep, options, progressMonitor); outcome = outcome.CombineWith(childResult.Outcome).Generalize(); } if (progressMonitor.IsCanceled) { outcome = TestOutcome.Canceled; } TestResult result = testContext.FinishStep(outcome, null); progressMonitor.Worked(1); return(result); }
protected override TestResult RunImpl( ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor ) { using(progressMonitor.BeginTask( "Verifying Specifications", rootTestCommand.TestCount ) ) { if( options.SkipTestExecution ) { return SkipAll( rootTestCommand, parentTestStep ); } else { ITestContext rootContext = rootTestCommand.StartPrimaryChildStep( parentTestStep ); TestStep rootStep = rootContext.TestStep; TestOutcome outcome = TestOutcome.Passed; foreach( ITestCommand command in rootTestCommand.Children ) { NSpecAssemblyTest assemblyTest = command.Test as NSpecAssemblyTest; if( assemblyTest == null ) continue; var assemblyResult = this.RunAssembly( command, rootStep ); outcome = outcome.CombineWith( assemblyResult.Outcome ); } return rootContext.FinishStep( outcome, null ); } } }
private static TestResult ReportTestError(ITestCommand testCommand, Model.Tree.TestStep parentTestStep, Exception ex, string message) { ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep); TestLog.Failures.WriteException(ex, message); return(context.FinishStep(TestOutcome.Error, null)); }
TestResult RunAssembly(MachineAssemblyTest assemblyTest, ITestCommand command, TestStep parentTestStep) { ITestContext assemblyContext = command.StartPrimaryChildStep(parentTestStep); AssemblyInfo assemblyInfo = new AssemblyInfo(assemblyTest.Name, assemblyTest.AssemblyFilePath); TestOutcome outcome = TestOutcome.Passed; _listener.OnAssemblyStart(assemblyInfo); assemblyTest.AssemblyContexts.Each(context => context.OnAssemblyStart()); foreach (ITestCommand contextCommand in command.Children) { MachineContextTest contextTest = contextCommand.Test as MachineContextTest; if (contextTest == null) { continue; } var contextResult = RunContextTest(assemblyTest, contextTest, contextCommand, assemblyContext.TestStep); outcome = outcome.CombineWith(contextResult.Outcome); assemblyContext.SetInterimOutcome(outcome); } assemblyTest.AssemblyContexts.Reverse().Each(context => context.OnAssemblyComplete()); _listener.OnAssemblyEnd(assemblyInfo); return(assemblyContext.FinishStep(outcome, null)); }
TestResult RunTest(NSpecContextTest contextTest, NSpecExampleTest exampleTest, ITestCommand testCommand, TestStep testStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(testStep); TestOutcome outcome = TestOutcome.Passed; if (exampleTest.Example.Pending) { outcome = TestOutcome.Pending; testContext.AddMetadata(MetadataKeys.PendingReason, "Needs to be implemented"); } else { contextTest.Context.Exercise(exampleTest.Example, contextTest.Context.GetInstance()); if (exampleTest.Example.Exception != null) { TestLog.Failures.WriteException(ConvertException(exampleTest.Example.Exception)); TestLog.Failures.Flush(); outcome = TestOutcome.Failed; } } return(testContext.FinishStep(outcome, null)); }
private ITestContext GetFixtureContext(ITestCommand fixtureCommand) { ITestContext parentContext = testContextStack.Peek(); if (parentContext.TestStep.Test != fixtureCommand.Test) { while (parentContext.TestStep.Test.Kind != CSUnitTestExplorer.AssemblyKind) { testContextStack.Pop(); TestOutcome outcome = GetFixtureOutcome(fixtureFailureCount, fixtureErrorCount); parentContext.FinishStep(outcome, null); progressMonitor.Worked(1); parentContext = testContextStack.Peek(); } parentContext = fixtureCommand.StartPrimaryChildStep(parentContext.TestStep); parentContext.LifecyclePhase = LifecyclePhases.Execute; progressMonitor.SetStatus(fixtureCommand.Test.Name); testContextStack.Push(parentContext); fixtureFailureCount = 0; fixtureErrorCount = 0; } return(parentContext); }
private TestResult RunContext(NSpecContextTest contextTest, ITestCommand command, TestStep testStep) { ITestContext testContext = command.StartPrimaryChildStep(testStep); TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand testCommand in command.Children) { NSpecExampleTest exampleTest = testCommand.Test as NSpecExampleTest; if (exampleTest == null) { continue; } outcome = outcome.CombineWith(this.RunTest(contextTest, exampleTest, testCommand, testContext.TestStep).Outcome); } foreach (ITestCommand testCommand in command.Children) { NSpecContextTest contextTestChild = testCommand.Test as NSpecContextTest; if (contextTestChild == null) { continue; } outcome = outcome.CombineWith(this.RunContext(contextTestChild, testCommand, testContext.TestStep).Outcome); } return(testContext.FinishStep(outcome, null)); }
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { Test test = testCommand.Test; progressMonitor.SetStatus(test.Name); // The first test should be an assembly test MSTestAssembly assemblyTest = testCommand.Test as MSTestAssembly; TestOutcome outcome; TestResult result; if (assemblyTest != null) { ITestContext assemblyContext = testCommand.StartPrimaryChildStep(parentTestStep); try { MSTestRunner runner = MSTestRunner.GetRunnerForFrameworkVersion(frameworkVersion); outcome = runner.RunSession(assemblyContext, assemblyTest, testCommand, parentTestStep, progressMonitor); } catch (Exception ex) { assemblyContext.LogWriter.Failures.WriteException(ex, "Internal Error"); outcome = TestOutcome.Error; } result = assemblyContext.FinishStep(outcome, null); } else { result = new TestResult(TestOutcome.Skipped); } progressMonitor.Worked(1); return result; }
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor) { using (progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount)) { if (options.SkipTestExecution) { return(SkipAll(rootTestCommand, parentTestStep)); } else { ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep); TestStep rootStep = rootContext.TestStep; TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand command in rootTestCommand.Children) { NSpecAssemblyTest assemblyTest = command.Test as NSpecAssemblyTest; if (assemblyTest == null) { continue; } var assemblyResult = this.RunAssembly(command, rootStep); outcome = outcome.CombineWith(assemblyResult.Outcome); } return(rootContext.FinishStep(outcome, null)); } } }
private TestResult RunContext( NSpecContextTest contextTest, ITestCommand command, TestStep testStep ) { ITestContext testContext = command.StartPrimaryChildStep( testStep ); TestOutcome outcome = TestOutcome.Passed; foreach( ITestCommand testCommand in command.Children ) { NSpecExampleTest exampleTest = testCommand.Test as NSpecExampleTest; if( exampleTest == null ) { continue; } outcome = outcome.CombineWith( this.RunTest( contextTest, exampleTest, testCommand, testContext.TestStep ).Outcome ); } foreach( ITestCommand testCommand in command.Children ) { NSpecContextTest contextTestChild = testCommand.Test as NSpecContextTest; if( contextTestChild == null ) { continue; } outcome = outcome.CombineWith( this.RunContext( contextTestChild, testCommand, testContext.TestStep ).Outcome ); } return testContext.FinishStep( outcome, null ); }
private static TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); bool passed = true; foreach (ITestCommand child in testCommand.Children) passed &= RunTest(child, testContext.TestStep, progressMonitor).Outcome.Status == TestStatus.Passed; return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null); }
/// <summary> /// Recursively generates single test steps for each <see cref="ITestCommand" /> and /// sets the final outcome to <see cref="TestOutcome.Skipped" />. /// </summary> /// <remarks> /// <para> /// This is useful for implementing fallback behavior when /// <see cref="TestExecutionOptions.SkipTestExecution" /> is true. /// </para> /// </remarks> /// <param name="rootTestCommand">The root test command.</param> /// <param name="parentTestStep">The parent test step.</param> /// <returns>The combined result of the test commands.</returns> protected static TestResult SkipAll(ITestCommand rootTestCommand, TestStep parentTestStep) { ITestContext context = rootTestCommand.StartPrimaryChildStep(parentTestStep); foreach (ITestCommand child in rootTestCommand.Children) { SkipAll(child, context.TestStep); } return(context.FinishStep(TestOutcome.Skipped, null)); }
private TestResult RunTestStep(ITestCommand testCommand, TestInfoData testStepInfo, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestStepResult testStepResult = repository.RunTest(testStepInfo); reporter.Run(testContext, testStepInfo, testStepResult); WriteToTestLog(testContext, testStepResult); testContext.AddAssertCount(testStepResult.AssertCount); progressMonitor.Worked(1); return(testContext.FinishStep(testStepResult.TestOutcome, testStepResult.Duration)); }
TestResult RunContextTest(MachineAssemblyTest assemblyTest, MachineContextTest contextTest, ITestCommand command, TestStep parentTestStep) { ITestContext testContext = command.StartPrimaryChildStep(parentTestStep); GallioRunListener listener = new GallioRunListener(_listener, _progressMonitor, testContext, command.Children); IContextRunner runner = ContextRunnerFactory.GetContextRunnerFor(contextTest.Context); runner.Run(contextTest.Context, listener, _options, assemblyTest.GlobalCleanup, assemblyTest.SpecificationSupplements); return(testContext.FinishStep(listener.Outcome, null)); }
public override void OnSpecificationStart(SpecificationInfo specification) { _listener.OnSpecificationStart(specification); ITestCommand specCommand = _commandsBySpec[specification.Name]; ITestContext specContext = specCommand.StartPrimaryChildStep(_testContext.TestStep); _contextsBySpec.Add(specification.Name, specContext); specContext.LifecyclePhase = LifecyclePhases.Starting; _progressMonitor.SetStatus("» " + specification.Name); }
private static TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); bool passed = true; foreach (ITestCommand child in testCommand.Children) { passed &= RunTest(child, testContext.TestStep, progressMonitor).Outcome.Status == TestStatus.Passed; } return(testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null)); }
private static bool RunTestMethod(ITestCommand testCommand, MethodInfo methodInfo, XunitTestClassCommand testClassCommand, TestStep parentTestStep) { List <XunitTestCommand> xunitTestCommands; try { xunitTestCommands = new List <XunitTestCommand>(XunitTestCommandFactory.Make(testClassCommand, XunitReflector.Wrap(methodInfo))); } catch (Exception ex) { // Xunit can throw exceptions when making commands if the test is malformed. ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); testContext.FinishStep(TestOutcome.Failed, null); return(false); } if (xunitTestCommands.Count == 0) { return(true); } if (xunitTestCommands.Count == 1) { return(RunTestCommands(testCommand, testClassCommand, xunitTestCommands, parentTestStep, true)); } // Introduce a common primary test step for theories. ITestContext primaryTestContext = testCommand.StartPrimaryChildStep(parentTestStep); bool result = RunTestCommands(testCommand, testClassCommand, xunitTestCommands, primaryTestContext.TestStep, false); primaryTestContext.FinishStep(result ? TestOutcome.Passed : TestOutcome.Failed, null); return(result); }
private TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestOutcome combinedOutCome = TestOutcome.Passed; var duration = TimeSpan.Zero; foreach (ITestCommand child in testCommand.Children) { TestResult testResult = RunTest(child, testContext.TestStep, progressMonitor); combinedOutCome = combinedOutCome.CombineWith(testResult.Outcome); duration += testResult.Duration; } return(testContext.FinishStep(combinedOutCome, duration)); }
private static TestResult RunTestFixture(ITestCommand testCommand, ConcordionTest concordionTest, TestStep parentTestStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); // The magic happens here! var concordion = new ConcordionBuilder() .WithSource(concordionTest.Source) .WithTarget(concordionTest.Target) .WithSpecificationProcessingListener(new GallioResultRenderer()) .Build(); ConstructorInfo constructor = concordionTest.FixtureType.GetConstructor(Type.EmptyTypes); var fixture=constructor.Invoke(new object[]{}); var summary = concordion.Process(concordionTest.Resource, fixture); bool passed = !(summary.HasFailures || summary.HasExceptions); testContext.AddAssertCount((int)summary.SuccessCount + (int)summary.FailureCount); return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null); }
private static bool RunTestFixture(ITestCommand testCommand, ConcordionTest concordionTest, ITestStep parentTestStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); // The magic happens here! var concordion = new ConcordionBuilder() .WithSource(concordionTest.Source) .WithTarget(concordionTest.Target) .WithSpecificationListener(new GallioResultRenderer()) .Build(); var summary = concordion.Process(concordionTest.Resource, concordionTest.Fixture); bool passed = !(summary.HasFailures || summary.HasExceptions); testContext.AddAssertCount((int)summary.SuccessCount + (int)summary.FailureCount); testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null); return(passed); }
private TestResult RunAssembly( ITestCommand command, TestStep rootStep ) { ITestContext assemblyContext = command.StartPrimaryChildStep( rootStep ); TestOutcome outcome = TestOutcome.Passed; foreach( ITestCommand contextCommand in command.Children ) { NSpecContextTest contextTest = contextCommand.Test as NSpecContextTest; if( contextTest == null ) continue; var contextResult = this.RunContext( contextTest, contextCommand, assemblyContext.TestStep ); outcome = outcome.CombineWith( contextResult.Outcome ); assemblyContext.SetInterimOutcome( outcome ); } return assemblyContext.FinishStep( outcome, null ); }
private static TestResult RunTestFixture(ITestCommand testCommand, XunitTypeInfoAdapter typeInfo, TestStep parentTestStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); XunitTestClassCommand testClassCommand; try { testClassCommand = XunitTestClassCommandFactory.Make(typeInfo); } catch (Exception ex) { // Xunit can throw exceptions when making commands if the test is malformed. testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); return(testContext.FinishStep(TestOutcome.Failed, null)); } return(RunTestClassCommandAndFinishStep(testCommand, testContext, testClassCommand)); }
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor) { using (progressMonitor) { progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount); if (options.SkipTestExecution) { return(SkipAll(rootTestCommand, parentTestStep)); } else { ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep); TestStep rootStep = rootContext.TestStep; TestOutcome outcome = TestOutcome.Passed; _progressMonitor = progressMonitor; SetupRunOptions(options); SetupListeners(options); _listener.OnRunStart(); foreach (ITestCommand command in rootTestCommand.Children) { MachineAssemblyTest assemblyTest = command.Test as MachineAssemblyTest; if (assemblyTest == null) { continue; } var assemblyResult = RunAssembly(assemblyTest, command, rootStep); outcome = outcome.CombineWith(assemblyResult.Outcome); } _listener.OnRunEnd(); return(rootContext.FinishStep(outcome, null)); } } }
private TestResult RunAssembly(ITestCommand command, TestStep rootStep) { ITestContext assemblyContext = command.StartPrimaryChildStep(rootStep); TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand contextCommand in command.Children) { NSpecContextTest contextTest = contextCommand.Test as NSpecContextTest; if (contextTest == null) { continue; } var contextResult = this.RunContext(contextTest, contextCommand, assemblyContext.TestStep); outcome = outcome.CombineWith(contextResult.Outcome); assemblyContext.SetInterimOutcome(outcome); } return(assemblyContext.FinishStep(outcome, null)); }
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor) { using (progressMonitor) { progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount); if (options.SkipTestExecution) { return SkipAll(rootTestCommand, parentTestStep); } else { ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep); TestStep rootStep = rootContext.TestStep; TestOutcome outcome = TestOutcome.Passed; _progressMonitor = progressMonitor; SetupRunOptions(options); SetupListeners(options); _listener.OnRunStart(); foreach (ITestCommand command in rootTestCommand.Children) { MachineAssemblyTest assemblyTest = command.Test as MachineAssemblyTest; if( assemblyTest == null ) continue; var assemblyResult = RunAssembly(assemblyTest, command, rootStep); outcome = outcome.CombineWith( assemblyResult.Outcome); } _listener.OnRunEnd(); return rootContext.FinishStep( outcome, null); } } }
private void RunContextTest(MachineContextTest description, ITestCommand testCommand, ITestStep parentTestStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); testContext.LifecyclePhase = LifecyclePhases.SetUp; description.SetupContext(); bool passed = true; foreach (ITestCommand child in testCommand.Children) { MachineSpecificationTest specification = child.Test as MachineSpecificationTest; if (specification != null) { passed &= RunSpecificationTest(specification, child, testContext.TestStep); } } testContext.LifecyclePhase = LifecyclePhases.TearDown; description.TeardownContext(); testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null); }
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { Test test = testCommand.Test; progressMonitor.SetStatus(test.Name); // The first test should be an assembly test MSTestAssembly assemblyTest = testCommand.Test as MSTestAssembly; TestOutcome outcome; TestResult result; if (assemblyTest != null) { ITestContext assemblyContext = testCommand.StartPrimaryChildStep(parentTestStep); try { MSTestRunner runner = MSTestRunner.GetRunnerForFrameworkVersion(frameworkVersion); outcome = runner.RunSession(assemblyContext, assemblyTest, testCommand, parentTestStep, progressMonitor); } catch (Exception ex) { assemblyContext.LogWriter.Failures.WriteException(ex, "Internal Error"); outcome = TestOutcome.Error; } result = assemblyContext.FinishStep(outcome, null); } else { result = new TestResult(TestOutcome.Skipped); } progressMonitor.Worked(1); return(result); }
TestResult RunAssembly(MachineAssemblyTest assemblyTest, ITestCommand command, TestStep parentTestStep) { ITestContext assemblyContext = command.StartPrimaryChildStep(parentTestStep); AssemblyInfo assemblyInfo = new AssemblyInfo(assemblyTest.Name, assemblyTest.AssemblyFilePath); TestOutcome outcome = TestOutcome.Passed; _listener.OnAssemblyStart(assemblyInfo); assemblyTest.AssemblyContexts.Each(context => context.OnAssemblyStart()); foreach (ITestCommand contextCommand in command.Children) { MachineContextTest contextTest = contextCommand.Test as MachineContextTest; if (contextTest == null) continue; var contextResult = RunContextTest( assemblyTest, contextTest, contextCommand, assemblyContext.TestStep); outcome = outcome.CombineWith(contextResult.Outcome); assemblyContext.SetInterimOutcome(outcome); } assemblyTest.AssemblyContexts.Reverse().Each(context => context.OnAssemblyComplete()); _listener.OnAssemblyEnd(assemblyInfo); return assemblyContext.FinishStep( outcome, null); }
private TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestOutcome combinedOutCome = TestOutcome.Passed; var duration = TimeSpan.Zero; foreach (ITestCommand child in testCommand.Children) { TestResult testResult = RunTest(child, testContext.TestStep, progressMonitor); combinedOutCome = combinedOutCome.CombineWith(testResult.Outcome); duration += testResult.Duration; } return testContext.FinishStep(combinedOutCome, duration); }
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor) { // NOTE: This method has been optimized to minimize the total stack depth of the action // by inlining blocks on the critical path that had previously been factored out. using (TestController testController = testControllerProvider(testCommand.Test)) { if (testController != null) { try { using (IProgressMonitor subProgressMonitor = progressMonitor.CreateSubProgressMonitor(testCommand.TestCount)) { // Calling RunImpl directly instead of Run to minimize stack depth // because we already know the arguments are valid. return testController.RunImpl(testCommand, parentTestStep, options, subProgressMonitor); } } catch (Exception ex) { ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep); context.LogWriter.Failures.WriteException(ex, "Fatal Exception in test controller"); return context.FinishStep(TestOutcome.Error, null); } } } // Enter the scope of the test and recurse until we find a controller. progressMonitor.SetStatus(testCommand.Test.FullName); ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand monitor in testCommand.Children) { if (progressMonitor.IsCanceled) break; TestResult childResult = RunTest(monitor, testContext.TestStep, options, progressMonitor); outcome = outcome.CombineWith(childResult.Outcome).Generalize(); } if (progressMonitor.IsCanceled) outcome = TestOutcome.Canceled; TestResult result = testContext.FinishStep(outcome, null); progressMonitor.Worked(1); return result; }
public void Run() { var test = (PatternTest)testCommand.Test; TestContextCookie?parentContextCookie = null; try { if (parentContext != null) { parentContextCookie = parentContext.Enter(); } // The first time we call Run, we check whether the ApartmentState of the // Thread is correct. If it is not, then we start a new thread and reenter // with a flag set to skip initial processing. if (!reentered) { if (executor.progressMonitor.IsCanceled) { result = new TestResult(TestOutcome.Canceled); return; } if (!testCommand.AreDependenciesSatisfied()) { ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep); context.LogWriter.Warnings.WriteLine("Skipped due to an unsatisfied test dependency."); result = context.FinishStep(TestOutcome.Skipped, null); return; } executor.progressMonitor.SetStatus(test.Name); if (test.ApartmentState != ApartmentState.Unknown && Thread.CurrentThread.GetApartmentState() != test.ApartmentState) { reentered = true; ThreadTask task = new TestEnvironmentAwareThreadTask("Test Runner " + test.ApartmentState, (GallioAction)Run, executor.environmentManager); task.ApartmentState = test.ApartmentState; task.Run(null); if (!task.Result.HasValue) { throw new ModelException( String.Format("Failed to perform action in thread with overridden apartment state {0}.", test.ApartmentState), task.Result.Exception); } return; } } // Actually run the test. // Yes, this is a monstrously long method due to the inlining optimzation to minimize stack depth. using (Sandbox sandbox = parentSandbox.CreateChild()) { using (new ProcessIsolation()) { using (sandbox.StartTimer(test.TimeoutFunc())) { TestOutcome outcome; PatternTestActions testActions = test.TestActions; if (testActionsDecorator != null) { outcome = testActionsDecorator(sandbox, ref testActions); } else { outcome = TestOutcome.Passed; } if (outcome.Status == TestStatus.Passed) { PatternTestStep primaryTestStep = new PatternTestStep(test, parentTestStep); PatternTestState testState = new PatternTestState(primaryTestStep, testActions, executor.converter, executor.formatter, testCommand.IsExplicit); bool invisibleTest = true; outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer, new BeforeTestAction(testState).Run, "Before Test")); if (outcome.Status == TestStatus.Passed) { bool reusePrimaryTestStep = !testState.BindingContext.HasBindings; if (!reusePrimaryTestStep) { primaryTestStep.IsTestCase = false; } invisibleTest = false; TestContext primaryContext = TestContext.PrepareContext( testCommand.StartStep(primaryTestStep), sandbox); testState.SetInContext(primaryContext); using (primaryContext.Enter()) { primaryContext.LifecyclePhase = LifecyclePhases.Initialize; outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer, new InitializeTestAction(testState).Run, "Initialize")); } if (outcome.Status == TestStatus.Passed) { var actions = new List <RunTestDataItemAction>(); try { foreach (IDataItem bindingItem in testState.BindingContext.GetItems(!executor.options.SkipDynamicTests)) { actions.Add(new RunTestDataItemAction(executor, testCommand, testState, primaryContext, reusePrimaryTestStep, bindingItem)); } if (actions.Count == 0) { TestLog.Warnings.WriteLine("Test skipped because it is parameterized but no data was provided."); outcome = TestOutcome.Skipped; } else { if (actions.Count == 1 || !test.IsParallelizable) { foreach (var action in actions) { action.Run(); } } else { executor.scheduler.Run(GenericCollectionUtils.ConvertAllToArray <RunTestDataItemAction, GallioAction>( actions, action => action.Run)); } TestOutcome combinedOutcome = TestOutcome.Passed; foreach (var action in actions) { combinedOutcome = combinedOutcome.CombineWith(action.Outcome); } outcome = outcome.CombineWith(reusePrimaryTestStep ? combinedOutcome : combinedOutcome.Generalize()); } } catch (TestException ex) { if (ex.Outcome.Status == TestStatus.Failed) { TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName)); } else { TestLog.Warnings.WriteException(ex); } outcome = ex.Outcome; } catch (Exception ex) { TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName)); outcome = TestOutcome.Error; } } primaryContext.SetInterimOutcome(outcome); using (primaryContext.Enter()) { primaryContext.LifecyclePhase = LifecyclePhases.Dispose; outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer, new DisposeTestAction(testState).Run, "Dispose")); } result = primaryContext.FinishStep(outcome); } outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer, new AfterTestAction(testState).Run, "After Test")); if (invisibleTest) { result = PublishOutcomeFromInvisibleTest(testCommand, primaryTestStep, outcome); } } } } } } catch (Exception ex) { result = ReportTestError(testCommand, parentTestStep, ex, String.Format("An exception occurred while preparing to run test '{0}'.", test.FullName)); } finally { if (parentContextCookie.HasValue) { parentContextCookie.Value.ExitContext(); } executor.progressMonitor.SetStatus(""); executor.progressMonitor.Worked(1); } }
private static TestResult ReportTestError(ITestCommand testCommand, Model.Tree.TestStep parentTestStep, Exception ex, string message) { ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep); TestLog.Failures.WriteException(ex, message); return context.FinishStep(TestOutcome.Error, null); }
private static TestResult RunTestFixture(ITestCommand testCommand, XunitTypeInfoAdapter typeInfo, TestStep parentTestStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); XunitTestClassCommand testClassCommand; try { testClassCommand = XunitTestClassCommandFactory.Make(typeInfo); } catch (Exception ex) { // Xunit can throw exceptions when making commands if the test is malformed. testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); return testContext.FinishStep(TestOutcome.Failed, null); } return RunTestClassCommandAndFinishStep(testCommand, testContext, testClassCommand); }
private bool RunSpecificationTest(MachineSpecificationTest specification, ITestCommand testCommand, ITestStep parentTestStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); testContext.LifecyclePhase = LifecyclePhases.Execute; var result = specification.Execute(); if (result.Passed) { testContext.FinishStep(TestOutcome.Passed, null); } else { testContext.FinishStep(TestOutcome.Failed, null); } return result.Passed; }
private static bool RunTestMethod(ITestCommand testCommand, MethodInfo methodInfo, XunitTestClassCommand testClassCommand, TestStep parentTestStep) { List<XunitTestCommand> xunitTestCommands; try { xunitTestCommands = new List<XunitTestCommand>(XunitTestCommandFactory.Make(testClassCommand, XunitReflector.Wrap(methodInfo))); } catch (Exception ex) { // Xunit can throw exceptions when making commands if the test is malformed. ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); testContext.FinishStep(TestOutcome.Failed, null); return false; } if (xunitTestCommands.Count == 0) return true; if (xunitTestCommands.Count == 1) return RunTestCommands(testCommand, testClassCommand, xunitTestCommands, parentTestStep, true); // Introduce a common primary test step for theories. ITestContext primaryTestContext = testCommand.StartPrimaryChildStep(parentTestStep); bool result = RunTestCommands(testCommand, testClassCommand, xunitTestCommands, primaryTestContext.TestStep, false); primaryTestContext.FinishStep(result ? TestOutcome.Passed : TestOutcome.Failed, null); return result; }
TestResult RunTest( NSpecContextTest contextTest, NSpecExampleTest exampleTest, ITestCommand testCommand, TestStep testStep ) { ITestContext testContext = testCommand.StartPrimaryChildStep( testStep ); TestOutcome outcome = TestOutcome.Passed; if( exampleTest.Example.Pending ) { outcome = TestOutcome.Pending; testContext.AddMetadata( MetadataKeys.PendingReason, "Needs to be implemented" ); } else { contextTest.Context.Exercise(exampleTest.Example,contextTest.Context.GetInstance()); if( exampleTest.Example.Exception != null ) { TestLog.Failures.WriteException( ConvertException( exampleTest.Example.Exception ) ); TestLog.Failures.Flush(); outcome = TestOutcome.Failed; } } return testContext.FinishStep( outcome, null ); }
private ITestContext GetFixtureContext(ITestCommand fixtureCommand) { ITestContext parentContext = testContextStack.Peek(); if (parentContext.TestStep.Test != fixtureCommand.Test) { while (parentContext.TestStep.Test.Kind != CSUnitTestExplorer.AssemblyKind) { testContextStack.Pop(); TestOutcome outcome = GetFixtureOutcome(fixtureFailureCount, fixtureErrorCount); parentContext.FinishStep(outcome, null); progressMonitor.Worked(1); parentContext = testContextStack.Peek(); } parentContext = fixtureCommand.StartPrimaryChildStep(parentContext.TestStep); parentContext.LifecyclePhase = LifecyclePhases.Execute; progressMonitor.SetStatus(fixtureCommand.Test.Name); testContextStack.Push(parentContext); fixtureFailureCount = 0; fixtureErrorCount = 0; } return parentContext; }
/// <summary> /// Recursively generates single test steps for each <see cref="ITestCommand" /> and /// sets the final outcome to <see cref="TestOutcome.Skipped" />. /// </summary> /// <remarks> /// <para> /// This is useful for implementing fallback behavior when /// <see cref="TestExecutionOptions.SkipTestExecution" /> is true. /// </para> /// </remarks> /// <param name="rootTestCommand">The root test command.</param> /// <param name="parentTestStep">The parent test step.</param> /// <returns>The combined result of the test commands.</returns> protected static TestResult SkipAll(ITestCommand rootTestCommand, TestStep parentTestStep) { ITestContext context = rootTestCommand.StartPrimaryChildStep(parentTestStep); foreach (ITestCommand child in rootTestCommand.Children) SkipAll(child, context.TestStep); return context.FinishStep(TestOutcome.Skipped, null); }
TestResult RunContextTest(MachineAssemblyTest assemblyTest, MachineContextTest contextTest, ITestCommand command, TestStep parentTestStep) { ITestContext testContext = command.StartPrimaryChildStep(parentTestStep); GallioRunListener listener = new GallioRunListener(_listener, _progressMonitor, testContext, command.Children); IContextRunner runner = ContextRunnerFactory.GetContextRunnerFor(contextTest.Context); runner.Run(contextTest.Context, listener, _options, assemblyTest.GlobalCleanup, assemblyTest.SpecificationSupplements); return testContext.FinishStep(listener.Outcome, null); }
private TestResult RunTestStep(ITestCommand testCommand, TestInfoData testStepInfo, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestStepResult testStepResult = repository.RunTest(testStepInfo); reporter.Run(testContext, testStepInfo, testStepResult); WriteToTestLog(testContext, testStepResult); testContext.AddAssertCount(testStepResult.AssertCount); progressMonitor.Worked(1); return testContext.FinishStep(testStepResult.TestOutcome, testStepResult.Duration); }
private void HandleAssemblyStart() { ITestContext assemblyTestContext = assemblyTestCommand.StartPrimaryChildStep(topTestStep); activeTestContexts.Add(assemblyTestCommand, assemblyTestContext); }