private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
// NOTE: This method has been optimized to minimize the total stack depth of the action
// by inlining blocks on the critical path that had previously been factored out.
using (TestController testController = testControllerProvider(testCommand.Test))
{
if (testController != null)
{
try
{
using (IProgressMonitor subProgressMonitor = progressMonitor.CreateSubProgressMonitor(testCommand.TestCount))
{
// Calling RunImpl directly instead of Run to minimize stack depth
// because we already know the arguments are valid.
return(testController.RunImpl(testCommand, parentTestStep, options, subProgressMonitor));
}
}
catch (Exception ex)
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
context.LogWriter.Failures.WriteException(ex, "Fatal Exception in test controller");
return(context.FinishStep(TestOutcome.Error, null));
}
}
}
// Enter the scope of the test and recurse until we find a controller.
progressMonitor.SetStatus(testCommand.Test.FullName);
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand monitor in testCommand.Children)
{
if (progressMonitor.IsCanceled)
{
break;
}
TestResult childResult = RunTest(monitor, testContext.TestStep, options, progressMonitor);
outcome = outcome.CombineWith(childResult.Outcome).Generalize();
}
if (progressMonitor.IsCanceled)
{
outcome = TestOutcome.Canceled;
}
TestResult result = testContext.FinishStep(outcome, null);
progressMonitor.Worked(1);
return(result);
}
protected override TestResult RunImpl( ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor )
{
using(progressMonitor.BeginTask( "Verifying Specifications", rootTestCommand.TestCount ) )
{
if( options.SkipTestExecution )
{
return SkipAll( rootTestCommand, parentTestStep );
}
else
{
ITestContext rootContext = rootTestCommand.StartPrimaryChildStep( parentTestStep );
TestStep rootStep = rootContext.TestStep;
TestOutcome outcome = TestOutcome.Passed;
foreach( ITestCommand command in rootTestCommand.Children )
{
NSpecAssemblyTest assemblyTest = command.Test as NSpecAssemblyTest;
if( assemblyTest == null )
continue;
var assemblyResult = this.RunAssembly( command, rootStep );
outcome = outcome.CombineWith( assemblyResult.Outcome );
}
return rootContext.FinishStep( outcome, null );
}
}
}
private static TestResult ReportTestError(ITestCommand testCommand, Model.Tree.TestStep parentTestStep, Exception ex, string message)
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
TestLog.Failures.WriteException(ex, message);
return(context.FinishStep(TestOutcome.Error, null));
}
TestResult RunAssembly(MachineAssemblyTest assemblyTest, ITestCommand command, TestStep parentTestStep)
{
ITestContext assemblyContext = command.StartPrimaryChildStep(parentTestStep);
AssemblyInfo assemblyInfo = new AssemblyInfo(assemblyTest.Name, assemblyTest.AssemblyFilePath);
TestOutcome outcome = TestOutcome.Passed;
_listener.OnAssemblyStart(assemblyInfo);
assemblyTest.AssemblyContexts.Each(context => context.OnAssemblyStart());
foreach (ITestCommand contextCommand in command.Children)
{
MachineContextTest contextTest = contextCommand.Test as MachineContextTest;
if (contextTest == null)
{
continue;
}
var contextResult = RunContextTest(assemblyTest, contextTest, contextCommand, assemblyContext.TestStep);
outcome = outcome.CombineWith(contextResult.Outcome);
assemblyContext.SetInterimOutcome(outcome);
}
assemblyTest.AssemblyContexts.Reverse().Each(context => context.OnAssemblyComplete());
_listener.OnAssemblyEnd(assemblyInfo);
return(assemblyContext.FinishStep(outcome, null));
}
TestResult RunTest(NSpecContextTest contextTest, NSpecExampleTest exampleTest,
ITestCommand testCommand, TestStep testStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(testStep);
TestOutcome outcome = TestOutcome.Passed;
if (exampleTest.Example.Pending)
{
outcome = TestOutcome.Pending;
testContext.AddMetadata(MetadataKeys.PendingReason, "Needs to be implemented");
}
else
{
contextTest.Context.Exercise(exampleTest.Example, contextTest.Context.GetInstance());
if (exampleTest.Example.Exception != null)
{
TestLog.Failures.WriteException(ConvertException(exampleTest.Example.Exception));
TestLog.Failures.Flush();
outcome = TestOutcome.Failed;
}
}
return(testContext.FinishStep(outcome, null));
}
private ITestContext GetFixtureContext(ITestCommand fixtureCommand)
{
ITestContext parentContext = testContextStack.Peek();
if (parentContext.TestStep.Test != fixtureCommand.Test)
{
while (parentContext.TestStep.Test.Kind != CSUnitTestExplorer.AssemblyKind)
{
testContextStack.Pop();
TestOutcome outcome = GetFixtureOutcome(fixtureFailureCount, fixtureErrorCount);
parentContext.FinishStep(outcome, null);
progressMonitor.Worked(1);
parentContext = testContextStack.Peek();
}
parentContext = fixtureCommand.StartPrimaryChildStep(parentContext.TestStep);
parentContext.LifecyclePhase = LifecyclePhases.Execute;
progressMonitor.SetStatus(fixtureCommand.Test.Name);
testContextStack.Push(parentContext);
fixtureFailureCount = 0;
fixtureErrorCount = 0;
}
return(parentContext);
}
private TestResult RunContext(NSpecContextTest contextTest, ITestCommand command, TestStep testStep)
{
ITestContext testContext = command.StartPrimaryChildStep(testStep);
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand testCommand in command.Children)
{
NSpecExampleTest exampleTest = testCommand.Test as NSpecExampleTest;
if (exampleTest == null)
{
continue;
}
outcome = outcome.CombineWith(this.RunTest(contextTest, exampleTest, testCommand, testContext.TestStep).Outcome);
}
foreach (ITestCommand testCommand in command.Children)
{
NSpecContextTest contextTestChild = testCommand.Test as NSpecContextTest;
if (contextTestChild == null)
{
continue;
}
outcome = outcome.CombineWith(this.RunContext(contextTestChild, testCommand, testContext.TestStep).Outcome);
}
return(testContext.FinishStep(outcome, null));
}
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
Test test = testCommand.Test;
progressMonitor.SetStatus(test.Name);
// The first test should be an assembly test
MSTestAssembly assemblyTest = testCommand.Test as MSTestAssembly;
TestOutcome outcome;
TestResult result;
if (assemblyTest != null)
{
ITestContext assemblyContext = testCommand.StartPrimaryChildStep(parentTestStep);
try
{
MSTestRunner runner = MSTestRunner.GetRunnerForFrameworkVersion(frameworkVersion);
outcome = runner.RunSession(assemblyContext, assemblyTest,
testCommand, parentTestStep, progressMonitor);
}
catch (Exception ex)
{
assemblyContext.LogWriter.Failures.WriteException(ex, "Internal Error");
outcome = TestOutcome.Error;
}
result = assemblyContext.FinishStep(outcome, null);
}
else
{
result = new TestResult(TestOutcome.Skipped);
}
progressMonitor.Worked(1);
return result;
}
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount))
{
if (options.SkipTestExecution)
{
return(SkipAll(rootTestCommand, parentTestStep));
}
else
{
ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep);
TestStep rootStep = rootContext.TestStep;
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand command in rootTestCommand.Children)
{
NSpecAssemblyTest assemblyTest = command.Test as NSpecAssemblyTest;
if (assemblyTest == null)
{
continue;
}
var assemblyResult = this.RunAssembly(command, rootStep);
outcome = outcome.CombineWith(assemblyResult.Outcome);
}
return(rootContext.FinishStep(outcome, null));
}
}
}
private TestResult RunContext( NSpecContextTest contextTest, ITestCommand command, TestStep testStep )
{
ITestContext testContext = command.StartPrimaryChildStep( testStep );
TestOutcome outcome = TestOutcome.Passed;
foreach( ITestCommand testCommand in command.Children )
{
NSpecExampleTest exampleTest = testCommand.Test as NSpecExampleTest;
if( exampleTest == null )
{
continue;
}
outcome = outcome.CombineWith( this.RunTest( contextTest, exampleTest, testCommand, testContext.TestStep ).Outcome );
}
foreach( ITestCommand testCommand in command.Children )
{
NSpecContextTest contextTestChild = testCommand.Test as NSpecContextTest;
if( contextTestChild == null )
{
continue;
}
outcome = outcome.CombineWith( this.RunContext( contextTestChild, testCommand, testContext.TestStep ).Outcome );
}
return testContext.FinishStep( outcome, null );
}
private static TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
bool passed = true;
foreach (ITestCommand child in testCommand.Children)
passed &= RunTest(child, testContext.TestStep, progressMonitor).Outcome.Status == TestStatus.Passed;
return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
}
/// <summary>
/// Recursively generates single test steps for each <see cref="ITestCommand" /> and
/// sets the final outcome to <see cref="TestOutcome.Skipped" />.
/// </summary>
/// <remarks>
/// <para>
/// This is useful for implementing fallback behavior when
/// <see cref="TestExecutionOptions.SkipTestExecution" /> is true.
/// </para>
/// </remarks>
/// <param name="rootTestCommand">The root test command.</param>
/// <param name="parentTestStep">The parent test step.</param>
/// <returns>The combined result of the test commands.</returns>
protected static TestResult SkipAll(ITestCommand rootTestCommand, TestStep parentTestStep)
{
ITestContext context = rootTestCommand.StartPrimaryChildStep(parentTestStep);
foreach (ITestCommand child in rootTestCommand.Children)
{
SkipAll(child, context.TestStep);
}
return(context.FinishStep(TestOutcome.Skipped, null));
}
private TestResult RunTestStep(ITestCommand testCommand, TestInfoData testStepInfo, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestStepResult testStepResult = repository.RunTest(testStepInfo);
reporter.Run(testContext, testStepInfo, testStepResult);
WriteToTestLog(testContext, testStepResult);
testContext.AddAssertCount(testStepResult.AssertCount);
progressMonitor.Worked(1);
return(testContext.FinishStep(testStepResult.TestOutcome, testStepResult.Duration));
}
TestResult RunContextTest(MachineAssemblyTest assemblyTest, MachineContextTest contextTest, ITestCommand command, TestStep parentTestStep)
{
ITestContext testContext = command.StartPrimaryChildStep(parentTestStep);
GallioRunListener listener = new GallioRunListener(_listener, _progressMonitor, testContext, command.Children);
IContextRunner runner = ContextRunnerFactory.GetContextRunnerFor(contextTest.Context);
runner.Run(contextTest.Context, listener, _options, assemblyTest.GlobalCleanup, assemblyTest.SpecificationSupplements);
return(testContext.FinishStep(listener.Outcome, null));
}
public override void OnSpecificationStart(SpecificationInfo specification)
{
_listener.OnSpecificationStart(specification);
ITestCommand specCommand = _commandsBySpec[specification.Name];
ITestContext specContext = specCommand.StartPrimaryChildStep(_testContext.TestStep);
_contextsBySpec.Add(specification.Name, specContext);
specContext.LifecyclePhase = LifecyclePhases.Starting;
_progressMonitor.SetStatus("» " + specification.Name);
}
private static TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
bool passed = true;
foreach (ITestCommand child in testCommand.Children)
{
passed &= RunTest(child, testContext.TestStep, progressMonitor).Outcome.Status == TestStatus.Passed;
}
return(testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null));
}
private static bool RunTestMethod(ITestCommand testCommand, MethodInfo methodInfo, XunitTestClassCommand testClassCommand,
TestStep parentTestStep)
{
List <XunitTestCommand> xunitTestCommands;
try
{
xunitTestCommands = new List <XunitTestCommand>(XunitTestCommandFactory.Make(testClassCommand,
XunitReflector.Wrap(methodInfo)));
}
catch (Exception ex)
{
// Xunit can throw exceptions when making commands if the test is malformed.
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
testContext.FinishStep(TestOutcome.Failed, null);
return(false);
}
if (xunitTestCommands.Count == 0)
{
return(true);
}
if (xunitTestCommands.Count == 1)
{
return(RunTestCommands(testCommand, testClassCommand, xunitTestCommands, parentTestStep, true));
}
// Introduce a common primary test step for theories.
ITestContext primaryTestContext = testCommand.StartPrimaryChildStep(parentTestStep);
bool result = RunTestCommands(testCommand, testClassCommand, xunitTestCommands, primaryTestContext.TestStep, false);
primaryTestContext.FinishStep(result ? TestOutcome.Passed : TestOutcome.Failed, null);
return(result);
}
private TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestOutcome combinedOutCome = TestOutcome.Passed;
var duration = TimeSpan.Zero;
foreach (ITestCommand child in testCommand.Children)
{
TestResult testResult = RunTest(child, testContext.TestStep, progressMonitor);
combinedOutCome = combinedOutCome.CombineWith(testResult.Outcome);
duration += testResult.Duration;
}
return(testContext.FinishStep(combinedOutCome, duration));
}
private static TestResult RunTestFixture(ITestCommand testCommand, ConcordionTest concordionTest, TestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
// The magic happens here!
var concordion = new ConcordionBuilder()
.WithSource(concordionTest.Source)
.WithTarget(concordionTest.Target)
.WithSpecificationProcessingListener(new GallioResultRenderer())
.Build();
ConstructorInfo constructor = concordionTest.FixtureType.GetConstructor(Type.EmptyTypes);
var fixture=constructor.Invoke(new object[]{});
var summary = concordion.Process(concordionTest.Resource, fixture);
bool passed = !(summary.HasFailures || summary.HasExceptions);
testContext.AddAssertCount((int)summary.SuccessCount + (int)summary.FailureCount);
return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
}
private static bool RunTestFixture(ITestCommand testCommand, ConcordionTest concordionTest, ITestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
// The magic happens here!
var concordion = new ConcordionBuilder()
.WithSource(concordionTest.Source)
.WithTarget(concordionTest.Target)
.WithSpecificationListener(new GallioResultRenderer())
.Build();
var summary = concordion.Process(concordionTest.Resource, concordionTest.Fixture);
bool passed = !(summary.HasFailures || summary.HasExceptions);
testContext.AddAssertCount((int)summary.SuccessCount + (int)summary.FailureCount);
testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
return(passed);
}
private TestResult RunAssembly( ITestCommand command, TestStep rootStep )
{
ITestContext assemblyContext = command.StartPrimaryChildStep( rootStep );
TestOutcome outcome = TestOutcome.Passed;
foreach( ITestCommand contextCommand in command.Children )
{
NSpecContextTest contextTest = contextCommand.Test as NSpecContextTest;
if( contextTest == null )
continue;
var contextResult = this.RunContext( contextTest, contextCommand, assemblyContext.TestStep );
outcome = outcome.CombineWith( contextResult.Outcome );
assemblyContext.SetInterimOutcome( outcome );
}
return assemblyContext.FinishStep( outcome, null );
}
private static TestResult RunTestFixture(ITestCommand testCommand, XunitTypeInfoAdapter typeInfo,
TestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
XunitTestClassCommand testClassCommand;
try
{
testClassCommand = XunitTestClassCommandFactory.Make(typeInfo);
}
catch (Exception ex)
{
// Xunit can throw exceptions when making commands if the test is malformed.
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
return(testContext.FinishStep(TestOutcome.Failed, null));
}
return(RunTestClassCommandAndFinishStep(testCommand, testContext, testClassCommand));
}
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor)
{
progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount);
if (options.SkipTestExecution)
{
return(SkipAll(rootTestCommand, parentTestStep));
}
else
{
ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep);
TestStep rootStep = rootContext.TestStep;
TestOutcome outcome = TestOutcome.Passed;
_progressMonitor = progressMonitor;
SetupRunOptions(options);
SetupListeners(options);
_listener.OnRunStart();
foreach (ITestCommand command in rootTestCommand.Children)
{
MachineAssemblyTest assemblyTest = command.Test as MachineAssemblyTest;
if (assemblyTest == null)
{
continue;
}
var assemblyResult = RunAssembly(assemblyTest, command, rootStep);
outcome = outcome.CombineWith(assemblyResult.Outcome);
}
_listener.OnRunEnd();
return(rootContext.FinishStep(outcome, null));
}
}
}
private TestResult RunAssembly(ITestCommand command, TestStep rootStep)
{
ITestContext assemblyContext = command.StartPrimaryChildStep(rootStep);
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand contextCommand in command.Children)
{
NSpecContextTest contextTest = contextCommand.Test as NSpecContextTest;
if (contextTest == null)
{
continue;
}
var contextResult = this.RunContext(contextTest, contextCommand, assemblyContext.TestStep);
outcome = outcome.CombineWith(contextResult.Outcome);
assemblyContext.SetInterimOutcome(outcome);
}
return(assemblyContext.FinishStep(outcome, null));
}
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor)
{
progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount);
if (options.SkipTestExecution)
{
return SkipAll(rootTestCommand, parentTestStep);
}
else
{
ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep);
TestStep rootStep = rootContext.TestStep;
TestOutcome outcome = TestOutcome.Passed;
_progressMonitor = progressMonitor;
SetupRunOptions(options);
SetupListeners(options);
_listener.OnRunStart();
foreach (ITestCommand command in rootTestCommand.Children)
{
MachineAssemblyTest assemblyTest = command.Test as MachineAssemblyTest;
if( assemblyTest == null )
continue;
var assemblyResult = RunAssembly(assemblyTest, command, rootStep);
outcome = outcome.CombineWith( assemblyResult.Outcome);
}
_listener.OnRunEnd();
return rootContext.FinishStep( outcome, null);
}
}
}
private void RunContextTest(MachineContextTest description, ITestCommand testCommand, ITestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
testContext.LifecyclePhase = LifecyclePhases.SetUp;
description.SetupContext();
bool passed = true;
foreach (ITestCommand child in testCommand.Children)
{
MachineSpecificationTest specification = child.Test as MachineSpecificationTest;
if (specification != null)
{
passed &= RunSpecificationTest(specification, child, testContext.TestStep);
}
}
testContext.LifecyclePhase = LifecyclePhases.TearDown;
description.TeardownContext();
testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
}
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
Test test = testCommand.Test;
progressMonitor.SetStatus(test.Name);
// The first test should be an assembly test
MSTestAssembly assemblyTest = testCommand.Test as MSTestAssembly;
TestOutcome outcome;
TestResult result;
if (assemblyTest != null)
{
ITestContext assemblyContext = testCommand.StartPrimaryChildStep(parentTestStep);
try
{
MSTestRunner runner = MSTestRunner.GetRunnerForFrameworkVersion(frameworkVersion);
outcome = runner.RunSession(assemblyContext, assemblyTest,
testCommand, parentTestStep, progressMonitor);
}
catch (Exception ex)
{
assemblyContext.LogWriter.Failures.WriteException(ex, "Internal Error");
outcome = TestOutcome.Error;
}
result = assemblyContext.FinishStep(outcome, null);
}
else
{
result = new TestResult(TestOutcome.Skipped);
}
progressMonitor.Worked(1);
return(result);
}
TestResult RunAssembly(MachineAssemblyTest assemblyTest, ITestCommand command, TestStep parentTestStep)
{
ITestContext assemblyContext = command.StartPrimaryChildStep(parentTestStep);
AssemblyInfo assemblyInfo = new AssemblyInfo(assemblyTest.Name, assemblyTest.AssemblyFilePath);
TestOutcome outcome = TestOutcome.Passed;
_listener.OnAssemblyStart(assemblyInfo);
assemblyTest.AssemblyContexts.Each(context => context.OnAssemblyStart());
foreach (ITestCommand contextCommand in command.Children)
{
MachineContextTest contextTest = contextCommand.Test as MachineContextTest;
if (contextTest == null)
continue;
var contextResult = RunContextTest( assemblyTest, contextTest, contextCommand, assemblyContext.TestStep);
outcome = outcome.CombineWith(contextResult.Outcome);
assemblyContext.SetInterimOutcome(outcome);
}
assemblyTest.AssemblyContexts.Reverse().Each(context => context.OnAssemblyComplete());
_listener.OnAssemblyEnd(assemblyInfo);
return assemblyContext.FinishStep( outcome, null);
}
private TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestOutcome combinedOutCome = TestOutcome.Passed;
var duration = TimeSpan.Zero;
foreach (ITestCommand child in testCommand.Children)
{
TestResult testResult = RunTest(child, testContext.TestStep, progressMonitor);
combinedOutCome = combinedOutCome.CombineWith(testResult.Outcome);
duration += testResult.Duration;
}
return testContext.FinishStep(combinedOutCome, duration);
}
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
// NOTE: This method has been optimized to minimize the total stack depth of the action
// by inlining blocks on the critical path that had previously been factored out.
using (TestController testController = testControllerProvider(testCommand.Test))
{
if (testController != null)
{
try
{
using (IProgressMonitor subProgressMonitor = progressMonitor.CreateSubProgressMonitor(testCommand.TestCount))
{
// Calling RunImpl directly instead of Run to minimize stack depth
// because we already know the arguments are valid.
return testController.RunImpl(testCommand, parentTestStep, options, subProgressMonitor);
}
}
catch (Exception ex)
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
context.LogWriter.Failures.WriteException(ex, "Fatal Exception in test controller");
return context.FinishStep(TestOutcome.Error, null);
}
}
}
// Enter the scope of the test and recurse until we find a controller.
progressMonitor.SetStatus(testCommand.Test.FullName);
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand monitor in testCommand.Children)
{
if (progressMonitor.IsCanceled)
break;
TestResult childResult = RunTest(monitor, testContext.TestStep, options, progressMonitor);
outcome = outcome.CombineWith(childResult.Outcome).Generalize();
}
if (progressMonitor.IsCanceled)
outcome = TestOutcome.Canceled;
TestResult result = testContext.FinishStep(outcome, null);
progressMonitor.Worked(1);
return result;
}
public void Run()
{
var test = (PatternTest)testCommand.Test;
TestContextCookie?parentContextCookie = null;
try
{
if (parentContext != null)
{
parentContextCookie = parentContext.Enter();
}
// The first time we call Run, we check whether the ApartmentState of the
// Thread is correct. If it is not, then we start a new thread and reenter
// with a flag set to skip initial processing.
if (!reentered)
{
if (executor.progressMonitor.IsCanceled)
{
result = new TestResult(TestOutcome.Canceled);
return;
}
if (!testCommand.AreDependenciesSatisfied())
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
context.LogWriter.Warnings.WriteLine("Skipped due to an unsatisfied test dependency.");
result = context.FinishStep(TestOutcome.Skipped, null);
return;
}
executor.progressMonitor.SetStatus(test.Name);
if (test.ApartmentState != ApartmentState.Unknown &&
Thread.CurrentThread.GetApartmentState() != test.ApartmentState)
{
reentered = true;
ThreadTask task = new TestEnvironmentAwareThreadTask("Test Runner " + test.ApartmentState,
(GallioAction)Run, executor.environmentManager);
task.ApartmentState = test.ApartmentState;
task.Run(null);
if (!task.Result.HasValue)
{
throw new ModelException(
String.Format("Failed to perform action in thread with overridden apartment state {0}.",
test.ApartmentState), task.Result.Exception);
}
return;
}
}
// Actually run the test.
// Yes, this is a monstrously long method due to the inlining optimzation to minimize stack depth.
using (Sandbox sandbox = parentSandbox.CreateChild())
{
using (new ProcessIsolation())
{
using (sandbox.StartTimer(test.TimeoutFunc()))
{
TestOutcome outcome;
PatternTestActions testActions = test.TestActions;
if (testActionsDecorator != null)
{
outcome = testActionsDecorator(sandbox, ref testActions);
}
else
{
outcome = TestOutcome.Passed;
}
if (outcome.Status == TestStatus.Passed)
{
PatternTestStep primaryTestStep = new PatternTestStep(test, parentTestStep);
PatternTestState testState = new PatternTestState(primaryTestStep, testActions,
executor.converter, executor.formatter, testCommand.IsExplicit);
bool invisibleTest = true;
outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer,
new BeforeTestAction(testState).Run, "Before Test"));
if (outcome.Status == TestStatus.Passed)
{
bool reusePrimaryTestStep = !testState.BindingContext.HasBindings;
if (!reusePrimaryTestStep)
{
primaryTestStep.IsTestCase = false;
}
invisibleTest = false;
TestContext primaryContext = TestContext.PrepareContext(
testCommand.StartStep(primaryTestStep), sandbox);
testState.SetInContext(primaryContext);
using (primaryContext.Enter())
{
primaryContext.LifecyclePhase = LifecyclePhases.Initialize;
outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer,
new InitializeTestAction(testState).Run, "Initialize"));
}
if (outcome.Status == TestStatus.Passed)
{
var actions = new List <RunTestDataItemAction>();
try
{
foreach (IDataItem bindingItem in testState.BindingContext.GetItems(!executor.options.SkipDynamicTests))
{
actions.Add(new RunTestDataItemAction(executor, testCommand, testState, primaryContext,
reusePrimaryTestStep, bindingItem));
}
if (actions.Count == 0)
{
TestLog.Warnings.WriteLine("Test skipped because it is parameterized but no data was provided.");
outcome = TestOutcome.Skipped;
}
else
{
if (actions.Count == 1 || !test.IsParallelizable)
{
foreach (var action in actions)
{
action.Run();
}
}
else
{
executor.scheduler.Run(GenericCollectionUtils.ConvertAllToArray <RunTestDataItemAction, GallioAction>(
actions, action => action.Run));
}
TestOutcome combinedOutcome = TestOutcome.Passed;
foreach (var action in actions)
{
combinedOutcome = combinedOutcome.CombineWith(action.Outcome);
}
outcome = outcome.CombineWith(reusePrimaryTestStep ? combinedOutcome : combinedOutcome.Generalize());
}
}
catch (TestException ex)
{
if (ex.Outcome.Status == TestStatus.Failed)
{
TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName));
}
else
{
TestLog.Warnings.WriteException(ex);
}
outcome = ex.Outcome;
}
catch (Exception ex)
{
TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName));
outcome = TestOutcome.Error;
}
}
primaryContext.SetInterimOutcome(outcome);
using (primaryContext.Enter())
{
primaryContext.LifecyclePhase = LifecyclePhases.Dispose;
outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer,
new DisposeTestAction(testState).Run, "Dispose"));
}
result = primaryContext.FinishStep(outcome);
}
outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer,
new AfterTestAction(testState).Run, "After Test"));
if (invisibleTest)
{
result = PublishOutcomeFromInvisibleTest(testCommand, primaryTestStep, outcome);
}
}
}
}
}
}
catch (Exception ex)
{
result = ReportTestError(testCommand, parentTestStep, ex, String.Format("An exception occurred while preparing to run test '{0}'.", test.FullName));
}
finally
{
if (parentContextCookie.HasValue)
{
parentContextCookie.Value.ExitContext();
}
executor.progressMonitor.SetStatus("");
executor.progressMonitor.Worked(1);
}
}
private static TestResult ReportTestError(ITestCommand testCommand, Model.Tree.TestStep parentTestStep, Exception ex, string message)
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
TestLog.Failures.WriteException(ex, message);
return context.FinishStep(TestOutcome.Error, null);
}
private static TestResult RunTestFixture(ITestCommand testCommand, XunitTypeInfoAdapter typeInfo,
TestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
XunitTestClassCommand testClassCommand;
try
{
testClassCommand = XunitTestClassCommandFactory.Make(typeInfo);
}
catch (Exception ex)
{
// Xunit can throw exceptions when making commands if the test is malformed.
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
return testContext.FinishStep(TestOutcome.Failed, null);
}
return RunTestClassCommandAndFinishStep(testCommand, testContext, testClassCommand);
}
private bool RunSpecificationTest(MachineSpecificationTest specification, ITestCommand testCommand, ITestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
testContext.LifecyclePhase = LifecyclePhases.Execute;
var result = specification.Execute();
if (result.Passed)
{
testContext.FinishStep(TestOutcome.Passed, null);
}
else
{
testContext.FinishStep(TestOutcome.Failed, null);
}
return result.Passed;
}
private static bool RunTestMethod(ITestCommand testCommand, MethodInfo methodInfo, XunitTestClassCommand testClassCommand,
TestStep parentTestStep)
{
List<XunitTestCommand> xunitTestCommands;
try
{
xunitTestCommands = new List<XunitTestCommand>(XunitTestCommandFactory.Make(testClassCommand,
XunitReflector.Wrap(methodInfo)));
}
catch (Exception ex)
{
// Xunit can throw exceptions when making commands if the test is malformed.
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
testContext.FinishStep(TestOutcome.Failed, null);
return false;
}
if (xunitTestCommands.Count == 0)
return true;
if (xunitTestCommands.Count == 1)
return RunTestCommands(testCommand, testClassCommand, xunitTestCommands, parentTestStep, true);
// Introduce a common primary test step for theories.
ITestContext primaryTestContext = testCommand.StartPrimaryChildStep(parentTestStep);
bool result = RunTestCommands(testCommand, testClassCommand, xunitTestCommands, primaryTestContext.TestStep, false);
primaryTestContext.FinishStep(result ? TestOutcome.Passed : TestOutcome.Failed, null);
return result;
}
TestResult RunTest( NSpecContextTest contextTest, NSpecExampleTest exampleTest,
ITestCommand testCommand, TestStep testStep )
{
ITestContext testContext = testCommand.StartPrimaryChildStep( testStep );
TestOutcome outcome = TestOutcome.Passed;
if( exampleTest.Example.Pending )
{
outcome = TestOutcome.Pending;
testContext.AddMetadata( MetadataKeys.PendingReason, "Needs to be implemented" );
}
else
{
contextTest.Context.Exercise(exampleTest.Example,contextTest.Context.GetInstance());
if( exampleTest.Example.Exception != null )
{
TestLog.Failures.WriteException( ConvertException( exampleTest.Example.Exception ) );
TestLog.Failures.Flush();
outcome = TestOutcome.Failed;
}
}
return testContext.FinishStep( outcome, null );
}
private ITestContext GetFixtureContext(ITestCommand fixtureCommand)
{
ITestContext parentContext = testContextStack.Peek();
if (parentContext.TestStep.Test != fixtureCommand.Test)
{
while (parentContext.TestStep.Test.Kind != CSUnitTestExplorer.AssemblyKind)
{
testContextStack.Pop();
TestOutcome outcome = GetFixtureOutcome(fixtureFailureCount, fixtureErrorCount);
parentContext.FinishStep(outcome, null);
progressMonitor.Worked(1);
parentContext = testContextStack.Peek();
}
parentContext = fixtureCommand.StartPrimaryChildStep(parentContext.TestStep);
parentContext.LifecyclePhase = LifecyclePhases.Execute;
progressMonitor.SetStatus(fixtureCommand.Test.Name);
testContextStack.Push(parentContext);
fixtureFailureCount = 0;
fixtureErrorCount = 0;
}
return parentContext;
}
/// <summary>
/// Recursively generates single test steps for each <see cref="ITestCommand" /> and
/// sets the final outcome to <see cref="TestOutcome.Skipped" />.
/// </summary>
/// <remarks>
/// <para>
/// This is useful for implementing fallback behavior when
/// <see cref="TestExecutionOptions.SkipTestExecution" /> is true.
/// </para>
/// </remarks>
/// <param name="rootTestCommand">The root test command.</param>
/// <param name="parentTestStep">The parent test step.</param>
/// <returns>The combined result of the test commands.</returns>
protected static TestResult SkipAll(ITestCommand rootTestCommand, TestStep parentTestStep)
{
ITestContext context = rootTestCommand.StartPrimaryChildStep(parentTestStep);
foreach (ITestCommand child in rootTestCommand.Children)
SkipAll(child, context.TestStep);
return context.FinishStep(TestOutcome.Skipped, null);
}
TestResult RunContextTest(MachineAssemblyTest assemblyTest, MachineContextTest contextTest, ITestCommand command, TestStep parentTestStep)
{
ITestContext testContext = command.StartPrimaryChildStep(parentTestStep);
GallioRunListener listener = new GallioRunListener(_listener, _progressMonitor, testContext, command.Children);
IContextRunner runner = ContextRunnerFactory.GetContextRunnerFor(contextTest.Context);
runner.Run(contextTest.Context, listener, _options, assemblyTest.GlobalCleanup, assemblyTest.SpecificationSupplements);
return testContext.FinishStep(listener.Outcome, null);
}
private TestResult RunTestStep(ITestCommand testCommand, TestInfoData testStepInfo, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestStepResult testStepResult = repository.RunTest(testStepInfo);
reporter.Run(testContext, testStepInfo, testStepResult);
WriteToTestLog(testContext, testStepResult);
testContext.AddAssertCount(testStepResult.AssertCount);
progressMonitor.Worked(1);
return testContext.FinishStep(testStepResult.TestOutcome, testStepResult.Duration);
}
private void HandleAssemblyStart()
{
ITestContext assemblyTestContext = assemblyTestCommand.StartPrimaryChildStep(topTestStep);
activeTestContexts.Add(assemblyTestCommand, assemblyTestContext);
}
