/// <summary>
/// Creates an initial test state object.
/// </summary>
/// <param name="primaryTestStep">The primary test step.</param>
/// <param name="testActions">The test actions.</param>
/// <param name="converter">The converter for data binding.</param>
/// <param name="formatter">The formatter for data binding.</param>
/// <param name="isExplicit">True if the test was selected explicitly.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="primaryTestStep"/>,
/// <paramref name="testActions"/>, <paramref name="converter"/>
/// or <paramref name="formatter"/> is null.</exception>
internal PatternTestState(PatternTestStep primaryTestStep,
PatternTestActions testActions, IConverter converter, IFormatter formatter, bool isExplicit)
{
if (primaryTestStep == null)
{
throw new ArgumentNullException("primaryTestStep");
}
if (testActions == null)
{
throw new ArgumentNullException("testActions");
}
if (converter == null)
{
throw new ArgumentNullException("converter");
}
if (formatter == null)
{
throw new ArgumentNullException("formatter");
}
this.primaryTestStep = primaryTestStep;
this.testActions = testActions;
this.converter = converter;
this.formatter = formatter;
this.isExplicit = isExplicit;
bindingContext = new DataBindingContext(converter);
testParameterDataAccessors = new Dictionary <PatternTestParameter, IDataAccessor>();
data = new UserDataCollection();
}
/// <summary>
/// Creates an initial test instance state object.
/// </summary>
/// <param name="testStep">The test step used to execute the test instance.</param>
/// <param name="testInstanceActions">The test instance actions.</param>
/// <param name="testState">The test state.</param>
/// <param name="bindingItem">The data item.</param>
/// <param name="body">The body of the test instance.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="testStep"/>,
/// <paramref name="testInstanceActions"/> or <paramref name="testState"/> or <paramref name="bindingItem"/> is null.</exception>
/// <exception cref="ArgumentException">Thrown if <paramref name="testState"/> belongs to a
/// different test from the <paramref name="testStep"/>.</exception>
internal PatternTestInstanceState(PatternTestStep testStep,
PatternTestInstanceActions testInstanceActions,
PatternTestState testState, IDataItem bindingItem, TestAction body)
{
if (testStep == null)
{
throw new ArgumentNullException("testStep");
}
if (testInstanceActions == null)
{
throw new ArgumentNullException("testInstanceActions");
}
if (testState == null)
{
throw new ArgumentNullException("testState");
}
if (testStep.Test != testState.Test)
{
throw new ArgumentException("The test state belongs to a different test from the test step.", "testState");
}
if (bindingItem == null)
{
throw new ArgumentNullException("bindingItem");
}
if (body == null)
{
throw new ArgumentNullException("body");
}
this.testStep = testStep;
this.testInstanceActions = testInstanceActions;
this.testState = testState;
this.bindingItem = bindingItem;
this.body = body;
testParameterValues = new Dictionary <PatternTestParameter, object>();
slotValues = new Dictionary <ISlotInfo, object>();
data = new UserDataCollection();
nameBase = testStep.Name;
nameSuffixes = string.Empty;
}
/// <summary>
/// Creates an initial test state object.
/// </summary>
/// <param name="primaryTestStep">The primary test step.</param>
/// <param name="testActions">The test actions.</param>
/// <param name="converter">The converter for data binding.</param>
/// <param name="formatter">The formatter for data binding.</param>
/// <param name="isExplicit">True if the test was selected explicitly.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="primaryTestStep"/>,
/// <paramref name="testActions"/>, <paramref name="converter"/>
/// or <paramref name="formatter"/> is null.</exception>
internal PatternTestState(PatternTestStep primaryTestStep,
PatternTestActions testActions, IConverter converter, IFormatter formatter, bool isExplicit)
{
if (primaryTestStep == null)
throw new ArgumentNullException("primaryTestStep");
if (testActions == null)
throw new ArgumentNullException("testActions");
if (converter == null)
throw new ArgumentNullException("converter");
if (formatter == null)
throw new ArgumentNullException("formatter");
this.primaryTestStep = primaryTestStep;
this.testActions = testActions;
this.converter = converter;
this.formatter = formatter;
this.isExplicit = isExplicit;
bindingContext = new DataBindingContext(converter);
testParameterDataAccessors = new Dictionary<PatternTestParameter, IDataAccessor>();
data = new UserDataCollection();
}
public void Run()
{
if (executor.progressMonitor.IsCanceled)
{
outcome = TestOutcome.Canceled;
return;
}
TestOutcome instanceOutcome;
try
{
PatternTestInstanceActions decoratedTestInstanceActions = testState.TestActions.TestInstanceActions.Copy();
instanceOutcome = primaryContext.Sandbox.Run(TestLog.Writer,
new DecorateTestInstanceAction(testState, decoratedTestInstanceActions).Run, "Decorate Child Test");
if (instanceOutcome.Status == TestStatus.Passed)
{
bool invisibleTestInstance = true;
PatternTestStep testStep;
if (reusePrimaryTestStep)
{
testStep = testState.PrimaryTestStep;
invisibleTestInstance = false;
PropertyBag map = DataItemUtils.GetMetadata(bindingItem);
foreach (KeyValuePair<string, string> entry in map.Pairs)
primaryContext.AddMetadata(entry.Key, entry.Value);
}
else
{
testStep = new PatternTestStep(testState.Test, testState.PrimaryTestStep,
testState.Test.Name, testState.Test.CodeElement, false);
testStep.Kind = testState.Test.Kind;
testStep.IsDynamic = bindingItem.IsDynamic;
bindingItem.PopulateMetadata(testStep.Metadata);
}
var bodyAction = new RunTestInstanceAction(executor, testCommand);
var testInstanceState = new PatternTestInstanceState(testStep, decoratedTestInstanceActions, testState, bindingItem, bodyAction.Run);
bodyAction.TestInstanceState = testInstanceState;
instanceOutcome = instanceOutcome.CombineWith(primaryContext.Sandbox.Run(
TestLog.Writer, new BeforeTestInstanceAction(testInstanceState).Run, "Before Test Instance"));
if (instanceOutcome.Status == TestStatus.Passed)
{
executor.progressMonitor.SetStatus(testStep.Name);
TestContext context = reusePrimaryTestStep
? primaryContext
: TestContext.PrepareContext(testCommand.StartStep(testStep), primaryContext.Sandbox.CreateChild());
testState.SetInContext(context);
testInstanceState.SetInContext(context);
invisibleTestInstance = false;
using (context.Enter())
{
if (RunTestInstanceBodyAction.CanOptimizeCallChainAndInvokeBodyActionDirectly(testInstanceState))
{
bodyAction.SkipProtect = true;
instanceOutcome = instanceOutcome.CombineWith(bodyAction.Run());
}
else
{
var runTestInstanceBodyAction = new RunTestInstanceBodyAction(testInstanceState);
TestOutcome sandboxOutcome = context.Sandbox.Run(TestLog.Writer, runTestInstanceBodyAction.Run, "Body");
instanceOutcome = instanceOutcome.CombineWith(runTestInstanceBodyAction.Outcome.CombineWith(sandboxOutcome));
}
}
if (!reusePrimaryTestStep)
context.FinishStep(instanceOutcome);
executor.progressMonitor.SetStatus("");
}
instanceOutcome = instanceOutcome.CombineWith(primaryContext.Sandbox.Run(
TestLog.Writer, new AfterTestInstanceAction(testInstanceState).Run, "After Test Instance"));
if (invisibleTestInstance)
instanceOutcome = PublishOutcomeFromInvisibleTest(testCommand, testStep, instanceOutcome).Outcome;
}
}
catch (Exception ex)
{
string message = String.Format("An exception occurred while preparing an instance of test '{0}'.", testState.Test.FullName);
if (reusePrimaryTestStep)
{
TestLog.Failures.WriteException(ex, message);
instanceOutcome = TestOutcome.Error;
}
else
{
instanceOutcome = ReportTestError(testCommand, testState.PrimaryTestStep, ex, message).Outcome;
}
}
outcome = instanceOutcome;
}
public void Run()
{
var test = (PatternTest)testCommand.Test;
TestContextCookie? parentContextCookie = null;
try
{
if (parentContext != null)
parentContextCookie = parentContext.Enter();
// The first time we call Run, we check whether the ApartmentState of the
// Thread is correct. If it is not, then we start a new thread and reenter
// with a flag set to skip initial processing.
if (!reentered)
{
if (executor.progressMonitor.IsCanceled)
{
result = new TestResult(TestOutcome.Canceled);
return;
}
if (!testCommand.AreDependenciesSatisfied())
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
context.LogWriter.Warnings.WriteLine("Skipped due to an unsatisfied test dependency.");
result = context.FinishStep(TestOutcome.Skipped, null);
return;
}
executor.progressMonitor.SetStatus(test.Name);
if (test.ApartmentState != ApartmentState.Unknown
&& Thread.CurrentThread.GetApartmentState() != test.ApartmentState)
{
reentered = true;
ThreadTask task = new TestEnvironmentAwareThreadTask("Test Runner " + test.ApartmentState,
(Action) Run, executor.environmentManager);
task.ApartmentState = test.ApartmentState;
task.Run(null);
if (!task.Result.HasValue)
{
throw new ModelException(
String.Format("Failed to perform action in thread with overridden apartment state {0}.",
test.ApartmentState), task.Result.Exception);
}
return;
}
}
// Actually run the test.
// Yes, this is a monstrously long method due to the inlining optimzation to minimize stack depth.
using (Sandbox sandbox = parentSandbox.CreateChild())
{
using (new ProcessIsolation())
{
using (sandbox.StartTimer(test.TimeoutFunc()))
{
TestOutcome outcome;
PatternTestActions testActions = test.TestActions;
if (testActionsDecorator != null)
outcome = testActionsDecorator(sandbox, ref testActions);
else
outcome = TestOutcome.Passed;
if (outcome.Status == TestStatus.Passed)
{
PatternTestStep primaryTestStep = new PatternTestStep(test, parentTestStep);
PatternTestState testState = new PatternTestState(primaryTestStep, testActions,
executor.converter, executor.formatter, testCommand.IsExplicit);
bool invisibleTest = true;
outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer,
new BeforeTestAction(testState).Run, "Before Test"));
if (outcome.Status == TestStatus.Passed)
{
bool reusePrimaryTestStep = !testState.BindingContext.HasBindings;
if (!reusePrimaryTestStep)
primaryTestStep.IsTestCase = false;
invisibleTest = false;
TestContext primaryContext = TestContext.PrepareContext(
testCommand.StartStep(primaryTestStep), sandbox);
testState.SetInContext(primaryContext);
using (primaryContext.Enter())
{
primaryContext.LifecyclePhase = LifecyclePhases.Initialize;
outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer,
new InitializeTestAction(testState).Run, "Initialize"));
}
if (outcome.Status == TestStatus.Passed)
{
var actions = new List<RunTestDataItemAction>();
try
{
foreach (IDataItem bindingItem in testState.BindingContext.GetItems(!executor.options.SkipDynamicTests))
actions.Add(new RunTestDataItemAction(executor, testCommand, testState, primaryContext,
reusePrimaryTestStep, bindingItem));
if (actions.Count == 0)
{
TestLog.Warnings.WriteLine("Test skipped because it is parameterized but no data was provided.");
outcome = TestOutcome.Skipped;
}
else
{
if (actions.Count == 1 || ! test.IsParallelizable)
{
foreach (var action in actions)
action.Run();
}
else
{
executor.scheduler.Run(GenericCollectionUtils.ConvertAllToArray<RunTestDataItemAction, Action>(
actions, action => action.Run));
}
TestOutcome combinedOutcome = TestOutcome.Passed;
foreach (var action in actions)
combinedOutcome = combinedOutcome.CombineWith(action.Outcome);
outcome = outcome.CombineWith(reusePrimaryTestStep ? combinedOutcome : combinedOutcome.Generalize());
}
}
catch (TestException ex)
{
if (ex.Outcome.Status == TestStatus.Failed)
{
TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName));
}
else
{
TestLog.Warnings.WriteException(ex);
}
outcome = ex.Outcome;
}
catch (Exception ex)
{
TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName));
outcome = TestOutcome.Error;
}
}
primaryContext.SetInterimOutcome(outcome);
using (primaryContext.Enter())
{
primaryContext.LifecyclePhase = LifecyclePhases.Dispose;
outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer,
new DisposeTestAction(testState).Run, "Dispose"));
}
result = primaryContext.FinishStep(outcome);
}
outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer,
new AfterTestAction(testState).Run, "After Test"));
if (invisibleTest)
result = PublishOutcomeFromInvisibleTest(testCommand, primaryTestStep, outcome);
}
}
}
}
}
catch (Exception ex)
{
result = ReportTestError(testCommand, parentTestStep, ex, String.Format("An exception occurred while preparing to run test '{0}'.", test.FullName));
}
finally
{
if (parentContextCookie.HasValue)
parentContextCookie.Value.ExitContext();
executor.progressMonitor.SetStatus("");
executor.progressMonitor.Worked(1);
}
}
/// <summary>
/// Creates an initial test instance state object.
/// </summary>
/// <param name="testStep">The test step used to execute the test instance.</param>
/// <param name="testInstanceActions">The test instance actions.</param>
/// <param name="testState">The test state.</param>
/// <param name="bindingItem">The data item.</param>
/// <param name="body">The body of the test instance.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="testStep"/>,
/// <paramref name="testInstanceActions"/> or <paramref name="testState"/> or <paramref name="bindingItem"/> is null.</exception>
/// <exception cref="ArgumentException">Thrown if <paramref name="testState"/> belongs to a
/// different test from the <paramref name="testStep"/>.</exception>
internal PatternTestInstanceState(PatternTestStep testStep,
PatternTestInstanceActions testInstanceActions,
PatternTestState testState, IDataItem bindingItem, TestAction body)
{
if (testStep == null)
throw new ArgumentNullException("testStep");
if (testInstanceActions == null)
throw new ArgumentNullException("testInstanceActions");
if (testState == null)
throw new ArgumentNullException("testState");
if (testStep.Test != testState.Test)
throw new ArgumentException("The test state belongs to a different test from the test step.", "testState");
if (bindingItem == null)
throw new ArgumentNullException("bindingItem");
if (body == null)
throw new ArgumentNullException("body");
this.testStep = testStep;
this.testInstanceActions = testInstanceActions;
this.testState = testState;
this.bindingItem = bindingItem;
this.body = body;
testParameterValues = new Dictionary<PatternTestParameter, object>();
slotValues = new Dictionary<ISlotInfo, object>();
data = new UserDataCollection();
nameBase = testStep.Name;
nameSuffixes = string.Empty;
}
public void Run()
{
if (executor.progressMonitor.IsCanceled)
{
outcome = TestOutcome.Canceled;
return;
}
TestOutcome instanceOutcome;
try
{
PatternTestInstanceActions decoratedTestInstanceActions = testState.TestActions.TestInstanceActions.Copy();
instanceOutcome = primaryContext.Sandbox.Run(TestLog.Writer,
new DecorateTestInstanceAction(testState, decoratedTestInstanceActions).Run, "Decorate Child Test");
if (instanceOutcome.Status == TestStatus.Passed)
{
bool invisibleTestInstance = true;
PatternTestStep testStep;
if (reusePrimaryTestStep)
{
testStep = testState.PrimaryTestStep;
invisibleTestInstance = false;
PropertyBag map = DataItemUtils.GetMetadata(bindingItem);
foreach (KeyValuePair <string, string> entry in map.Pairs)
{
primaryContext.AddMetadata(entry.Key, entry.Value);
}
}
else
{
testStep = new PatternTestStep(testState.Test, testState.PrimaryTestStep,
testState.Test.Name, testState.Test.CodeElement, false);
testStep.Kind = testState.Test.Kind;
testStep.IsDynamic = bindingItem.IsDynamic;
bindingItem.PopulateMetadata(testStep.Metadata);
}
var bodyAction = new RunTestInstanceAction(executor, testCommand);
var testInstanceState = new PatternTestInstanceState(testStep, decoratedTestInstanceActions, testState, bindingItem, bodyAction.Run);
bodyAction.TestInstanceState = testInstanceState;
instanceOutcome = instanceOutcome.CombineWith(primaryContext.Sandbox.Run(
TestLog.Writer, new BeforeTestInstanceAction(testInstanceState).Run, "Before Test Instance"));
if (instanceOutcome.Status == TestStatus.Passed)
{
executor.progressMonitor.SetStatus(testStep.Name);
TestContext context = reusePrimaryTestStep
? primaryContext
: TestContext.PrepareContext(testCommand.StartStep(testStep), primaryContext.Sandbox.CreateChild());
testState.SetInContext(context);
testInstanceState.SetInContext(context);
invisibleTestInstance = false;
using (context.Enter())
{
if (RunTestInstanceBodyAction.CanOptimizeCallChainAndInvokeBodyActionDirectly(testInstanceState))
{
bodyAction.SkipProtect = true;
instanceOutcome = instanceOutcome.CombineWith(bodyAction.Run());
}
else
{
var runTestInstanceBodyAction = new RunTestInstanceBodyAction(testInstanceState);
TestOutcome sandboxOutcome = context.Sandbox.Run(TestLog.Writer, runTestInstanceBodyAction.Run, "Body");
instanceOutcome = instanceOutcome.CombineWith(runTestInstanceBodyAction.Outcome.CombineWith(sandboxOutcome));
}
}
if (!reusePrimaryTestStep)
{
context.FinishStep(instanceOutcome);
}
executor.progressMonitor.SetStatus("");
}
instanceOutcome = instanceOutcome.CombineWith(primaryContext.Sandbox.Run(
TestLog.Writer, new AfterTestInstanceAction(testInstanceState).Run, "After Test Instance"));
if (invisibleTestInstance)
{
instanceOutcome = PublishOutcomeFromInvisibleTest(testCommand, testStep, instanceOutcome).Outcome;
}
}
}
catch (Exception ex)
{
string message = String.Format("An exception occurred while preparing an instance of test '{0}'.", testState.Test.FullName);
if (reusePrimaryTestStep)
{
TestLog.Failures.WriteException(ex, message);
instanceOutcome = TestOutcome.Error;
}
else
{
instanceOutcome = ReportTestError(testCommand, testState.PrimaryTestStep, ex, message).Outcome;
}
}
outcome = instanceOutcome;
}
public void Run()
{
var test = (PatternTest)testCommand.Test;
TestContextCookie?parentContextCookie = null;
try
{
if (parentContext != null)
{
parentContextCookie = parentContext.Enter();
}
// The first time we call Run, we check whether the ApartmentState of the
// Thread is correct. If it is not, then we start a new thread and reenter
// with a flag set to skip initial processing.
if (!reentered)
{
if (executor.progressMonitor.IsCanceled)
{
result = new TestResult(TestOutcome.Canceled);
return;
}
if (!testCommand.AreDependenciesSatisfied())
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
context.LogWriter.Warnings.WriteLine("Skipped due to an unsatisfied test dependency.");
result = context.FinishStep(TestOutcome.Skipped, null);
return;
}
executor.progressMonitor.SetStatus(test.Name);
if (test.ApartmentState != ApartmentState.Unknown &&
Thread.CurrentThread.GetApartmentState() != test.ApartmentState)
{
reentered = true;
ThreadTask task = new TestEnvironmentAwareThreadTask("Test Runner " + test.ApartmentState,
(GallioAction)Run, executor.environmentManager);
task.ApartmentState = test.ApartmentState;
task.Run(null);
if (!task.Result.HasValue)
{
throw new ModelException(
String.Format("Failed to perform action in thread with overridden apartment state {0}.",
test.ApartmentState), task.Result.Exception);
}
return;
}
}
// Actually run the test.
// Yes, this is a monstrously long method due to the inlining optimzation to minimize stack depth.
using (Sandbox sandbox = parentSandbox.CreateChild())
{
using (new ProcessIsolation())
{
using (sandbox.StartTimer(test.TimeoutFunc()))
{
TestOutcome outcome;
PatternTestActions testActions = test.TestActions;
if (testActionsDecorator != null)
{
outcome = testActionsDecorator(sandbox, ref testActions);
}
else
{
outcome = TestOutcome.Passed;
}
if (outcome.Status == TestStatus.Passed)
{
PatternTestStep primaryTestStep = new PatternTestStep(test, parentTestStep);
PatternTestState testState = new PatternTestState(primaryTestStep, testActions,
executor.converter, executor.formatter, testCommand.IsExplicit);
bool invisibleTest = true;
outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer,
new BeforeTestAction(testState).Run, "Before Test"));
if (outcome.Status == TestStatus.Passed)
{
bool reusePrimaryTestStep = !testState.BindingContext.HasBindings;
if (!reusePrimaryTestStep)
{
primaryTestStep.IsTestCase = false;
}
invisibleTest = false;
TestContext primaryContext = TestContext.PrepareContext(
testCommand.StartStep(primaryTestStep), sandbox);
testState.SetInContext(primaryContext);
using (primaryContext.Enter())
{
primaryContext.LifecyclePhase = LifecyclePhases.Initialize;
outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer,
new InitializeTestAction(testState).Run, "Initialize"));
}
if (outcome.Status == TestStatus.Passed)
{
var actions = new List <RunTestDataItemAction>();
try
{
foreach (IDataItem bindingItem in testState.BindingContext.GetItems(!executor.options.SkipDynamicTests))
{
actions.Add(new RunTestDataItemAction(executor, testCommand, testState, primaryContext,
reusePrimaryTestStep, bindingItem));
}
if (actions.Count == 0)
{
TestLog.Warnings.WriteLine("Test skipped because it is parameterized but no data was provided.");
outcome = TestOutcome.Skipped;
}
else
{
if (actions.Count == 1 || !test.IsParallelizable)
{
foreach (var action in actions)
{
action.Run();
}
}
else
{
executor.scheduler.Run(GenericCollectionUtils.ConvertAllToArray <RunTestDataItemAction, GallioAction>(
actions, action => action.Run));
}
TestOutcome combinedOutcome = TestOutcome.Passed;
foreach (var action in actions)
{
combinedOutcome = combinedOutcome.CombineWith(action.Outcome);
}
outcome = outcome.CombineWith(reusePrimaryTestStep ? combinedOutcome : combinedOutcome.Generalize());
}
}
catch (TestException ex)
{
if (ex.Outcome.Status == TestStatus.Failed)
{
TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName));
}
else
{
TestLog.Warnings.WriteException(ex);
}
outcome = ex.Outcome;
}
catch (Exception ex)
{
TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName));
outcome = TestOutcome.Error;
}
}
primaryContext.SetInterimOutcome(outcome);
using (primaryContext.Enter())
{
primaryContext.LifecyclePhase = LifecyclePhases.Dispose;
outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer,
new DisposeTestAction(testState).Run, "Dispose"));
}
result = primaryContext.FinishStep(outcome);
}
outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer,
new AfterTestAction(testState).Run, "After Test"));
if (invisibleTest)
{
result = PublishOutcomeFromInvisibleTest(testCommand, primaryTestStep, outcome);
}
}
}
}
}
}
catch (Exception ex)
{
result = ReportTestError(testCommand, parentTestStep, ex, String.Format("An exception occurred while preparing to run test '{0}'.", test.FullName));
}
finally
{
if (parentContextCookie.HasValue)
{
parentContextCookie.Value.ExitContext();
}
executor.progressMonitor.SetStatus("");
executor.progressMonitor.Worked(1);
}
}
