private TestResult RunContext( NSpecContextTest contextTest, ITestCommand command, TestStep testStep )
{
ITestContext testContext = command.StartPrimaryChildStep( testStep );
TestOutcome outcome = TestOutcome.Passed;
foreach( ITestCommand testCommand in command.Children )
{
NSpecExampleTest exampleTest = testCommand.Test as NSpecExampleTest;
if( exampleTest == null )
{
continue;
}
outcome = outcome.CombineWith( this.RunTest( contextTest, exampleTest, testCommand, testContext.TestStep ).Outcome );
}
foreach( ITestCommand testCommand in command.Children )
{
NSpecContextTest contextTestChild = testCommand.Test as NSpecContextTest;
if( contextTestChild == null )
{
continue;
}
outcome = outcome.CombineWith( this.RunContext( contextTestChild, testCommand, testContext.TestStep ).Outcome );
}
return testContext.FinishStep( outcome, null );
}
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
Test test = testCommand.Test;
progressMonitor.SetStatus(test.Name);
// The first test should be an assembly test
MSTestAssembly assemblyTest = testCommand.Test as MSTestAssembly;
TestOutcome outcome;
TestResult result;
if (assemblyTest != null)
{
ITestContext assemblyContext = testCommand.StartPrimaryChildStep(parentTestStep);
try
{
MSTestRunner runner = MSTestRunner.GetRunnerForFrameworkVersion(frameworkVersion);
outcome = runner.RunSession(assemblyContext, assemblyTest,
testCommand, parentTestStep, progressMonitor);
}
catch (Exception ex)
{
assemblyContext.LogWriter.Failures.WriteException(ex, "Internal Error");
outcome = TestOutcome.Error;
}
result = assemblyContext.FinishStep(outcome, null);
}
else
{
result = new TestResult(TestOutcome.Skipped);
}
progressMonitor.Worked(1);
return result;
}
protected override TestResult RunImpl( ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor )
{
using(progressMonitor.BeginTask( "Verifying Specifications", rootTestCommand.TestCount ) )
{
if( options.SkipTestExecution )
{
return SkipAll( rootTestCommand, parentTestStep );
}
else
{
ITestContext rootContext = rootTestCommand.StartPrimaryChildStep( parentTestStep );
TestStep rootStep = rootContext.TestStep;
TestOutcome outcome = TestOutcome.Passed;
foreach( ITestCommand command in rootTestCommand.Children )
{
NSpecAssemblyTest assemblyTest = command.Test as NSpecAssemblyTest;
if( assemblyTest == null )
continue;
var assemblyResult = this.RunAssembly( command, rootStep );
outcome = outcome.CombineWith( assemblyResult.Outcome );
}
return rootContext.FinishStep( outcome, null );
}
}
}
/// <inheritdoc />
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
ThrowIfDisposed();
using (progressMonitor.BeginTask(Resources.MbUnit2TestController_RunningMbUnitTests, 1))
{
if (progressMonitor.IsCanceled)
return new TestResult(TestOutcome.Canceled);
if (options.SkipTestExecution)
{
return SkipAll(rootTestCommand, parentTestStep);
}
else
{
IList<ITestCommand> testCommands = rootTestCommand.GetAllCommands();
using (InstrumentedFixtureRunner fixtureRunner = new InstrumentedFixtureRunner(fixtureExplorer,
testCommands, progressMonitor, parentTestStep))
{
return fixtureRunner.Run();
}
}
}
}
protected internal override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Running tests.", rootTestCommand.TestCount))
{
return RunTest(rootTestCommand, parentTestStep, options, progressMonitor);
}
}
protected internal override TestResult RunImpl(ITestCommand rootTestCommand, Model.Tree.TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Running tests.", rootTestCommand.TestCount))
{
// Note: We do not check options.SkipTestExecution here because we want to build up
// the tree of data-driven test steps. So we actually check it later on in the
// PatternTestExecutor. This is different from framework adapters
// at this time (because they do not generally support dynamically generated data-driven tests).
Sandbox sandbox = new Sandbox();
EventHandler canceledHandler = delegate { sandbox.Abort(TestOutcome.Canceled, "The user canceled the test run."); };
try
{
progressMonitor.Canceled += canceledHandler;
TestAssemblyExecutionParameters.Reset();
PatternTestExecutor executor = new PatternTestExecutor(options, progressMonitor, formatter, converter, environmentManager);
// Inlined to minimize stack depth.
var action = executor.CreateActionToRunTest(rootTestCommand, parentTestStep, sandbox, null);
action.Run();
return action.Result;
}
finally
{
progressMonitor.Canceled -= canceledHandler;
sandbox.Dispose();
}
}
}
public void SetUp()
{
testCommand = MockRepository.GenerateStub<ITestCommand>();
testName = new TestName { TestID = new TestID(), FullName = "fullName" };
var testCommandsByTestName = new Dictionary<TestName, ITestCommand> { { testName, testCommand } };
testFilter = new NUnitTestFilter(testCommandsByTestName);
test = MockRepository.GenerateStub<ITest>();
}
public NuwaTestCommand(ITestCommand innerCommand)
{
if (innerCommand == null)
{
throw new ArgumentNullException("innerCommand");
}
_proxy = innerCommand;
}
static bool OnTestStart(ITestCommand command, ExecutorCallback callback)
{
XmlNode node = command.ToStartXml();
if (node != null)
callback.Notify(node.OuterXml);
return callback.ShouldContinue();
}
private static TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
bool passed = true;
foreach (ITestCommand child in testCommand.Children)
passed &= RunTest(child, testContext.TestStep, progressMonitor).Outcome.Status == TestStatus.Passed;
return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
}
private void RunAll(IProgressMonitor progressMonitor, ITestCommand rootTestCommand)
{
using (var subProgressMonitor = progressMonitor.CreateSubProgressMonitor(1))
{
using (subProgressMonitor.BeginTask("Running the tests.", rootTestCommand.TestCount))
{
RunTest(rootTestCommand, null, subProgressMonitor);
}
}
}
/// <summary>
/// Runs the tests.
/// </summary>
/// <remarks>
/// <para>
/// This method can be called at most once during the lifetime of a test controller.
/// </para>
/// </remarks>
/// <param name="rootTestCommand">The root test monitor.</param>
/// <param name="parentTestStep">The parent test step, or null if starting a root step.</param>
/// <param name="options">The test execution options.</param>
/// <param name="progressMonitor">The progress monitor.</param>
/// <returns>The combined result of the root test command.</returns>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="rootTestCommand"/>
/// <paramref name="progressMonitor"/>, or <paramref name="options"/> is null.</exception>
public TestResult Run(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
if (rootTestCommand == null)
throw new ArgumentNullException("rootTestCommand");
if (progressMonitor == null)
throw new ArgumentNullException("progressMonitor");
if (options == null)
throw new ArgumentNullException("options");
return RunImpl(rootTestCommand, parentTestStep, options, progressMonitor);
}
public void WrapsPassedInCommandIntoFixtureCommand([Frozen] Dictionary<MethodInfo, object> fixtures, FixtureSet sut, ITestCommand command)
{
var result = sut.ApplyFixturesToCommand(command);
Assert.IsType<FixtureCommand>(result);
var fixtureCommand = (FixtureCommand) result;
Assert.Same(command, fixtureCommand.InnerCommand);
//can't do anything else since fixtures property is not exposed
var savedFixtures = typeof (FixtureCommand).GetField("fixtures", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(fixtureCommand);
Assert.Same(fixtures, savedFixtures);
}
static bool OnTestStart(ITestCommand command, ICallbackEventHandler callback)
{
if (callback == null)
return true;
XmlNode node = command.ToStartXml();
if (node != null)
callback.RaiseCallbackEvent(node.OuterXml);
return bool.Parse(callback.GetCallbackResult());
}
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
// NOTE: This method has been optimized to minimize the total stack depth of the action
// by inlining blocks on the critical path that had previously been factored out.
using (TestController testController = testControllerProvider(testCommand.Test))
{
if (testController != null)
{
try
{
using (IProgressMonitor subProgressMonitor = progressMonitor.CreateSubProgressMonitor(testCommand.TestCount))
{
// Calling RunImpl directly instead of Run to minimize stack depth
// because we already know the arguments are valid.
return testController.RunImpl(testCommand, parentTestStep, options, subProgressMonitor);
}
}
catch (Exception ex)
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
context.LogWriter.Failures.WriteException(ex, "Fatal Exception in test controller");
return context.FinishStep(TestOutcome.Error, null);
}
}
}
// Enter the scope of the test and recurse until we find a controller.
progressMonitor.SetStatus(testCommand.Test.FullName);
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand monitor in testCommand.Children)
{
if (progressMonitor.IsCanceled)
break;
TestResult childResult = RunTest(monitor, testContext.TestStep, options, progressMonitor);
outcome = outcome.CombineWith(childResult.Outcome).Generalize();
}
if (progressMonitor.IsCanceled)
outcome = TestOutcome.Canceled;
TestResult result = testContext.FinishStep(outcome, null);
progressMonitor.Worked(1);
return result;
}
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask(Resources.XunitTestController_RunningXunitTests, rootTestCommand.TestCount))
{
if (options.SkipTestExecution)
{
return SkipAll(rootTestCommand, parentTestStep);
}
else
{
return RunTest(rootTestCommand, parentTestStep, progressMonitor);
}
}
}
public TestOutcome RunSession(ITestContext assemblyTestContext, MSTestAssembly assemblyTest,
ITestCommand assemblyTestCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
DirectoryInfo tempDir = SpecialPathPolicy.For("MSTestAdapter").CreateTempDirectoryWithUniqueName();
try
{
// Set the test results path. Among other things, the test results path
// will determine where the deployed test files go.
string testResultsPath = Path.Combine(tempDir.FullName, "tests.trx");
// Set the test results root directory.
// This path determines both where MSTest searches for test files which
// is used to resolve relative paths to test files in the "*.vsmdi" file.
string searchPathRoot = Path.GetDirectoryName(assemblyTest.AssemblyFilePath);
// Set the test metadata and run config paths. These are just temporary
// files that can go anywhere on the filesystem. It happens to be convenient
// to store them in the same temporary directory as the test results.
string testMetadataPath = Path.Combine(tempDir.FullName, "tests.vsmdi");
string runConfigPath = Path.Combine(tempDir.FullName, "tests.runconfig");
progressMonitor.SetStatus("Generating test metadata file.");
CreateTestMetadataFile(testMetadataPath,
GetTestsFromCommands(assemblyTestCommand.PreOrderTraversal), assemblyTest.AssemblyFilePath);
progressMonitor.SetStatus("Generating run config file.");
CreateRunConfigFile(runConfigPath);
progressMonitor.SetStatus("Executing tests.");
Executor executor = new Executor(this, assemblyTestContext, assemblyTestCommand);
TestOutcome outcome = executor.Execute(testMetadataPath, testResultsPath,
runConfigPath, searchPathRoot);
return outcome;
}
finally
{
try
{
tempDir.Delete(true);
}
catch
{
// Ignore I/O exceptions deleting temporary files.
// They will probably be deleted by the OS later on during a file cleanup.
}
}
}
protected override void RunTestsInternal(ITestCommand rootTestCommand, ITestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor)
{
progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount);
if (options.SkipTestExecution)
{
SkipAll(rootTestCommand, parentTestStep);
}
else
{
RunTest(rootTestCommand, parentTestStep, progressMonitor);
}
}
}
/// <inheritdoc />
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask(Resources.ConcordionTestController_RunningConcordionTests, rootTestCommand.TestCount))
{
if (progressMonitor.IsCanceled)
return new TestResult(TestOutcome.Canceled);
if (options.SkipTestExecution)
{
return SkipAll(rootTestCommand, parentTestStep);
}
else
{
return RunTest(rootTestCommand, parentTestStep, progressMonitor);
}
}
}
private static TestResult RunTestFixture(ITestCommand testCommand, ConcordionTest concordionTest, TestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
// The magic happens here!
var concordion = new ConcordionBuilder()
.WithSource(concordionTest.Source)
.WithTarget(concordionTest.Target)
.WithSpecificationProcessingListener(new GallioResultRenderer())
.Build();
ConstructorInfo constructor = concordionTest.FixtureType.GetConstructor(Type.EmptyTypes);
var fixture=constructor.Invoke(new object[]{});
var summary = concordion.Process(concordionTest.Resource, fixture);
bool passed = !(summary.HasFailures || summary.HasExceptions);
testContext.AddAssertCount((int)summary.SuccessCount + (int)summary.FailureCount);
return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
}
private static TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
Test test = testCommand.Test;
progressMonitor.SetStatus(test.Name);
TestResult result;
XunitTest xunitTest = test as XunitTest;
if (xunitTest == null)
{
result = RunChildTests(testCommand, parentTestStep, progressMonitor);
}
else
{
result = RunTestFixture(testCommand, xunitTest.TypeInfo, parentTestStep);
}
progressMonitor.Worked(1);
return result;
}
private TestResult RunAssembly( ITestCommand command, TestStep rootStep )
{
ITestContext assemblyContext = command.StartPrimaryChildStep( rootStep );
TestOutcome outcome = TestOutcome.Passed;
foreach( ITestCommand contextCommand in command.Children )
{
NSpecContextTest contextTest = contextCommand.Test as NSpecContextTest;
if( contextTest == null )
continue;
var contextResult = this.RunContext( contextTest, contextCommand, assemblyContext.TestStep );
outcome = outcome.CombineWith( contextResult.Outcome );
assemblyContext.SetInterimOutcome( outcome );
}
return assemblyContext.FinishStep( outcome, null );
}
private static TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
Test test = testCommand.Test;
progressMonitor.SetStatus(test.Name);
TestResult result;
ConcordionTest concordionTest = test as ConcordionTest;
if (concordionTest == null)
{
result = RunChildTests(testCommand, parentTestStep, progressMonitor);
}
else
{
result = RunTestFixture(testCommand, concordionTest, parentTestStep);
}
progressMonitor.Worked(1);
return result;
}
public TestHarnessCommand(TestHarnessClassCommandAttribute attribute, ITestCommand inner)
: base(inner)
{
if (!Int32.TryParse(KuduUtils.GetTestSetting(RunsSettingKey), out _runs))
{
_runs = attribute.Runs;
}
_runs = Math.Max(DefaultRuns, _runs);
if (!Int32.TryParse(KuduUtils.GetTestSetting(RetriesSettingKey), out _retries))
{
_retries = attribute.Retries;
}
_retries = Math.Max(DefaultRetries, _retries);
if (!Boolean.TryParse(KuduUtils.GetTestSetting(SuppressErrorSettingKey), out _suppressError))
{
_suppressError = attribute.SuppressError;
}
}
private void RunTest(ITestCommand testCommand, ITestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITest test = testCommand.Test;
progressMonitor.SetStatus(test.Name);
MachineSpecificationTest specification = test as MachineSpecificationTest;
MachineContextTest description = test as MachineContextTest;
if (specification != null)
{
//RunContextTest(specification, testComman);
}
else if (description != null)
{
RunContextTest(description, testCommand, parentTestStep);
}
else
{
Debug.WriteLine("Got something weird " + test.GetType().ToString());
}
}
/// <inheritdoc />
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
IList<Test> allTheTest = parentTestStep.Test.Children;
PopulateSuiteFixtureData(allTheTest, options);
using (progressMonitor.BeginTask(Resources.ConcordionTestController_RunningConcordionTests, rootTestCommand.TestCount))
{
if (progressMonitor.IsCanceled)
{
return new TestResult(TestOutcome.Canceled);
}
if (options.SkipTestExecution)
{
return SkipAll(rootTestCommand, parentTestStep);
}
else
{
return RunTest(rootTestCommand, parentTestStep, progressMonitor);
}
}
}
/// <inheritdoc />
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
IList<ITestCommand> testCommands = rootTestCommand.GetAllCommands();
using (progressMonitor.BeginTask(Resources.CSUnitTestController_RunningCSUnitTests, testCommands.Count))
{
if (progressMonitor.IsCanceled)
{
return new TestResult(TestOutcome.Canceled);
}
if (options.SkipTestExecution)
{
return SkipAll(rootTestCommand, parentTestStep);
}
using (RunnerMonitor monitor = new RunnerMonitor(testCommands, parentTestStep, progressMonitor))
{
return monitor.Run(assemblyLocation);
}
}
}
private MethodResult Execute(ITestCommand inner)
{
if (! _tasks.Any())
{
foreach (var testCommand in _innerCommands)
{
var command = (DelegatingTestCommand) testCommand;
Func<MethodResult> action =
() =>
{
var innerCommand = (IMethodInfo) _methodInfoField.GetValue(command.InnerCommand);
return new LifetimeCommand(command, innerCommand).Execute(null);
};
var task = new Task<MethodResult>(action, TaskCreationOptions.LongRunning);
_tasks.Add(testCommand, task);
}
StartAllTasks(_tasks.Values);
}
try
{
return _tasks[inner].Result;
}
catch (AggregateException e)
{
if (e.InnerExceptions.Count == 1)
{
//This looks better in nunit
ExceptionUtility.RethrowWithNoStackTraceLoss(e.InnerException);
throw;
}
else
{
throw;
}
}
}
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor)
{
progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount);
if (options.SkipTestExecution)
{
return SkipAll(rootTestCommand, parentTestStep);
}
else
{
ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep);
TestStep rootStep = rootContext.TestStep;
TestOutcome outcome = TestOutcome.Passed;
_progressMonitor = progressMonitor;
SetupRunOptions(options);
SetupListeners(options);
_listener.OnRunStart();
foreach (ITestCommand command in rootTestCommand.Children)
{
MachineAssemblyTest assemblyTest = command.Test as MachineAssemblyTest;
if( assemblyTest == null )
continue;
var assemblyResult = RunAssembly(assemblyTest, command, rootStep);
outcome = outcome.CombineWith( assemblyResult.Outcome);
}
_listener.OnRunEnd();
return rootContext.FinishStep( outcome, null);
}
}
}
/// <inheritdoc />
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
ThrowIfDisposed();
IList<ITestCommand> testCommands = rootTestCommand.GetAllCommands();
using (progressMonitor.BeginTask(Resources.NUnitTestController_RunningNUnitTests, testCommands.Count))
{
if (progressMonitor.IsCanceled)
return new TestResult(TestOutcome.Canceled);
if (options.SkipTestExecution)
{
return SkipAll(rootTestCommand, parentTestStep);
}
else
{
using (RunMonitor monitor = new RunMonitor(runner, testCommands, parentTestStep, progressMonitor))
{
return monitor.Run();
}
}
}
}
