void ITestListener.OnAssemblyFinished(object sender, AssemblyEventArgs args)
{
ITestCommand testCommand;
string testName = args.AssemblyFullName.Split(',')[0];
if (!testCommandsByName.TryGetValue(testName, out testCommand))
{
return;
}
if (testContextStack.Count == 0)
{
return;
}
ITestContext testContext = testContextStack.Pop();
while (testContext.TestStep.Test != testCommand.Test)
{
testContext.FinishStep(GetFixtureOutcome(fixtureFailureCount, fixtureErrorCount), null);
testContext = testContextStack.Pop();
progressMonitor.Worked(1);
}
topResult = testContext.FinishStep(CalculateOutcome(assemblyFailureCount, assemblyErrorCount), null);
progressMonitor.Worked(1);
}
private static TestResult ReportTestError(ITestCommand testCommand, Model.Tree.TestStep parentTestStep, Exception ex, string message)
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
TestLog.Failures.WriteException(ex, message);
return(context.FinishStep(TestOutcome.Error, null));
}
private TestResult RunContext(NSpecContextTest contextTest, ITestCommand command, TestStep testStep)
{
ITestContext testContext = command.StartPrimaryChildStep(testStep);
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand testCommand in command.Children)
{
NSpecExampleTest exampleTest = testCommand.Test as NSpecExampleTest;
if (exampleTest == null)
{
continue;
}
outcome = outcome.CombineWith(this.RunTest(contextTest, exampleTest, testCommand, testContext.TestStep).Outcome);
}
foreach (ITestCommand testCommand in command.Children)
{
NSpecContextTest contextTestChild = testCommand.Test as NSpecContextTest;
if (contextTestChild == null)
{
continue;
}
outcome = outcome.CombineWith(this.RunContext(contextTestChild, testCommand, testContext.TestStep).Outcome);
}
return(testContext.FinishStep(outcome, null));
}
TestResult RunAssembly(MachineAssemblyTest assemblyTest, ITestCommand command, TestStep parentTestStep)
{
ITestContext assemblyContext = command.StartPrimaryChildStep(parentTestStep);
AssemblyInfo assemblyInfo = new AssemblyInfo(assemblyTest.Name, assemblyTest.AssemblyFilePath);
TestOutcome outcome = TestOutcome.Passed;
_listener.OnAssemblyStart(assemblyInfo);
assemblyTest.AssemblyContexts.Each(context => context.OnAssemblyStart());
foreach (ITestCommand contextCommand in command.Children)
{
MachineContextTest contextTest = contextCommand.Test as MachineContextTest;
if (contextTest == null)
{
continue;
}
var contextResult = RunContextTest(assemblyTest, contextTest, contextCommand, assemblyContext.TestStep);
outcome = outcome.CombineWith(contextResult.Outcome);
assemblyContext.SetInterimOutcome(outcome);
}
assemblyTest.AssemblyContexts.Reverse().Each(context => context.OnAssemblyComplete());
_listener.OnAssemblyEnd(assemblyInfo);
return(assemblyContext.FinishStep(outcome, null));
}
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount))
{
if (options.SkipTestExecution)
{
return(SkipAll(rootTestCommand, parentTestStep));
}
else
{
ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep);
TestStep rootStep = rootContext.TestStep;
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand command in rootTestCommand.Children)
{
NSpecAssemblyTest assemblyTest = command.Test as NSpecAssemblyTest;
if (assemblyTest == null)
{
continue;
}
var assemblyResult = this.RunAssembly(command, rootStep);
outcome = outcome.CombineWith(assemblyResult.Outcome);
}
return(rootContext.FinishStep(outcome, null));
}
}
}
private ITestContext GetFixtureContext(ITestCommand fixtureCommand)
{
ITestContext parentContext = testContextStack.Peek();
if (parentContext.TestStep.Test != fixtureCommand.Test)
{
while (parentContext.TestStep.Test.Kind != CSUnitTestExplorer.AssemblyKind)
{
testContextStack.Pop();
TestOutcome outcome = GetFixtureOutcome(fixtureFailureCount, fixtureErrorCount);
parentContext.FinishStep(outcome, null);
progressMonitor.Worked(1);
parentContext = testContextStack.Peek();
}
parentContext = fixtureCommand.StartPrimaryChildStep(parentContext.TestStep);
parentContext.LifecyclePhase = LifecyclePhases.Execute;
progressMonitor.SetStatus(fixtureCommand.Test.Name);
testContextStack.Push(parentContext);
fixtureFailureCount = 0;
fixtureErrorCount = 0;
}
return(parentContext);
}
TestResult RunTest(NSpecContextTest contextTest, NSpecExampleTest exampleTest,
ITestCommand testCommand, TestStep testStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(testStep);
TestOutcome outcome = TestOutcome.Passed;
if (exampleTest.Example.Pending)
{
outcome = TestOutcome.Pending;
testContext.AddMetadata(MetadataKeys.PendingReason, "Needs to be implemented");
}
else
{
contextTest.Context.Exercise(exampleTest.Example, contextTest.Context.GetInstance());
if (exampleTest.Example.Exception != null)
{
TestLog.Failures.WriteException(ConvertException(exampleTest.Example.Exception));
TestLog.Failures.Flush();
outcome = TestOutcome.Failed;
}
}
return(testContext.FinishStep(outcome, null));
}
private void TestFinished(TestOutcome outcome, string fixtureName, string testName, int assertCount, ulong durationNanosec, string reason, TestContextCallback callback)
{
ITestCommand fixtureCommand;
if (!testCommandsByName.TryGetValue(fixtureName, out fixtureCommand))
{
return;
}
ITestCommand testCommand;
ITestContext testContext = null;
if (testCommandsByName.TryGetValue(fixtureName + @"." + testName, out testCommand))
{
if (testContextStack.Peek().TestStep.Test == testCommand.Test)
{
// Remove our test context from the stack
testContext = testContextStack.Pop();
}
}
ITestContext fixtureContext = GetFixtureContext(fixtureCommand);
if (testCommand != null)
{
if (testContext == null)
{
testContext = testCommand.StartPrimaryChildStep(fixtureContext.TestStep);
testContext.LifecyclePhase = LifecyclePhases.Execute;
progressMonitor.SetStatus(testCommand.Test.Name);
}
TimeSpan?duration = null;
if (durationNanosec > 0)
{
// A tick is equal to 100 nanoseconds
duration = TimeSpan.FromTicks((long)(durationNanosec / 100UL));
}
if (callback != null)
{
callback(testContext);
}
testContext.AddAssertCount(assertCount);
testContext.FinishStep(outcome, duration);
progressMonitor.Worked(1);
}
else if (!String.IsNullOrEmpty(reason))
{
MarkupStreamWriter log = fixtureContext.LogWriter.Failures;
using (log.BeginSection(Resources.CSUnitTestController_ResultMessageSectionName))
{
log.Write(reason);
}
}
}
private static bool LogMethodResultAndFinishStep(ITestContext testContext, XunitMethodResult result, bool useXunitTime)
{
TimeSpan?testTime = useXunitTime ? (TimeSpan?)TimeSpan.FromSeconds(result.ExecutionTime) : null;
if (!string.IsNullOrEmpty(result.Output))
{
testContext.LogWriter.ConsoleOutput.Write(result.Output);
}
if (result is XunitPassedResult)
{
testContext.FinishStep(TestOutcome.Passed, testTime);
return(true);
}
XunitFailedResult failedResult = result as XunitFailedResult;
if (failedResult != null)
{
// Report the failure exception.
testContext.LogWriter.Failures.WriteException(
new ExceptionData(
failedResult.ExceptionType ?? "",
failedResult.Message ?? "",
failedResult.StackTrace ?? "",
ExceptionData.NoProperties,
null), "Exception");
testContext.FinishStep(TestOutcome.Failed, testTime);
return(false);
}
XunitSkipResult skipResult = result as XunitSkipResult;
if (skipResult != null)
{
testContext.LogWriter.Warnings.Write("The test was skipped. Reason: {0}\n",
string.IsNullOrEmpty(skipResult.Reason) ? "<unspecified>" : skipResult.Reason);
testContext.FinishStep(TestOutcome.Skipped, testTime);
return(true);
}
throw new NotSupportedException(String.Format("Unrecognized Xunit method result type: '{0}'.", result.GetType()));
}
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
// NOTE: This method has been optimized to minimize the total stack depth of the action
// by inlining blocks on the critical path that had previously been factored out.
using (TestController testController = testControllerProvider(testCommand.Test))
{
if (testController != null)
{
try
{
using (IProgressMonitor subProgressMonitor = progressMonitor.CreateSubProgressMonitor(testCommand.TestCount))
{
// Calling RunImpl directly instead of Run to minimize stack depth
// because we already know the arguments are valid.
return(testController.RunImpl(testCommand, parentTestStep, options, subProgressMonitor));
}
}
catch (Exception ex)
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
context.LogWriter.Failures.WriteException(ex, "Fatal Exception in test controller");
return(context.FinishStep(TestOutcome.Error, null));
}
}
}
// Enter the scope of the test and recurse until we find a controller.
progressMonitor.SetStatus(testCommand.Test.FullName);
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand monitor in testCommand.Children)
{
if (progressMonitor.IsCanceled)
{
break;
}
TestResult childResult = RunTest(monitor, testContext.TestStep, options, progressMonitor);
outcome = outcome.CombineWith(childResult.Outcome).Generalize();
}
if (progressMonitor.IsCanceled)
{
outcome = TestOutcome.Canceled;
}
TestResult result = testContext.FinishStep(outcome, null);
progressMonitor.Worked(1);
return(result);
}
/// <summary>
/// Recursively generates single test steps for each <see cref="ITestCommand" /> and
/// sets the final outcome to <see cref="TestOutcome.Skipped" />.
/// </summary>
/// <remarks>
/// <para>
/// This is useful for implementing fallback behavior when
/// <see cref="TestExecutionOptions.SkipTestExecution" /> is true.
/// </para>
/// </remarks>
/// <param name="rootTestCommand">The root test command.</param>
/// <param name="parentTestStep">The parent test step.</param>
/// <returns>The combined result of the test commands.</returns>
protected static TestResult SkipAll(ITestCommand rootTestCommand, TestStep parentTestStep)
{
ITestContext context = rootTestCommand.StartPrimaryChildStep(parentTestStep);
foreach (ITestCommand child in rootTestCommand.Children)
{
SkipAll(child, context.TestStep);
}
return(context.FinishStep(TestOutcome.Skipped, null));
}
private TestResult RunTestStep(ITestCommand testCommand, TestInfoData testStepInfo, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestStepResult testStepResult = repository.RunTest(testStepInfo);
reporter.Run(testContext, testStepInfo, testStepResult);
WriteToTestLog(testContext, testStepResult);
testContext.AddAssertCount(testStepResult.AssertCount);
progressMonitor.Worked(1);
return(testContext.FinishStep(testStepResult.TestOutcome, testStepResult.Duration));
}
private void FinishStepWithReportRunResult(ITestContext testContext, ReportRunResult reportRunResult)
{
TestOutcome outcome = GetOutcomeFromReportRunResult(reportRunResult);
testContext.FinishStep(outcome, null);
// only update assembly status if more severe
if (outcome.Status > assemblyTestOutcome.Status)
{
assemblyTestOutcome = outcome;
}
}
TestResult RunContextTest(MachineAssemblyTest assemblyTest, MachineContextTest contextTest, ITestCommand command, TestStep parentTestStep)
{
ITestContext testContext = command.StartPrimaryChildStep(parentTestStep);
GallioRunListener listener = new GallioRunListener(_listener, _progressMonitor, testContext, command.Children);
IContextRunner runner = ContextRunnerFactory.GetContextRunnerFor(contextTest.Context);
runner.Run(contextTest.Context, listener, _options, assemblyTest.GlobalCleanup, assemblyTest.SpecificationSupplements);
return(testContext.FinishStep(listener.Outcome, null));
}
private static TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
bool passed = true;
foreach (ITestCommand child in testCommand.Children)
{
passed &= RunTest(child, testContext.TestStep, progressMonitor).Outcome.Status == TestStatus.Passed;
}
return(testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null));
}
private void HandleAssemblyFinish(TestOutcome outcome)
{
ITestContext assemblyTestContext = activeTestContexts[assemblyTestCommand];
activeTestContexts.Remove(assemblyTestCommand);
// only update status if more severe
if (outcome.Status > assemblyTestOutcome.Status)
{
assemblyTestOutcome = outcome;
}
assemblyTestResult = assemblyTestContext.FinishStep(assemblyTestOutcome, null);
}
private TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
TestOutcome combinedOutCome = TestOutcome.Passed;
var duration = TimeSpan.Zero;
foreach (ITestCommand child in testCommand.Children)
{
TestResult testResult = RunTest(child, testContext.TestStep, progressMonitor);
combinedOutCome = combinedOutCome.CombineWith(testResult.Outcome);
duration += testResult.Duration;
}
return(testContext.FinishStep(combinedOutCome, duration));
}
private static bool RunTestCommandAndFinishStep(ITestContext testContext, XunitTestClassCommand testClassCommand, XunitTestCommand testCommand)
{
try
{
testContext.LifecyclePhase = LifecyclePhases.Execute;
XunitMethodResult result = testCommand.Execute(testClassCommand.ObjectUnderTest);
return(LogMethodResultAndFinishStep(testContext, result, false));
}
catch (Exception ex)
{
// Xunit probably shouldn't throw an exception in a test command.
// But just in case...
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
testContext.FinishStep(TestOutcome.Failed, null);
return(false);
}
}
private static TestResult PublishOutcomeFromInvisibleTest(ITestCommand testCommand, Model.Tree.TestStep testStep, TestOutcome outcome)
{
switch (outcome.Status)
{
case TestStatus.Skipped:
case TestStatus.Passed:
// Either nothing interesting happened or the test was silently skipped during Before/After.
return(new TestResult(TestOutcome.Passed));
case TestStatus.Failed:
case TestStatus.Inconclusive:
default:
// Something bad happened during Before/After that prevented the test from running.
ITestContext context = testCommand.StartStep(testStep);
context.LogWriter.Failures.Write("The test did not run. Consult the parent test log for more details.");
return(context.FinishStep(outcome, null));
}
}
private static bool RunTestFixture(ITestCommand testCommand, ConcordionTest concordionTest, ITestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
// The magic happens here!
var concordion = new ConcordionBuilder()
.WithSource(concordionTest.Source)
.WithTarget(concordionTest.Target)
.WithSpecificationListener(new GallioResultRenderer())
.Build();
var summary = concordion.Process(concordionTest.Resource, concordionTest.Fixture);
bool passed = !(summary.HasFailures || summary.HasExceptions);
testContext.AddAssertCount((int)summary.SuccessCount + (int)summary.FailureCount);
testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
return(passed);
}
public override void OnSpecificationEnd(SpecificationInfo specification, Result result)
{
_listener.OnSpecificationEnd(specification, result);
ITestContext testContext = _contextsBySpec[specification.Name];
testContext.LifecyclePhase = LifecyclePhases.Finishing;
TestOutcome outcome;
TimeSpan? span = null;
if (result.Status == Status.NotImplemented)
{
TestLog.Warnings.WriteLine("{0} ({1})", specification.Name, "NOT IMPLEMENTED");
TestLog.Warnings.Flush();
outcome = TestOutcome.Pending;
span = new TimeSpan(0);
}
else if (result.Status == Status.Ignored)
{
TestLog.Warnings.WriteLine("{0} ({1})", specification.Name, "IGNORED");
TestLog.Warnings.Flush();
outcome = TestOutcome.Ignored;
span = new TimeSpan(0);
}
else if (result.Passed)
{
outcome = TestOutcome.Passed;
}
else
{
TestLog.Failures.WriteException(Convert(result.Exception));
TestLog.Failures.Flush();
outcome = TestOutcome.Failed;
}
testContext.FinishStep(outcome, span);
_outcome = _outcome.CombineWith(outcome);
_progressMonitor.Worked(1);
}
private void HandleTestOrSuiteFinished(NUnit.Core.TestResult nunitResult)
{
if (testContextStack.Count == 0)
{
return;
}
ITestContext testContext = testContextStack.Peek();
NUnitTest test = (NUnitTest)testContext.TestStep.Test;
if (test.Test.TestName != nunitResult.Test.TestName)
{
return;
}
testContextStack.Pop();
progressMonitor.Worked(1);
string logStreamName = nunitResult.ResultState == ResultState.Success ? MarkupStreamNames.Warnings : MarkupStreamNames.Failures;
MarkupDocumentWriter logWriter = testContext.LogWriter;
if (nunitResult.Message != null)
{
using (logWriter[logStreamName].BeginSection(Resources.NUnitTestController_ResultMessageSectionName))
logWriter[logStreamName].Write(nunitResult.Message);
}
if (nunitResult.StackTrace != null)
{
using (logWriter[logStreamName].BeginSection(Resources.NUnitTestController_ResultStackTraceSectionName))
using (logWriter[logStreamName].BeginMarker(Marker.StackTrace))
logWriter[logStreamName].Write(nunitResult.StackTrace);
}
testContext.AddAssertCount(nunitResult.AssertCount);
TestResult result = testContext.FinishStep(CreateOutcomeFromResult(nunitResult), null);
if (testContextStack.Count == 0)
{
topResult = result;
}
}
private static TestResult RunTestFixture(ITestCommand testCommand, XunitTypeInfoAdapter typeInfo,
TestStep parentTestStep)
{
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
XunitTestClassCommand testClassCommand;
try
{
testClassCommand = XunitTestClassCommandFactory.Make(typeInfo);
}
catch (Exception ex)
{
// Xunit can throw exceptions when making commands if the test is malformed.
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
return(testContext.FinishStep(TestOutcome.Failed, null));
}
return(RunTestClassCommandAndFinishStep(testCommand, testContext, testClassCommand));
}
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep,
TestExecutionOptions options, IProgressMonitor progressMonitor)
{
using (progressMonitor)
{
progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount);
if (options.SkipTestExecution)
{
return(SkipAll(rootTestCommand, parentTestStep));
}
else
{
ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep);
TestStep rootStep = rootContext.TestStep;
TestOutcome outcome = TestOutcome.Passed;
_progressMonitor = progressMonitor;
SetupRunOptions(options);
SetupListeners(options);
_listener.OnRunStart();
foreach (ITestCommand command in rootTestCommand.Children)
{
MachineAssemblyTest assemblyTest = command.Test as MachineAssemblyTest;
if (assemblyTest == null)
{
continue;
}
var assemblyResult = RunAssembly(assemblyTest, command, rootStep);
outcome = outcome.CombineWith(assemblyResult.Outcome);
}
_listener.OnRunEnd();
return(rootContext.FinishStep(outcome, null));
}
}
}
private TestResult RunAssembly(ITestCommand command, TestStep rootStep)
{
ITestContext assemblyContext = command.StartPrimaryChildStep(rootStep);
TestOutcome outcome = TestOutcome.Passed;
foreach (ITestCommand contextCommand in command.Children)
{
NSpecContextTest contextTest = contextCommand.Test as NSpecContextTest;
if (contextTest == null)
{
continue;
}
var contextResult = this.RunContext(contextTest, contextCommand, assemblyContext.TestStep);
outcome = outcome.CombineWith(contextResult.Outcome);
assemblyContext.SetInterimOutcome(outcome);
}
return(assemblyContext.FinishStep(outcome, null));
}
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
{
Test test = testCommand.Test;
progressMonitor.SetStatus(test.Name);
// The first test should be an assembly test
MSTestAssembly assemblyTest = testCommand.Test as MSTestAssembly;
TestOutcome outcome;
TestResult result;
if (assemblyTest != null)
{
ITestContext assemblyContext = testCommand.StartPrimaryChildStep(parentTestStep);
try
{
MSTestRunner runner = MSTestRunner.GetRunnerForFrameworkVersion(frameworkVersion);
outcome = runner.RunSession(assemblyContext, assemblyTest,
testCommand, parentTestStep, progressMonitor);
}
catch (Exception ex)
{
assemblyContext.LogWriter.Failures.WriteException(ex, "Internal Error");
outcome = TestOutcome.Error;
}
result = assemblyContext.FinishStep(outcome, null);
}
else
{
result = new TestResult(TestOutcome.Skipped);
}
progressMonitor.Worked(1);
return(result);
}
private static bool RunTestMethod(ITestCommand testCommand, MethodInfo methodInfo, XunitTestClassCommand testClassCommand,
TestStep parentTestStep)
{
List <XunitTestCommand> xunitTestCommands;
try
{
xunitTestCommands = new List <XunitTestCommand>(XunitTestCommandFactory.Make(testClassCommand,
XunitReflector.Wrap(methodInfo)));
}
catch (Exception ex)
{
// Xunit can throw exceptions when making commands if the test is malformed.
ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
testContext.FinishStep(TestOutcome.Failed, null);
return(false);
}
if (xunitTestCommands.Count == 0)
{
return(true);
}
if (xunitTestCommands.Count == 1)
{
return(RunTestCommands(testCommand, testClassCommand, xunitTestCommands, parentTestStep, true));
}
// Introduce a common primary test step for theories.
ITestContext primaryTestContext = testCommand.StartPrimaryChildStep(parentTestStep);
bool result = RunTestCommands(testCommand, testClassCommand, xunitTestCommands, primaryTestContext.TestStep, false);
primaryTestContext.FinishStep(result ? TestOutcome.Passed : TestOutcome.Failed, null);
return(result);
}
private static bool LogMethodResultAndFinishStep(ITestContext testContext, XunitMethodResult result, bool useXunitTime)
{
TimeSpan? testTime = useXunitTime ? (TimeSpan?)TimeSpan.FromSeconds(result.ExecutionTime) : null;
if (!string.IsNullOrEmpty(result.Output))
testContext.LogWriter.ConsoleOutput.Write(result.Output);
if (result is XunitPassedResult)
{
testContext.FinishStep(TestOutcome.Passed, testTime);
return true;
}
XunitFailedResult failedResult = result as XunitFailedResult;
if (failedResult != null)
{
// Report the failure exception.
testContext.LogWriter.Failures.WriteException(
new ExceptionData(
failedResult.ExceptionType ?? "",
failedResult.Message ?? "",
failedResult.StackTrace ?? "",
ExceptionData.NoProperties,
null), "Exception");
testContext.FinishStep(TestOutcome.Failed, testTime);
return false;
}
XunitSkipResult skipResult = result as XunitSkipResult;
if (skipResult != null)
{
testContext.LogWriter.Warnings.Write("The test was skipped. Reason: {0}\n",
string.IsNullOrEmpty(skipResult.Reason) ? "<unspecified>" : skipResult.Reason);
testContext.FinishStep(TestOutcome.Skipped, testTime);
return true;
}
throw new NotSupportedException(String.Format("Unrecognized Xunit method result type: '{0}'.", result.GetType()));
}
private static bool RunTestCommandAndFinishStep(ITestContext testContext, XunitTestClassCommand testClassCommand, XunitTestCommand testCommand)
{
try
{
testContext.LifecyclePhase = LifecyclePhases.Execute;
XunitMethodResult result = testCommand.Execute(testClassCommand.ObjectUnderTest);
return LogMethodResultAndFinishStep(testContext, result, false);
}
catch (Exception ex)
{
// Xunit probably shouldn't throw an exception in a test command.
// But just in case...
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
testContext.FinishStep(TestOutcome.Failed, null);
return false;
}
}
private static TestResult RunTestClassCommandAndFinishStep(ITestCommand testCommand, ITestContext testContext, XunitTestClassCommand testClassCommand)
{
try
{
bool passed = true;
// Run ClassStart behavior, if applicable.
testContext.LifecyclePhase = LifecyclePhases.SetUp;
Exception ex = testClassCommand.ClassStart();
// Run tests.
if (ex == null)
{
List<MethodInfo> testMethods = new List<MethodInfo>();
List<ITestCommand> testCommands = new List<ITestCommand>();
foreach (ITestCommand child in testCommand.Children)
{
XunitTest test = child.Test as XunitTest;
if (test != null)
{
testMethods.Add(test.MethodInfo.Target.Resolve(false));
testCommands.Add(child);
}
}
while (testMethods.Count != 0)
{
XunitMethodInfo[] xunitTestMethods = GenericCollectionUtils.ConvertAllToArray(
testMethods, x => XunitReflector.Wrap(x));
int nextTestIndex = testClassCommand.ChooseNextTest(xunitTestMethods);
ITestCommand nextTestCommand = testCommands[nextTestIndex];
MethodInfo nextTestMethodInfo = testMethods[nextTestIndex];
testMethods.RemoveAt(nextTestIndex);
testCommands.RemoveAt(nextTestIndex);
passed &= RunTestMethod(nextTestCommand, nextTestMethodInfo, testClassCommand,
testContext.TestStep);
}
}
// Run ClassFinish behavior, if applicable.
testContext.LifecyclePhase = LifecyclePhases.TearDown;
ex = testClassCommand.ClassFinish() ?? ex;
if (ex != null)
{
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
passed = false;
}
return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
}
catch (Exception ex)
{
// Xunit probably shouldn't throw an exception in a test command.
// But just in case...
testContext.LogWriter.Failures.WriteException(ex, "Internal Error");
return testContext.FinishStep(TestOutcome.Failed, null);
}
}
/// <summary>
/// Finishes the step represented by the context.
/// </summary>
/// <param name="outcome">The outcome.</param>
/// <returns>The final test result.</returns>
internal TestResult FinishStep(TestOutcome outcome)
{
return(inner.FinishStep(outcome, null));
}
private static TestResult SafeFinishStep(ITestContext testContext, TestOutcome outcome, TimeSpan duration)
{
testContext.BecomeMultiThreadAware();
using (TestContextTrackerAccessor.Instance.EnterContext(testContext))
return testContext.FinishStep(outcome, duration);
}
private TestResult RunTests(string assemblyPath)
{
AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.Default);
if (assemblyMetadata == null)
{
ITestContext testContext = listOfTestCommands[0].StartPrimaryChildStep(topTestStep);
testContext.LifecyclePhase = LifecyclePhases.Execute;
testContext.LogWriter.Failures.WriteLine("Test assembly does not exist or is not a valid .Net assembly. [{0}]", assemblyPath ?? String.Empty);
return(testContext.FinishStep(TestOutcome.Error, null));
}
// Remark: We cannot use the RemoteLoader directly from this AppDomain.
// csUnit v2.5 contains a bug in its detection of the NUnitAdapter. It tries
// to enumerate ALL types in ALL assemblies that are loaded in the AppDomain.
// Bad news for us because some of these types derived from other types in
// assemblies that cannot be loaded (eg. VisualStudio APIs).
// So csUnit promptly blows up. The workaround is to create our own AppDomain.
// We cannot use the csUnit Loader because it does things like report
// events asynchronously and possibly out of order or even in parallel. -- Jeff.
// See also: http://sourceforge.net/tracker/index.php?func=detail&aid=2111390&group_id=23919&atid=380010
HostSetup hostSetup = new HostSetup();
hostSetup.ApplicationBaseDirectory = Path.GetDirectoryName(assemblyPath);
hostSetup.WorkingDirectory = hostSetup.ApplicationBaseDirectory;
hostSetup.ShadowCopy = true;
hostSetup.ConfigurationFileLocation = ConfigurationFileLocation.AppBase;
hostSetup.ProcessorArchitecture = assemblyMetadata.ProcessorArchitecture;
string configFile = assemblyPath + ".config";
if (File.Exists(configFile))
{
hostSetup.Configuration.ConfigurationXml = File.ReadAllText(configFile);
}
var hostFactory = (IHostFactory)RuntimeAccessor.ServiceLocator.ResolveByComponentId(IsolatedAppDomainHostFactory.ComponentId);
using (IHost host = hostFactory.CreateHost(hostSetup, RuntimeAccessor.Logger))
{
HostAssemblyResolverHook.InstallCallback(host);
Type loaderType = typeof(RemoteLoader);
using (RemoteLoader loader = (RemoteLoader)host.GetHostService().CreateInstance(
loaderType.Assembly.FullName,
loaderType.FullName).Unwrap())
{
// Attach ourself to get feedback
loader.Listener = this;
// Load the test assembly
loader.LoadAssembly(assemblyPath);
// Run the tests of that assembly
TextWriter consoleOutputWriter = new ContextualLogTextWriter(MarkupStreamNames.ConsoleOutput);
var spec = new CallbackTestSpec(this);
loader.RunTests(spec, consoleOutputWriter);
}
}
return(topResult ?? new TestResult(TestOutcome.Error));
}
private void FinishStepWithReportRunResult(ITestContext testContext, ReportRunResult reportRunResult)
{
TestOutcome outcome = GetOutcomeFromReportRunResult(reportRunResult);
testContext.FinishStep(outcome, null);
// only update assembly status if more severe
if (outcome.Status > assemblyTestOutcome.Status)
assemblyTestOutcome = outcome;
}
public void Run()
{
var test = (PatternTest)testCommand.Test;
TestContextCookie?parentContextCookie = null;
try
{
if (parentContext != null)
{
parentContextCookie = parentContext.Enter();
}
// The first time we call Run, we check whether the ApartmentState of the
// Thread is correct. If it is not, then we start a new thread and reenter
// with a flag set to skip initial processing.
if (!reentered)
{
if (executor.progressMonitor.IsCanceled)
{
result = new TestResult(TestOutcome.Canceled);
return;
}
if (!testCommand.AreDependenciesSatisfied())
{
ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
context.LogWriter.Warnings.WriteLine("Skipped due to an unsatisfied test dependency.");
result = context.FinishStep(TestOutcome.Skipped, null);
return;
}
executor.progressMonitor.SetStatus(test.Name);
if (test.ApartmentState != ApartmentState.Unknown &&
Thread.CurrentThread.GetApartmentState() != test.ApartmentState)
{
reentered = true;
ThreadTask task = new TestEnvironmentAwareThreadTask("Test Runner " + test.ApartmentState,
(GallioAction)Run, executor.environmentManager);
task.ApartmentState = test.ApartmentState;
task.Run(null);
if (!task.Result.HasValue)
{
throw new ModelException(
String.Format("Failed to perform action in thread with overridden apartment state {0}.",
test.ApartmentState), task.Result.Exception);
}
return;
}
}
// Actually run the test.
// Yes, this is a monstrously long method due to the inlining optimzation to minimize stack depth.
using (Sandbox sandbox = parentSandbox.CreateChild())
{
using (new ProcessIsolation())
{
using (sandbox.StartTimer(test.TimeoutFunc()))
{
TestOutcome outcome;
PatternTestActions testActions = test.TestActions;
if (testActionsDecorator != null)
{
outcome = testActionsDecorator(sandbox, ref testActions);
}
else
{
outcome = TestOutcome.Passed;
}
if (outcome.Status == TestStatus.Passed)
{
PatternTestStep primaryTestStep = new PatternTestStep(test, parentTestStep);
PatternTestState testState = new PatternTestState(primaryTestStep, testActions,
executor.converter, executor.formatter, testCommand.IsExplicit);
bool invisibleTest = true;
outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer,
new BeforeTestAction(testState).Run, "Before Test"));
if (outcome.Status == TestStatus.Passed)
{
bool reusePrimaryTestStep = !testState.BindingContext.HasBindings;
if (!reusePrimaryTestStep)
{
primaryTestStep.IsTestCase = false;
}
invisibleTest = false;
TestContext primaryContext = TestContext.PrepareContext(
testCommand.StartStep(primaryTestStep), sandbox);
testState.SetInContext(primaryContext);
using (primaryContext.Enter())
{
primaryContext.LifecyclePhase = LifecyclePhases.Initialize;
outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer,
new InitializeTestAction(testState).Run, "Initialize"));
}
if (outcome.Status == TestStatus.Passed)
{
var actions = new List <RunTestDataItemAction>();
try
{
foreach (IDataItem bindingItem in testState.BindingContext.GetItems(!executor.options.SkipDynamicTests))
{
actions.Add(new RunTestDataItemAction(executor, testCommand, testState, primaryContext,
reusePrimaryTestStep, bindingItem));
}
if (actions.Count == 0)
{
TestLog.Warnings.WriteLine("Test skipped because it is parameterized but no data was provided.");
outcome = TestOutcome.Skipped;
}
else
{
if (actions.Count == 1 || !test.IsParallelizable)
{
foreach (var action in actions)
{
action.Run();
}
}
else
{
executor.scheduler.Run(GenericCollectionUtils.ConvertAllToArray <RunTestDataItemAction, GallioAction>(
actions, action => action.Run));
}
TestOutcome combinedOutcome = TestOutcome.Passed;
foreach (var action in actions)
{
combinedOutcome = combinedOutcome.CombineWith(action.Outcome);
}
outcome = outcome.CombineWith(reusePrimaryTestStep ? combinedOutcome : combinedOutcome.Generalize());
}
}
catch (TestException ex)
{
if (ex.Outcome.Status == TestStatus.Failed)
{
TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName));
}
else
{
TestLog.Warnings.WriteException(ex);
}
outcome = ex.Outcome;
}
catch (Exception ex)
{
TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName));
outcome = TestOutcome.Error;
}
}
primaryContext.SetInterimOutcome(outcome);
using (primaryContext.Enter())
{
primaryContext.LifecyclePhase = LifecyclePhases.Dispose;
outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer,
new DisposeTestAction(testState).Run, "Dispose"));
}
result = primaryContext.FinishStep(outcome);
}
outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer,
new AfterTestAction(testState).Run, "After Test"));
if (invisibleTest)
{
result = PublishOutcomeFromInvisibleTest(testCommand, primaryTestStep, outcome);
}
}
}
}
}
}
catch (Exception ex)
{
result = ReportTestError(testCommand, parentTestStep, ex, String.Format("An exception occurred while preparing to run test '{0}'.", test.FullName));
}
finally
{
if (parentContextCookie.HasValue)
{
parentContextCookie.Value.ExitContext();
}
executor.progressMonitor.SetStatus("");
executor.progressMonitor.Worked(1);
}
}
