void ITestListener.OnAssemblyFinished(object sender, AssemblyEventArgs args) { ITestCommand testCommand; string testName = args.AssemblyFullName.Split(',')[0]; if (!testCommandsByName.TryGetValue(testName, out testCommand)) { return; } if (testContextStack.Count == 0) { return; } ITestContext testContext = testContextStack.Pop(); while (testContext.TestStep.Test != testCommand.Test) { testContext.FinishStep(GetFixtureOutcome(fixtureFailureCount, fixtureErrorCount), null); testContext = testContextStack.Pop(); progressMonitor.Worked(1); } topResult = testContext.FinishStep(CalculateOutcome(assemblyFailureCount, assemblyErrorCount), null); progressMonitor.Worked(1); }
private static TestResult ReportTestError(ITestCommand testCommand, Model.Tree.TestStep parentTestStep, Exception ex, string message) { ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep); TestLog.Failures.WriteException(ex, message); return(context.FinishStep(TestOutcome.Error, null)); }
private TestResult RunContext(NSpecContextTest contextTest, ITestCommand command, TestStep testStep) { ITestContext testContext = command.StartPrimaryChildStep(testStep); TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand testCommand in command.Children) { NSpecExampleTest exampleTest = testCommand.Test as NSpecExampleTest; if (exampleTest == null) { continue; } outcome = outcome.CombineWith(this.RunTest(contextTest, exampleTest, testCommand, testContext.TestStep).Outcome); } foreach (ITestCommand testCommand in command.Children) { NSpecContextTest contextTestChild = testCommand.Test as NSpecContextTest; if (contextTestChild == null) { continue; } outcome = outcome.CombineWith(this.RunContext(contextTestChild, testCommand, testContext.TestStep).Outcome); } return(testContext.FinishStep(outcome, null)); }
TestResult RunAssembly(MachineAssemblyTest assemblyTest, ITestCommand command, TestStep parentTestStep) { ITestContext assemblyContext = command.StartPrimaryChildStep(parentTestStep); AssemblyInfo assemblyInfo = new AssemblyInfo(assemblyTest.Name, assemblyTest.AssemblyFilePath); TestOutcome outcome = TestOutcome.Passed; _listener.OnAssemblyStart(assemblyInfo); assemblyTest.AssemblyContexts.Each(context => context.OnAssemblyStart()); foreach (ITestCommand contextCommand in command.Children) { MachineContextTest contextTest = contextCommand.Test as MachineContextTest; if (contextTest == null) { continue; } var contextResult = RunContextTest(assemblyTest, contextTest, contextCommand, assemblyContext.TestStep); outcome = outcome.CombineWith(contextResult.Outcome); assemblyContext.SetInterimOutcome(outcome); } assemblyTest.AssemblyContexts.Reverse().Each(context => context.OnAssemblyComplete()); _listener.OnAssemblyEnd(assemblyInfo); return(assemblyContext.FinishStep(outcome, null)); }
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor) { using (progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount)) { if (options.SkipTestExecution) { return(SkipAll(rootTestCommand, parentTestStep)); } else { ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep); TestStep rootStep = rootContext.TestStep; TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand command in rootTestCommand.Children) { NSpecAssemblyTest assemblyTest = command.Test as NSpecAssemblyTest; if (assemblyTest == null) { continue; } var assemblyResult = this.RunAssembly(command, rootStep); outcome = outcome.CombineWith(assemblyResult.Outcome); } return(rootContext.FinishStep(outcome, null)); } } }
private ITestContext GetFixtureContext(ITestCommand fixtureCommand) { ITestContext parentContext = testContextStack.Peek(); if (parentContext.TestStep.Test != fixtureCommand.Test) { while (parentContext.TestStep.Test.Kind != CSUnitTestExplorer.AssemblyKind) { testContextStack.Pop(); TestOutcome outcome = GetFixtureOutcome(fixtureFailureCount, fixtureErrorCount); parentContext.FinishStep(outcome, null); progressMonitor.Worked(1); parentContext = testContextStack.Peek(); } parentContext = fixtureCommand.StartPrimaryChildStep(parentContext.TestStep); parentContext.LifecyclePhase = LifecyclePhases.Execute; progressMonitor.SetStatus(fixtureCommand.Test.Name); testContextStack.Push(parentContext); fixtureFailureCount = 0; fixtureErrorCount = 0; } return(parentContext); }
TestResult RunTest(NSpecContextTest contextTest, NSpecExampleTest exampleTest, ITestCommand testCommand, TestStep testStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(testStep); TestOutcome outcome = TestOutcome.Passed; if (exampleTest.Example.Pending) { outcome = TestOutcome.Pending; testContext.AddMetadata(MetadataKeys.PendingReason, "Needs to be implemented"); } else { contextTest.Context.Exercise(exampleTest.Example, contextTest.Context.GetInstance()); if (exampleTest.Example.Exception != null) { TestLog.Failures.WriteException(ConvertException(exampleTest.Example.Exception)); TestLog.Failures.Flush(); outcome = TestOutcome.Failed; } } return(testContext.FinishStep(outcome, null)); }
private void TestFinished(TestOutcome outcome, string fixtureName, string testName, int assertCount, ulong durationNanosec, string reason, TestContextCallback callback) { ITestCommand fixtureCommand; if (!testCommandsByName.TryGetValue(fixtureName, out fixtureCommand)) { return; } ITestCommand testCommand; ITestContext testContext = null; if (testCommandsByName.TryGetValue(fixtureName + @"." + testName, out testCommand)) { if (testContextStack.Peek().TestStep.Test == testCommand.Test) { // Remove our test context from the stack testContext = testContextStack.Pop(); } } ITestContext fixtureContext = GetFixtureContext(fixtureCommand); if (testCommand != null) { if (testContext == null) { testContext = testCommand.StartPrimaryChildStep(fixtureContext.TestStep); testContext.LifecyclePhase = LifecyclePhases.Execute; progressMonitor.SetStatus(testCommand.Test.Name); } TimeSpan?duration = null; if (durationNanosec > 0) { // A tick is equal to 100 nanoseconds duration = TimeSpan.FromTicks((long)(durationNanosec / 100UL)); } if (callback != null) { callback(testContext); } testContext.AddAssertCount(assertCount); testContext.FinishStep(outcome, duration); progressMonitor.Worked(1); } else if (!String.IsNullOrEmpty(reason)) { MarkupStreamWriter log = fixtureContext.LogWriter.Failures; using (log.BeginSection(Resources.CSUnitTestController_ResultMessageSectionName)) { log.Write(reason); } } }
private static bool LogMethodResultAndFinishStep(ITestContext testContext, XunitMethodResult result, bool useXunitTime) { TimeSpan?testTime = useXunitTime ? (TimeSpan?)TimeSpan.FromSeconds(result.ExecutionTime) : null; if (!string.IsNullOrEmpty(result.Output)) { testContext.LogWriter.ConsoleOutput.Write(result.Output); } if (result is XunitPassedResult) { testContext.FinishStep(TestOutcome.Passed, testTime); return(true); } XunitFailedResult failedResult = result as XunitFailedResult; if (failedResult != null) { // Report the failure exception. testContext.LogWriter.Failures.WriteException( new ExceptionData( failedResult.ExceptionType ?? "", failedResult.Message ?? "", failedResult.StackTrace ?? "", ExceptionData.NoProperties, null), "Exception"); testContext.FinishStep(TestOutcome.Failed, testTime); return(false); } XunitSkipResult skipResult = result as XunitSkipResult; if (skipResult != null) { testContext.LogWriter.Warnings.Write("The test was skipped. Reason: {0}\n", string.IsNullOrEmpty(skipResult.Reason) ? "<unspecified>" : skipResult.Reason); testContext.FinishStep(TestOutcome.Skipped, testTime); return(true); } throw new NotSupportedException(String.Format("Unrecognized Xunit method result type: '{0}'.", result.GetType())); }
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor) { // NOTE: This method has been optimized to minimize the total stack depth of the action // by inlining blocks on the critical path that had previously been factored out. using (TestController testController = testControllerProvider(testCommand.Test)) { if (testController != null) { try { using (IProgressMonitor subProgressMonitor = progressMonitor.CreateSubProgressMonitor(testCommand.TestCount)) { // Calling RunImpl directly instead of Run to minimize stack depth // because we already know the arguments are valid. return(testController.RunImpl(testCommand, parentTestStep, options, subProgressMonitor)); } } catch (Exception ex) { ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep); context.LogWriter.Failures.WriteException(ex, "Fatal Exception in test controller"); return(context.FinishStep(TestOutcome.Error, null)); } } } // Enter the scope of the test and recurse until we find a controller. progressMonitor.SetStatus(testCommand.Test.FullName); ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand monitor in testCommand.Children) { if (progressMonitor.IsCanceled) { break; } TestResult childResult = RunTest(monitor, testContext.TestStep, options, progressMonitor); outcome = outcome.CombineWith(childResult.Outcome).Generalize(); } if (progressMonitor.IsCanceled) { outcome = TestOutcome.Canceled; } TestResult result = testContext.FinishStep(outcome, null); progressMonitor.Worked(1); return(result); }
/// <summary> /// Recursively generates single test steps for each <see cref="ITestCommand" /> and /// sets the final outcome to <see cref="TestOutcome.Skipped" />. /// </summary> /// <remarks> /// <para> /// This is useful for implementing fallback behavior when /// <see cref="TestExecutionOptions.SkipTestExecution" /> is true. /// </para> /// </remarks> /// <param name="rootTestCommand">The root test command.</param> /// <param name="parentTestStep">The parent test step.</param> /// <returns>The combined result of the test commands.</returns> protected static TestResult SkipAll(ITestCommand rootTestCommand, TestStep parentTestStep) { ITestContext context = rootTestCommand.StartPrimaryChildStep(parentTestStep); foreach (ITestCommand child in rootTestCommand.Children) { SkipAll(child, context.TestStep); } return(context.FinishStep(TestOutcome.Skipped, null)); }
private TestResult RunTestStep(ITestCommand testCommand, TestInfoData testStepInfo, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestStepResult testStepResult = repository.RunTest(testStepInfo); reporter.Run(testContext, testStepInfo, testStepResult); WriteToTestLog(testContext, testStepResult); testContext.AddAssertCount(testStepResult.AssertCount); progressMonitor.Worked(1); return(testContext.FinishStep(testStepResult.TestOutcome, testStepResult.Duration)); }
private void FinishStepWithReportRunResult(ITestContext testContext, ReportRunResult reportRunResult) { TestOutcome outcome = GetOutcomeFromReportRunResult(reportRunResult); testContext.FinishStep(outcome, null); // only update assembly status if more severe if (outcome.Status > assemblyTestOutcome.Status) { assemblyTestOutcome = outcome; } }
TestResult RunContextTest(MachineAssemblyTest assemblyTest, MachineContextTest contextTest, ITestCommand command, TestStep parentTestStep) { ITestContext testContext = command.StartPrimaryChildStep(parentTestStep); GallioRunListener listener = new GallioRunListener(_listener, _progressMonitor, testContext, command.Children); IContextRunner runner = ContextRunnerFactory.GetContextRunnerFor(contextTest.Context); runner.Run(contextTest.Context, listener, _options, assemblyTest.GlobalCleanup, assemblyTest.SpecificationSupplements); return(testContext.FinishStep(listener.Outcome, null)); }
private static TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); bool passed = true; foreach (ITestCommand child in testCommand.Children) { passed &= RunTest(child, testContext.TestStep, progressMonitor).Outcome.Status == TestStatus.Passed; } return(testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null)); }
private void HandleAssemblyFinish(TestOutcome outcome) { ITestContext assemblyTestContext = activeTestContexts[assemblyTestCommand]; activeTestContexts.Remove(assemblyTestCommand); // only update status if more severe if (outcome.Status > assemblyTestOutcome.Status) { assemblyTestOutcome = outcome; } assemblyTestResult = assemblyTestContext.FinishStep(assemblyTestOutcome, null); }
private TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); TestOutcome combinedOutCome = TestOutcome.Passed; var duration = TimeSpan.Zero; foreach (ITestCommand child in testCommand.Children) { TestResult testResult = RunTest(child, testContext.TestStep, progressMonitor); combinedOutCome = combinedOutCome.CombineWith(testResult.Outcome); duration += testResult.Duration; } return(testContext.FinishStep(combinedOutCome, duration)); }
private static bool RunTestCommandAndFinishStep(ITestContext testContext, XunitTestClassCommand testClassCommand, XunitTestCommand testCommand) { try { testContext.LifecyclePhase = LifecyclePhases.Execute; XunitMethodResult result = testCommand.Execute(testClassCommand.ObjectUnderTest); return(LogMethodResultAndFinishStep(testContext, result, false)); } catch (Exception ex) { // Xunit probably shouldn't throw an exception in a test command. // But just in case... testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); testContext.FinishStep(TestOutcome.Failed, null); return(false); } }
private static TestResult PublishOutcomeFromInvisibleTest(ITestCommand testCommand, Model.Tree.TestStep testStep, TestOutcome outcome) { switch (outcome.Status) { case TestStatus.Skipped: case TestStatus.Passed: // Either nothing interesting happened or the test was silently skipped during Before/After. return(new TestResult(TestOutcome.Passed)); case TestStatus.Failed: case TestStatus.Inconclusive: default: // Something bad happened during Before/After that prevented the test from running. ITestContext context = testCommand.StartStep(testStep); context.LogWriter.Failures.Write("The test did not run. Consult the parent test log for more details."); return(context.FinishStep(outcome, null)); } }
private static bool RunTestFixture(ITestCommand testCommand, ConcordionTest concordionTest, ITestStep parentTestStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); // The magic happens here! var concordion = new ConcordionBuilder() .WithSource(concordionTest.Source) .WithTarget(concordionTest.Target) .WithSpecificationListener(new GallioResultRenderer()) .Build(); var summary = concordion.Process(concordionTest.Resource, concordionTest.Fixture); bool passed = !(summary.HasFailures || summary.HasExceptions); testContext.AddAssertCount((int)summary.SuccessCount + (int)summary.FailureCount); testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null); return(passed); }
public override void OnSpecificationEnd(SpecificationInfo specification, Result result) { _listener.OnSpecificationEnd(specification, result); ITestContext testContext = _contextsBySpec[specification.Name]; testContext.LifecyclePhase = LifecyclePhases.Finishing; TestOutcome outcome; TimeSpan? span = null; if (result.Status == Status.NotImplemented) { TestLog.Warnings.WriteLine("{0} ({1})", specification.Name, "NOT IMPLEMENTED"); TestLog.Warnings.Flush(); outcome = TestOutcome.Pending; span = new TimeSpan(0); } else if (result.Status == Status.Ignored) { TestLog.Warnings.WriteLine("{0} ({1})", specification.Name, "IGNORED"); TestLog.Warnings.Flush(); outcome = TestOutcome.Ignored; span = new TimeSpan(0); } else if (result.Passed) { outcome = TestOutcome.Passed; } else { TestLog.Failures.WriteException(Convert(result.Exception)); TestLog.Failures.Flush(); outcome = TestOutcome.Failed; } testContext.FinishStep(outcome, span); _outcome = _outcome.CombineWith(outcome); _progressMonitor.Worked(1); }
private void HandleTestOrSuiteFinished(NUnit.Core.TestResult nunitResult) { if (testContextStack.Count == 0) { return; } ITestContext testContext = testContextStack.Peek(); NUnitTest test = (NUnitTest)testContext.TestStep.Test; if (test.Test.TestName != nunitResult.Test.TestName) { return; } testContextStack.Pop(); progressMonitor.Worked(1); string logStreamName = nunitResult.ResultState == ResultState.Success ? MarkupStreamNames.Warnings : MarkupStreamNames.Failures; MarkupDocumentWriter logWriter = testContext.LogWriter; if (nunitResult.Message != null) { using (logWriter[logStreamName].BeginSection(Resources.NUnitTestController_ResultMessageSectionName)) logWriter[logStreamName].Write(nunitResult.Message); } if (nunitResult.StackTrace != null) { using (logWriter[logStreamName].BeginSection(Resources.NUnitTestController_ResultStackTraceSectionName)) using (logWriter[logStreamName].BeginMarker(Marker.StackTrace)) logWriter[logStreamName].Write(nunitResult.StackTrace); } testContext.AddAssertCount(nunitResult.AssertCount); TestResult result = testContext.FinishStep(CreateOutcomeFromResult(nunitResult), null); if (testContextStack.Count == 0) { topResult = result; } }
private static TestResult RunTestFixture(ITestCommand testCommand, XunitTypeInfoAdapter typeInfo, TestStep parentTestStep) { ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); XunitTestClassCommand testClassCommand; try { testClassCommand = XunitTestClassCommandFactory.Make(typeInfo); } catch (Exception ex) { // Xunit can throw exceptions when making commands if the test is malformed. testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); return(testContext.FinishStep(TestOutcome.Failed, null)); } return(RunTestClassCommandAndFinishStep(testCommand, testContext, testClassCommand)); }
protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor) { using (progressMonitor) { progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount); if (options.SkipTestExecution) { return(SkipAll(rootTestCommand, parentTestStep)); } else { ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep); TestStep rootStep = rootContext.TestStep; TestOutcome outcome = TestOutcome.Passed; _progressMonitor = progressMonitor; SetupRunOptions(options); SetupListeners(options); _listener.OnRunStart(); foreach (ITestCommand command in rootTestCommand.Children) { MachineAssemblyTest assemblyTest = command.Test as MachineAssemblyTest; if (assemblyTest == null) { continue; } var assemblyResult = RunAssembly(assemblyTest, command, rootStep); outcome = outcome.CombineWith(assemblyResult.Outcome); } _listener.OnRunEnd(); return(rootContext.FinishStep(outcome, null)); } } }
private TestResult RunAssembly(ITestCommand command, TestStep rootStep) { ITestContext assemblyContext = command.StartPrimaryChildStep(rootStep); TestOutcome outcome = TestOutcome.Passed; foreach (ITestCommand contextCommand in command.Children) { NSpecContextTest contextTest = contextCommand.Test as NSpecContextTest; if (contextTest == null) { continue; } var contextResult = this.RunContext(contextTest, contextCommand, assemblyContext.TestStep); outcome = outcome.CombineWith(contextResult.Outcome); assemblyContext.SetInterimOutcome(outcome); } return(assemblyContext.FinishStep(outcome, null)); }
private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor) { Test test = testCommand.Test; progressMonitor.SetStatus(test.Name); // The first test should be an assembly test MSTestAssembly assemblyTest = testCommand.Test as MSTestAssembly; TestOutcome outcome; TestResult result; if (assemblyTest != null) { ITestContext assemblyContext = testCommand.StartPrimaryChildStep(parentTestStep); try { MSTestRunner runner = MSTestRunner.GetRunnerForFrameworkVersion(frameworkVersion); outcome = runner.RunSession(assemblyContext, assemblyTest, testCommand, parentTestStep, progressMonitor); } catch (Exception ex) { assemblyContext.LogWriter.Failures.WriteException(ex, "Internal Error"); outcome = TestOutcome.Error; } result = assemblyContext.FinishStep(outcome, null); } else { result = new TestResult(TestOutcome.Skipped); } progressMonitor.Worked(1); return(result); }
private static bool RunTestMethod(ITestCommand testCommand, MethodInfo methodInfo, XunitTestClassCommand testClassCommand, TestStep parentTestStep) { List <XunitTestCommand> xunitTestCommands; try { xunitTestCommands = new List <XunitTestCommand>(XunitTestCommandFactory.Make(testClassCommand, XunitReflector.Wrap(methodInfo))); } catch (Exception ex) { // Xunit can throw exceptions when making commands if the test is malformed. ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep); testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); testContext.FinishStep(TestOutcome.Failed, null); return(false); } if (xunitTestCommands.Count == 0) { return(true); } if (xunitTestCommands.Count == 1) { return(RunTestCommands(testCommand, testClassCommand, xunitTestCommands, parentTestStep, true)); } // Introduce a common primary test step for theories. ITestContext primaryTestContext = testCommand.StartPrimaryChildStep(parentTestStep); bool result = RunTestCommands(testCommand, testClassCommand, xunitTestCommands, primaryTestContext.TestStep, false); primaryTestContext.FinishStep(result ? TestOutcome.Passed : TestOutcome.Failed, null); return(result); }
private static bool LogMethodResultAndFinishStep(ITestContext testContext, XunitMethodResult result, bool useXunitTime) { TimeSpan? testTime = useXunitTime ? (TimeSpan?)TimeSpan.FromSeconds(result.ExecutionTime) : null; if (!string.IsNullOrEmpty(result.Output)) testContext.LogWriter.ConsoleOutput.Write(result.Output); if (result is XunitPassedResult) { testContext.FinishStep(TestOutcome.Passed, testTime); return true; } XunitFailedResult failedResult = result as XunitFailedResult; if (failedResult != null) { // Report the failure exception. testContext.LogWriter.Failures.WriteException( new ExceptionData( failedResult.ExceptionType ?? "", failedResult.Message ?? "", failedResult.StackTrace ?? "", ExceptionData.NoProperties, null), "Exception"); testContext.FinishStep(TestOutcome.Failed, testTime); return false; } XunitSkipResult skipResult = result as XunitSkipResult; if (skipResult != null) { testContext.LogWriter.Warnings.Write("The test was skipped. Reason: {0}\n", string.IsNullOrEmpty(skipResult.Reason) ? "<unspecified>" : skipResult.Reason); testContext.FinishStep(TestOutcome.Skipped, testTime); return true; } throw new NotSupportedException(String.Format("Unrecognized Xunit method result type: '{0}'.", result.GetType())); }
private static bool RunTestCommandAndFinishStep(ITestContext testContext, XunitTestClassCommand testClassCommand, XunitTestCommand testCommand) { try { testContext.LifecyclePhase = LifecyclePhases.Execute; XunitMethodResult result = testCommand.Execute(testClassCommand.ObjectUnderTest); return LogMethodResultAndFinishStep(testContext, result, false); } catch (Exception ex) { // Xunit probably shouldn't throw an exception in a test command. // But just in case... testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); testContext.FinishStep(TestOutcome.Failed, null); return false; } }
private static TestResult RunTestClassCommandAndFinishStep(ITestCommand testCommand, ITestContext testContext, XunitTestClassCommand testClassCommand) { try { bool passed = true; // Run ClassStart behavior, if applicable. testContext.LifecyclePhase = LifecyclePhases.SetUp; Exception ex = testClassCommand.ClassStart(); // Run tests. if (ex == null) { List<MethodInfo> testMethods = new List<MethodInfo>(); List<ITestCommand> testCommands = new List<ITestCommand>(); foreach (ITestCommand child in testCommand.Children) { XunitTest test = child.Test as XunitTest; if (test != null) { testMethods.Add(test.MethodInfo.Target.Resolve(false)); testCommands.Add(child); } } while (testMethods.Count != 0) { XunitMethodInfo[] xunitTestMethods = GenericCollectionUtils.ConvertAllToArray( testMethods, x => XunitReflector.Wrap(x)); int nextTestIndex = testClassCommand.ChooseNextTest(xunitTestMethods); ITestCommand nextTestCommand = testCommands[nextTestIndex]; MethodInfo nextTestMethodInfo = testMethods[nextTestIndex]; testMethods.RemoveAt(nextTestIndex); testCommands.RemoveAt(nextTestIndex); passed &= RunTestMethod(nextTestCommand, nextTestMethodInfo, testClassCommand, testContext.TestStep); } } // Run ClassFinish behavior, if applicable. testContext.LifecyclePhase = LifecyclePhases.TearDown; ex = testClassCommand.ClassFinish() ?? ex; if (ex != null) { testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); passed = false; } return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null); } catch (Exception ex) { // Xunit probably shouldn't throw an exception in a test command. // But just in case... testContext.LogWriter.Failures.WriteException(ex, "Internal Error"); return testContext.FinishStep(TestOutcome.Failed, null); } }
/// <summary> /// Finishes the step represented by the context. /// </summary> /// <param name="outcome">The outcome.</param> /// <returns>The final test result.</returns> internal TestResult FinishStep(TestOutcome outcome) { return(inner.FinishStep(outcome, null)); }
private static TestResult SafeFinishStep(ITestContext testContext, TestOutcome outcome, TimeSpan duration) { testContext.BecomeMultiThreadAware(); using (TestContextTrackerAccessor.Instance.EnterContext(testContext)) return testContext.FinishStep(outcome, duration); }
private TestResult RunTests(string assemblyPath) { AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.Default); if (assemblyMetadata == null) { ITestContext testContext = listOfTestCommands[0].StartPrimaryChildStep(topTestStep); testContext.LifecyclePhase = LifecyclePhases.Execute; testContext.LogWriter.Failures.WriteLine("Test assembly does not exist or is not a valid .Net assembly. [{0}]", assemblyPath ?? String.Empty); return(testContext.FinishStep(TestOutcome.Error, null)); } // Remark: We cannot use the RemoteLoader directly from this AppDomain. // csUnit v2.5 contains a bug in its detection of the NUnitAdapter. It tries // to enumerate ALL types in ALL assemblies that are loaded in the AppDomain. // Bad news for us because some of these types derived from other types in // assemblies that cannot be loaded (eg. VisualStudio APIs). // So csUnit promptly blows up. The workaround is to create our own AppDomain. // We cannot use the csUnit Loader because it does things like report // events asynchronously and possibly out of order or even in parallel. -- Jeff. // See also: http://sourceforge.net/tracker/index.php?func=detail&aid=2111390&group_id=23919&atid=380010 HostSetup hostSetup = new HostSetup(); hostSetup.ApplicationBaseDirectory = Path.GetDirectoryName(assemblyPath); hostSetup.WorkingDirectory = hostSetup.ApplicationBaseDirectory; hostSetup.ShadowCopy = true; hostSetup.ConfigurationFileLocation = ConfigurationFileLocation.AppBase; hostSetup.ProcessorArchitecture = assemblyMetadata.ProcessorArchitecture; string configFile = assemblyPath + ".config"; if (File.Exists(configFile)) { hostSetup.Configuration.ConfigurationXml = File.ReadAllText(configFile); } var hostFactory = (IHostFactory)RuntimeAccessor.ServiceLocator.ResolveByComponentId(IsolatedAppDomainHostFactory.ComponentId); using (IHost host = hostFactory.CreateHost(hostSetup, RuntimeAccessor.Logger)) { HostAssemblyResolverHook.InstallCallback(host); Type loaderType = typeof(RemoteLoader); using (RemoteLoader loader = (RemoteLoader)host.GetHostService().CreateInstance( loaderType.Assembly.FullName, loaderType.FullName).Unwrap()) { // Attach ourself to get feedback loader.Listener = this; // Load the test assembly loader.LoadAssembly(assemblyPath); // Run the tests of that assembly TextWriter consoleOutputWriter = new ContextualLogTextWriter(MarkupStreamNames.ConsoleOutput); var spec = new CallbackTestSpec(this); loader.RunTests(spec, consoleOutputWriter); } } return(topResult ?? new TestResult(TestOutcome.Error)); }
private void FinishStepWithReportRunResult(ITestContext testContext, ReportRunResult reportRunResult) { TestOutcome outcome = GetOutcomeFromReportRunResult(reportRunResult); testContext.FinishStep(outcome, null); // only update assembly status if more severe if (outcome.Status > assemblyTestOutcome.Status) assemblyTestOutcome = outcome; }
public void Run() { var test = (PatternTest)testCommand.Test; TestContextCookie?parentContextCookie = null; try { if (parentContext != null) { parentContextCookie = parentContext.Enter(); } // The first time we call Run, we check whether the ApartmentState of the // Thread is correct. If it is not, then we start a new thread and reenter // with a flag set to skip initial processing. if (!reentered) { if (executor.progressMonitor.IsCanceled) { result = new TestResult(TestOutcome.Canceled); return; } if (!testCommand.AreDependenciesSatisfied()) { ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep); context.LogWriter.Warnings.WriteLine("Skipped due to an unsatisfied test dependency."); result = context.FinishStep(TestOutcome.Skipped, null); return; } executor.progressMonitor.SetStatus(test.Name); if (test.ApartmentState != ApartmentState.Unknown && Thread.CurrentThread.GetApartmentState() != test.ApartmentState) { reentered = true; ThreadTask task = new TestEnvironmentAwareThreadTask("Test Runner " + test.ApartmentState, (GallioAction)Run, executor.environmentManager); task.ApartmentState = test.ApartmentState; task.Run(null); if (!task.Result.HasValue) { throw new ModelException( String.Format("Failed to perform action in thread with overridden apartment state {0}.", test.ApartmentState), task.Result.Exception); } return; } } // Actually run the test. // Yes, this is a monstrously long method due to the inlining optimzation to minimize stack depth. using (Sandbox sandbox = parentSandbox.CreateChild()) { using (new ProcessIsolation()) { using (sandbox.StartTimer(test.TimeoutFunc())) { TestOutcome outcome; PatternTestActions testActions = test.TestActions; if (testActionsDecorator != null) { outcome = testActionsDecorator(sandbox, ref testActions); } else { outcome = TestOutcome.Passed; } if (outcome.Status == TestStatus.Passed) { PatternTestStep primaryTestStep = new PatternTestStep(test, parentTestStep); PatternTestState testState = new PatternTestState(primaryTestStep, testActions, executor.converter, executor.formatter, testCommand.IsExplicit); bool invisibleTest = true; outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer, new BeforeTestAction(testState).Run, "Before Test")); if (outcome.Status == TestStatus.Passed) { bool reusePrimaryTestStep = !testState.BindingContext.HasBindings; if (!reusePrimaryTestStep) { primaryTestStep.IsTestCase = false; } invisibleTest = false; TestContext primaryContext = TestContext.PrepareContext( testCommand.StartStep(primaryTestStep), sandbox); testState.SetInContext(primaryContext); using (primaryContext.Enter()) { primaryContext.LifecyclePhase = LifecyclePhases.Initialize; outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer, new InitializeTestAction(testState).Run, "Initialize")); } if (outcome.Status == TestStatus.Passed) { var actions = new List <RunTestDataItemAction>(); try { foreach (IDataItem bindingItem in testState.BindingContext.GetItems(!executor.options.SkipDynamicTests)) { actions.Add(new RunTestDataItemAction(executor, testCommand, testState, primaryContext, reusePrimaryTestStep, bindingItem)); } if (actions.Count == 0) { TestLog.Warnings.WriteLine("Test skipped because it is parameterized but no data was provided."); outcome = TestOutcome.Skipped; } else { if (actions.Count == 1 || !test.IsParallelizable) { foreach (var action in actions) { action.Run(); } } else { executor.scheduler.Run(GenericCollectionUtils.ConvertAllToArray <RunTestDataItemAction, GallioAction>( actions, action => action.Run)); } TestOutcome combinedOutcome = TestOutcome.Passed; foreach (var action in actions) { combinedOutcome = combinedOutcome.CombineWith(action.Outcome); } outcome = outcome.CombineWith(reusePrimaryTestStep ? combinedOutcome : combinedOutcome.Generalize()); } } catch (TestException ex) { if (ex.Outcome.Status == TestStatus.Failed) { TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName)); } else { TestLog.Warnings.WriteException(ex); } outcome = ex.Outcome; } catch (Exception ex) { TestLog.Failures.WriteException(ex, String.Format("An exception occurred while getting data items for test '{0}'.", testState.Test.FullName)); outcome = TestOutcome.Error; } } primaryContext.SetInterimOutcome(outcome); using (primaryContext.Enter()) { primaryContext.LifecyclePhase = LifecyclePhases.Dispose; outcome = outcome.CombineWith(primaryContext.Sandbox.Run(TestLog.Writer, new DisposeTestAction(testState).Run, "Dispose")); } result = primaryContext.FinishStep(outcome); } outcome = outcome.CombineWith(sandbox.Run(TestLog.Writer, new AfterTestAction(testState).Run, "After Test")); if (invisibleTest) { result = PublishOutcomeFromInvisibleTest(testCommand, primaryTestStep, outcome); } } } } } } catch (Exception ex) { result = ReportTestError(testCommand, parentTestStep, ex, String.Format("An exception occurred while preparing to run test '{0}'.", test.FullName)); } finally { if (parentContextCookie.HasValue) { parentContextCookie.Value.ExitContext(); } executor.progressMonitor.SetStatus(""); executor.progressMonitor.Worked(1); } }