Beispiel #1
0
        private TestResult RunContext( NSpecContextTest contextTest, ITestCommand command, TestStep testStep )
        {
            ITestContext testContext = command.StartPrimaryChildStep( testStep );
            TestOutcome outcome = TestOutcome.Passed;

            foreach( ITestCommand testCommand in command.Children )
            {
                NSpecExampleTest exampleTest = testCommand.Test as NSpecExampleTest;
                if( exampleTest == null )
                {
                    continue;
                }
                outcome = outcome.CombineWith( this.RunTest( contextTest, exampleTest, testCommand, testContext.TestStep ).Outcome );
            }
            foreach( ITestCommand testCommand in command.Children )
            {
                NSpecContextTest contextTestChild = testCommand.Test as NSpecContextTest;
                if( contextTestChild == null )
                {
                    continue;
                }
                outcome = outcome.CombineWith( this.RunContext( contextTestChild, testCommand, testContext.TestStep ).Outcome );
            }

            return testContext.FinishStep( outcome, null );
        }
        private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
        {
            Test test = testCommand.Test;
            progressMonitor.SetStatus(test.Name);

            // The first test should be an assembly test
            MSTestAssembly assemblyTest = testCommand.Test as MSTestAssembly;
            TestOutcome outcome;
            TestResult result;
            if (assemblyTest != null)
            {
                ITestContext assemblyContext = testCommand.StartPrimaryChildStep(parentTestStep);
                try
                {
                    MSTestRunner runner = MSTestRunner.GetRunnerForFrameworkVersion(frameworkVersion);

                    outcome = runner.RunSession(assemblyContext, assemblyTest,
                        testCommand, parentTestStep, progressMonitor);
                }
                catch (Exception ex)
                {
                    assemblyContext.LogWriter.Failures.WriteException(ex, "Internal Error");
                    outcome = TestOutcome.Error;
                }

                result = assemblyContext.FinishStep(outcome, null);
            }
            else
            {
                result = new TestResult(TestOutcome.Skipped);
            }

            progressMonitor.Worked(1);
            return result;
        }
Beispiel #3
0
        protected override TestResult RunImpl( ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor )
        {
            using(progressMonitor.BeginTask( "Verifying Specifications", rootTestCommand.TestCount ) )
            {
                if( options.SkipTestExecution )
                {
                    return SkipAll( rootTestCommand, parentTestStep );
                }
                else
                {
                    ITestContext rootContext = rootTestCommand.StartPrimaryChildStep( parentTestStep );
                    TestStep rootStep = rootContext.TestStep;
                    TestOutcome outcome = TestOutcome.Passed;

                    foreach( ITestCommand command in rootTestCommand.Children )
                    {
                        NSpecAssemblyTest assemblyTest = command.Test as NSpecAssemblyTest;
                        if( assemblyTest == null )
                            continue;

                        var assemblyResult = this.RunAssembly( command, rootStep );
                        outcome = outcome.CombineWith( assemblyResult.Outcome );
                    }

                    return rootContext.FinishStep( outcome, null );
                }
            }
        }
        /// <inheritdoc />
        protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
        {
            ThrowIfDisposed();

            using (progressMonitor.BeginTask(Resources.MbUnit2TestController_RunningMbUnitTests, 1))
            {
                if (progressMonitor.IsCanceled)
                    return new TestResult(TestOutcome.Canceled);

                if (options.SkipTestExecution)
                {
                    return SkipAll(rootTestCommand, parentTestStep);
                }
                else
                {
                    IList<ITestCommand> testCommands = rootTestCommand.GetAllCommands();

                    using (InstrumentedFixtureRunner fixtureRunner = new InstrumentedFixtureRunner(fixtureExplorer,
                        testCommands, progressMonitor, parentTestStep))
                    {
                        return fixtureRunner.Run();
                    }
                }
            }
        }
 protected internal override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
 {
     using (progressMonitor.BeginTask("Running tests.", rootTestCommand.TestCount))
     {
         return RunTest(rootTestCommand, parentTestStep, options, progressMonitor);
     }
 }
        protected internal override TestResult RunImpl(ITestCommand rootTestCommand, Model.Tree.TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
        {
            using (progressMonitor.BeginTask("Running tests.", rootTestCommand.TestCount))
            {
                // Note: We do not check options.SkipTestExecution here because we want to build up
                // the tree of data-driven test steps.  So we actually check it later on in the
                // PatternTestExecutor.  This is different from framework adapters
                // at this time (because they do not generally support dynamically generated data-driven tests).
                Sandbox sandbox = new Sandbox();
                EventHandler canceledHandler = delegate { sandbox.Abort(TestOutcome.Canceled, "The user canceled the test run."); };
                try
                {
                    progressMonitor.Canceled += canceledHandler;

                    TestAssemblyExecutionParameters.Reset();

                    PatternTestExecutor executor = new PatternTestExecutor(options, progressMonitor, formatter, converter, environmentManager);

                    // Inlined to minimize stack depth.
                    var action = executor.CreateActionToRunTest(rootTestCommand, parentTestStep, sandbox, null);
                    action.Run();
                    return action.Result;
                }
                finally
                {
                    progressMonitor.Canceled -= canceledHandler;
                    sandbox.Dispose();
                }
            }
        }
 public void SetUp()
 {
     testCommand = MockRepository.GenerateStub<ITestCommand>();
     testName = new TestName { TestID = new TestID(), FullName = "fullName" };
     var testCommandsByTestName = new Dictionary<TestName, ITestCommand> { { testName, testCommand } };
     testFilter = new NUnitTestFilter(testCommandsByTestName);
     test = MockRepository.GenerateStub<ITest>();
 }
        public NuwaTestCommand(ITestCommand innerCommand)
        {
            if (innerCommand == null)
            {
                throw new ArgumentNullException("innerCommand");
            }

            _proxy = innerCommand;
        }
Beispiel #9
0
        static bool OnTestStart(ITestCommand command, ExecutorCallback callback)
        {
            XmlNode node = command.ToStartXml();

            if (node != null)
                callback.Notify(node.OuterXml);

            return callback.ShouldContinue();
        }
        private static TestResult RunChildTests(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
        {
            ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);

            bool passed = true;
            foreach (ITestCommand child in testCommand.Children)
                passed &= RunTest(child, testContext.TestStep, progressMonitor).Outcome.Status == TestStatus.Passed;

            return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
        }
Beispiel #11
0
 private void RunAll(IProgressMonitor progressMonitor, ITestCommand rootTestCommand)
 {
     using (var subProgressMonitor = progressMonitor.CreateSubProgressMonitor(1))
     {
         using (subProgressMonitor.BeginTask("Running the tests.", rootTestCommand.TestCount))
         {
             RunTest(rootTestCommand, null, subProgressMonitor);
         }
     }
 }
        /// <summary>
        /// Runs the tests.
        /// </summary>
        /// <remarks>
        /// <para>
        /// This method can be called at most once during the lifetime of a test controller.
        /// </para>
        /// </remarks>
        /// <param name="rootTestCommand">The root test monitor.</param>
        /// <param name="parentTestStep">The parent test step, or null if starting a root step.</param>
        /// <param name="options">The test execution options.</param>
        /// <param name="progressMonitor">The progress monitor.</param>
        /// <returns>The combined result of the root test command.</returns>
        /// <exception cref="ArgumentNullException">Thrown if <paramref name="rootTestCommand"/>
        /// <paramref name="progressMonitor"/>, or <paramref name="options"/> is null.</exception>
        public TestResult Run(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
        {
            if (rootTestCommand == null)
                throw new ArgumentNullException("rootTestCommand");
            if (progressMonitor == null)
                throw new ArgumentNullException("progressMonitor");
            if (options == null)
                throw new ArgumentNullException("options");

            return RunImpl(rootTestCommand, parentTestStep, options, progressMonitor);
        }
        public void WrapsPassedInCommandIntoFixtureCommand([Frozen] Dictionary<MethodInfo, object> fixtures, FixtureSet sut, ITestCommand command)
        {
            var result = sut.ApplyFixturesToCommand(command);

            Assert.IsType<FixtureCommand>(result);
            var fixtureCommand = (FixtureCommand) result;
            Assert.Same(command, fixtureCommand.InnerCommand);

            //can't do anything else since fixtures property is not exposed
            var savedFixtures = typeof (FixtureCommand).GetField("fixtures", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(fixtureCommand);
            Assert.Same(fixtures, savedFixtures);
        }
Beispiel #14
0
        static bool OnTestStart(ITestCommand command, ICallbackEventHandler callback)
        {
            if (callback == null)
                return true;

            XmlNode node = command.ToStartXml();

            if (node != null)
                callback.RaiseCallbackEvent(node.OuterXml);

            return bool.Parse(callback.GetCallbackResult());
        }
        private TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep,
            TestExecutionOptions options, IProgressMonitor progressMonitor)
        {
            // NOTE: This method has been optimized to minimize the total stack depth of the action
            //       by inlining blocks on the critical path that had previously been factored out.

            using (TestController testController = testControllerProvider(testCommand.Test))
            {
                if (testController != null)
                {
                    try
                    {
                        using (IProgressMonitor subProgressMonitor = progressMonitor.CreateSubProgressMonitor(testCommand.TestCount))
                        {
                            // Calling RunImpl directly instead of Run to minimize stack depth
                            // because we already know the arguments are valid.
                            return testController.RunImpl(testCommand, parentTestStep, options, subProgressMonitor);
                        }
                    }
                    catch (Exception ex)
                    {
                        ITestContext context = testCommand.StartPrimaryChildStep(parentTestStep);
                        context.LogWriter.Failures.WriteException(ex, "Fatal Exception in test controller");
                        return context.FinishStep(TestOutcome.Error, null);
                    }
                }
            }

            // Enter the scope of the test and recurse until we find a controller.
            progressMonitor.SetStatus(testCommand.Test.FullName);

            ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);
            TestOutcome outcome = TestOutcome.Passed;

            foreach (ITestCommand monitor in testCommand.Children)
            {
                if (progressMonitor.IsCanceled)
                    break;

                TestResult childResult = RunTest(monitor, testContext.TestStep, options, progressMonitor);
                outcome = outcome.CombineWith(childResult.Outcome).Generalize();
            }

            if (progressMonitor.IsCanceled)
                outcome = TestOutcome.Canceled;

            TestResult result = testContext.FinishStep(outcome, null);

            progressMonitor.Worked(1);
            return result;
        }
 protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
 {
     using (progressMonitor.BeginTask(Resources.XunitTestController_RunningXunitTests, rootTestCommand.TestCount))
     {
         if (options.SkipTestExecution)
         {
             return SkipAll(rootTestCommand, parentTestStep);
         }
         else
         {
             return RunTest(rootTestCommand, parentTestStep, progressMonitor);
         }
     }
 }
Beispiel #17
0
        public TestOutcome RunSession(ITestContext assemblyTestContext, MSTestAssembly assemblyTest,
            ITestCommand assemblyTestCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
        {
            DirectoryInfo tempDir = SpecialPathPolicy.For("MSTestAdapter").CreateTempDirectoryWithUniqueName();
            try
            {
                // Set the test results path.  Among other things, the test results path
                // will determine where the deployed test files go.
                string testResultsPath = Path.Combine(tempDir.FullName, "tests.trx");

                // Set the test results root directory.
                // This path determines both where MSTest searches for test files which
                // is used to resolve relative paths to test files in the "*.vsmdi" file.
                string searchPathRoot = Path.GetDirectoryName(assemblyTest.AssemblyFilePath);

                // Set the test metadata and run config paths.  These are just temporary
                // files that can go anywhere on the filesystem.  It happens to be convenient
                // to store them in the same temporary directory as the test results.
                string testMetadataPath = Path.Combine(tempDir.FullName, "tests.vsmdi");
                string runConfigPath = Path.Combine(tempDir.FullName, "tests.runconfig");

                progressMonitor.SetStatus("Generating test metadata file.");
                CreateTestMetadataFile(testMetadataPath,
                    GetTestsFromCommands(assemblyTestCommand.PreOrderTraversal), assemblyTest.AssemblyFilePath);

                progressMonitor.SetStatus("Generating run config file.");
                CreateRunConfigFile(runConfigPath);

                progressMonitor.SetStatus("Executing tests.");
                Executor executor = new Executor(this, assemblyTestContext, assemblyTestCommand);
                TestOutcome outcome = executor.Execute(testMetadataPath, testResultsPath,
                    runConfigPath, searchPathRoot);
                return outcome;
            }
            finally
            {
                try
                {
                    tempDir.Delete(true);
                }
                catch
                {
                    // Ignore I/O exceptions deleting temporary files.
                    // They will probably be deleted by the OS later on during a file cleanup.
                }
            }
        }
    protected override void RunTestsInternal(ITestCommand rootTestCommand, ITestStep parentTestStep,
                                             TestExecutionOptions options, IProgressMonitor progressMonitor)
    {
      using (progressMonitor)
      {
        progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount);

        if (options.SkipTestExecution)
        {
          SkipAll(rootTestCommand, parentTestStep);
        }
        else
        {
          RunTest(rootTestCommand, parentTestStep, progressMonitor);
        }
      }
    }
        /// <inheritdoc />
        protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
        {
            using (progressMonitor.BeginTask(Resources.ConcordionTestController_RunningConcordionTests, rootTestCommand.TestCount))
            {
                if (progressMonitor.IsCanceled)
                    return new TestResult(TestOutcome.Canceled);

                if (options.SkipTestExecution)
                {
                    return SkipAll(rootTestCommand, parentTestStep);
                }
                else
                {
                    return RunTest(rootTestCommand, parentTestStep, progressMonitor);
                }
            }
        }
        private static TestResult RunTestFixture(ITestCommand testCommand, ConcordionTest concordionTest, TestStep parentTestStep)
        {
            ITestContext testContext = testCommand.StartPrimaryChildStep(parentTestStep);

            // The magic happens here!
            var concordion = new ConcordionBuilder()
                                    .WithSource(concordionTest.Source)
                                    .WithTarget(concordionTest.Target)
                                    .WithSpecificationProcessingListener(new GallioResultRenderer())
                                    .Build();

            ConstructorInfo constructor = concordionTest.FixtureType.GetConstructor(Type.EmptyTypes);
            var fixture=constructor.Invoke(new object[]{});
            var summary = concordion.Process(concordionTest.Resource, fixture);
            bool passed = !(summary.HasFailures || summary.HasExceptions);
            testContext.AddAssertCount((int)summary.SuccessCount + (int)summary.FailureCount);
            return testContext.FinishStep(passed ? TestOutcome.Passed : TestOutcome.Failed, null);
        }
        private static TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
        {
            Test test = testCommand.Test;
            progressMonitor.SetStatus(test.Name);

            TestResult result;
            XunitTest xunitTest = test as XunitTest;
            if (xunitTest == null)
            {
                result = RunChildTests(testCommand, parentTestStep, progressMonitor);
            }
            else
            {
                result = RunTestFixture(testCommand, xunitTest.TypeInfo, parentTestStep);
            }

            progressMonitor.Worked(1);
            return result;
        }
Beispiel #22
0
        private TestResult RunAssembly( ITestCommand command, TestStep rootStep )
        {
            ITestContext assemblyContext = command.StartPrimaryChildStep( rootStep );

            TestOutcome outcome = TestOutcome.Passed;

            foreach( ITestCommand contextCommand in command.Children )
            {
                NSpecContextTest contextTest = contextCommand.Test as NSpecContextTest;
                if( contextTest == null )
                    continue;

                var contextResult = this.RunContext( contextTest, contextCommand, assemblyContext.TestStep );
                outcome = outcome.CombineWith( contextResult.Outcome );
                assemblyContext.SetInterimOutcome( outcome );
            }

            return assemblyContext.FinishStep( outcome, null );
        }
        private static TestResult RunTest(ITestCommand testCommand, TestStep parentTestStep, IProgressMonitor progressMonitor)
        {
            Test test = testCommand.Test;
            progressMonitor.SetStatus(test.Name);

            TestResult result;
            ConcordionTest concordionTest = test as ConcordionTest;
            if (concordionTest == null)
            {
                result = RunChildTests(testCommand, parentTestStep, progressMonitor);
            }
            else
            {
                result = RunTestFixture(testCommand, concordionTest, parentTestStep);
            }

            progressMonitor.Worked(1);
            return result;
        }
                public TestHarnessCommand(TestHarnessClassCommandAttribute attribute, ITestCommand inner)
                    : base(inner)
                {
                    if (!Int32.TryParse(KuduUtils.GetTestSetting(RunsSettingKey), out _runs))
                    {
                        _runs = attribute.Runs;
                    }
                    _runs = Math.Max(DefaultRuns, _runs);

                    if (!Int32.TryParse(KuduUtils.GetTestSetting(RetriesSettingKey), out _retries))
                    {
                        _retries = attribute.Retries;
                    }
                    _retries = Math.Max(DefaultRetries, _retries);

                    if (!Boolean.TryParse(KuduUtils.GetTestSetting(SuppressErrorSettingKey), out _suppressError))
                    {
                        _suppressError = attribute.SuppressError;
                    }
                }
    private void RunTest(ITestCommand testCommand, ITestStep parentTestStep, IProgressMonitor progressMonitor)
    {
      ITest test = testCommand.Test;
      progressMonitor.SetStatus(test.Name);

      MachineSpecificationTest specification = test as MachineSpecificationTest;
      MachineContextTest description = test as MachineContextTest;
      if (specification != null)
      {
        //RunContextTest(specification, testComman);
      }
      else if (description != null)
      {
        RunContextTest(description, testCommand, parentTestStep);
      }
      else
      {
        Debug.WriteLine("Got something weird " + test.GetType().ToString());
      }
    }
        /// <inheritdoc />
        protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
        {
            IList<Test> allTheTest = parentTestStep.Test.Children;
            PopulateSuiteFixtureData(allTheTest, options);
            using (progressMonitor.BeginTask(Resources.ConcordionTestController_RunningConcordionTests, rootTestCommand.TestCount))
            {
                if (progressMonitor.IsCanceled)
                {
                    return new TestResult(TestOutcome.Canceled);
                }

                if (options.SkipTestExecution)
                {
                    return SkipAll(rootTestCommand, parentTestStep);
                }
                else
                {
                    return RunTest(rootTestCommand, parentTestStep, progressMonitor);
                }
            }
        }
        /// <inheritdoc />
        protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
        {
            IList<ITestCommand> testCommands = rootTestCommand.GetAllCommands();
            using (progressMonitor.BeginTask(Resources.CSUnitTestController_RunningCSUnitTests, testCommands.Count))
            {
                if (progressMonitor.IsCanceled)
                {
                    return new TestResult(TestOutcome.Canceled);
                }

                if (options.SkipTestExecution)
                {
                    return SkipAll(rootTestCommand, parentTestStep);
                }

                using (RunnerMonitor monitor = new RunnerMonitor(testCommands, parentTestStep, progressMonitor))
                {
                    return monitor.Run(assemblyLocation);
                }
            }            
        }
        private MethodResult Execute(ITestCommand inner)
        {
            if (! _tasks.Any())
            {
                foreach (var testCommand in _innerCommands)
                {
                    var command = (DelegatingTestCommand) testCommand;

                    Func<MethodResult> action =
                        () =>
                        {
                            var innerCommand = (IMethodInfo) _methodInfoField.GetValue(command.InnerCommand);
                            return new LifetimeCommand(command, innerCommand).Execute(null);
                        };

                    var task = new Task<MethodResult>(action, TaskCreationOptions.LongRunning);
                    _tasks.Add(testCommand, task);
                }

                StartAllTasks(_tasks.Values);
            }

            try
            {
                return _tasks[inner].Result;
            }
            catch (AggregateException e)
            {
                if (e.InnerExceptions.Count == 1)
                {
                    //This looks better in nunit
                    ExceptionUtility.RethrowWithNoStackTraceLoss(e.InnerException);
                    throw;
                }
                else
                {
                    throw;
                }
            }
        }
    protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep,
                    TestExecutionOptions options, IProgressMonitor progressMonitor)
    {
      using (progressMonitor)
      {
        progressMonitor.BeginTask("Verifying Specifications", rootTestCommand.TestCount);

        if (options.SkipTestExecution)
        {
          return SkipAll(rootTestCommand, parentTestStep);
        }
        else
        {
          ITestContext rootContext = rootTestCommand.StartPrimaryChildStep(parentTestStep);
          TestStep rootStep = rootContext.TestStep;
          TestOutcome outcome = TestOutcome.Passed;

          _progressMonitor = progressMonitor;
          SetupRunOptions(options);
          SetupListeners(options);

          _listener.OnRunStart();

          foreach (ITestCommand command in rootTestCommand.Children)
          {
            MachineAssemblyTest assemblyTest = command.Test as MachineAssemblyTest;
            if( assemblyTest == null )
              continue;

            var assemblyResult = RunAssembly(assemblyTest, command, rootStep);
            outcome = outcome.CombineWith( assemblyResult.Outcome);
          }

          _listener.OnRunEnd();

          return rootContext.FinishStep( outcome, null);
        }
      }
    }
        /// <inheritdoc />
        protected override TestResult RunImpl(ITestCommand rootTestCommand, TestStep parentTestStep, TestExecutionOptions options, IProgressMonitor progressMonitor)
        {
            ThrowIfDisposed();

            IList<ITestCommand> testCommands = rootTestCommand.GetAllCommands();
            using (progressMonitor.BeginTask(Resources.NUnitTestController_RunningNUnitTests, testCommands.Count))
            {
                if (progressMonitor.IsCanceled)
                    return new TestResult(TestOutcome.Canceled);

                if (options.SkipTestExecution)
                {
                    return SkipAll(rootTestCommand, parentTestStep);
                }
                else
                {
                    using (RunMonitor monitor = new RunMonitor(runner, testCommands, parentTestStep, progressMonitor))
                    {
                        return monitor.Run();
                    }
                }
            }
        }