private static IList<TestData> Explore(IEnumerable<string> sources, ILogger frameworkLogger)
        {
            try
            {
                var testDriver = GetTestDriver(frameworkLogger);

                var tests = new List<TestData>();
                var messageConsumer = new MessageConsumer()
                    .Handle<TestDiscoveredMessage>(message =>
                    {
                        if (message.Test.IsTestCase)
                            tests.Add(message.Test);
                    })
                    .Handle<AnnotationDiscoveredMessage>(message => message.Annotation.Log(frameworkLogger, true));

                var loader = new ReflectionOnlyAssemblyLoader();
                var assemblyInfos = sources.Select(source => LoadAssembly(source, loader)).Where(assembly => assembly != null).ToList();
                var testExplorationOptions = new TestExplorationOptions();

                testDriver.Describe(loader.ReflectionPolicy, assemblyInfos, testExplorationOptions, messageConsumer, NullProgressMonitor.CreateInstance());

                return ResetCollectionForExposedTests(tests) ? null : tests;
            }
            catch (Exception ex)
            {
                frameworkLogger.Log(LogSeverity.Error, "Gallio failed to load tests", ex);
                return null;
            }
        }
 /// <inheritdoc />
 protected override void DescribeImpl(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements,
     TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
     using (progressMonitor.BeginTask("Describing unsupported tests.", Math.Max(codeElements.Count, 1)))
     {
         PublishTestModelFromCodeElements(codeElements, messageSink, progressMonitor);
     }
 }
 public void SetUp()
 {
     logger = MockRepository.GenerateStub<ILogger>();
     driver = new IronRubyTestDriver(logger);
     var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
     testIsolationContext = testIsolationProvider.CreateContext(new TestIsolationOptions(), logger);
     testPackage = new TestPackage();
     testExplorationOptions = new TestExplorationOptions();
     testExecutionOptions = new TestExecutionOptions();
     messageSink = MockRepository.GenerateStub<IMessageSink>();
     progressMonitor = NullProgressMonitor.CreateInstance();
 }
        /// <summary>
        /// Initializes the event arguments.
        /// </summary>
        /// <param name="testPackage">The test package configuration.</param>
        /// <param name="testExplorationOptions">The test exploration options.</param>
        /// <param name="reportLockBox">The report lock-box which may be used to access the report asynchronously during execution.</param>
        /// <exception cref="ArgumentNullException">Thrown if <paramref name="testPackage"/>
        /// or <paramref name="testExplorationOptions"/> is null.</exception>
        public ExploreStartedEventArgs(TestPackage testPackage, TestExplorationOptions testExplorationOptions,
            LockBox<Report> reportLockBox)
        {
            if (testPackage == null)
                throw new ArgumentNullException("testPackage");
            if (testExplorationOptions == null)
                throw new ArgumentNullException("testExplorationOptions");

            this.testPackage = testPackage;
            this.testExplorationOptions = testExplorationOptions;
            this.reportLockBox = reportLockBox;
        }
        /// <inheritdoc />
        public void Describe(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
        {
            if (reflectionPolicy == null)
                throw new ArgumentNullException("reflectionPolicy");
            if (codeElements == null || codeElements.Contains(null))
                throw new ArgumentNullException("codeElements");
            if (testExplorationOptions == null)
                throw new ArgumentNullException("testExplorationOptions");
            if (messageSink == null)
                throw new ArgumentNullException("messageSink");
            if (progressMonitor == null)
                throw new ArgumentNullException("progressMonitor");

            DescribeImpl(reflectionPolicy, codeElements, testExplorationOptions, messageSink, progressMonitor);
        }
        /// <inheritdoc />
        public void Explore(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
            IMessageSink messageSink, IProgressMonitor progressMonitor)
        {
            if (testIsolationContext == null)
                throw new ArgumentNullException("testIsolationContext");
            if (testPackage == null)
                throw new ArgumentNullException("testPackage");
            if (testExplorationOptions == null)
                throw new ArgumentNullException("testExplorationOptions");
            if (messageSink == null)
                throw new ArgumentNullException("messageSink");
            if (progressMonitor == null)
                throw new ArgumentNullException("progressMonitor");

            ExploreImpl(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
        }
Exemple #7
0
        public override ICollection Load(string location, ProjectData projectData, IWarningHandler warningHandler)
        {
            // Skip loading if the extension is not fully initalized unless we are not
            // running in Visual Studio (because we are running in MSTest instead).
            if (!TipShellExtension.IsInitialized && ShellEnvironment.IsRunningInVisualStudio)
                return EmptyArray<TestElement>.Instance;

            // Explore the tests.
            ITestFrameworkManager testFrameworkManager = RuntimeAccessor.ServiceLocator.Resolve<ITestFrameworkManager>();
            WarningLogger logger = new WarningLogger(warningHandler);

            ReflectionOnlyAssemblyLoader loader = new ReflectionOnlyAssemblyLoader();
            loader.AddHintDirectory(Path.GetDirectoryName(location));

            IAssemblyInfo assembly = loader.ReflectionPolicy.LoadAssemblyFrom(location);

            var testFrameworkSelector = new TestFrameworkSelector()
            {
                Filter = testFrameworkHandle => testFrameworkHandle.Id != "MSTestAdapter.TestFramework",
                FallbackMode = TestFrameworkFallbackMode.Approximate
            };

            ITestDriver driver = testFrameworkManager.GetTestDriver(testFrameworkSelector, logger);
            TestExplorationOptions testExplorationOptions = new TestExplorationOptions();

            ArrayList tests = new ArrayList();
            MessageConsumer messageConsumer = new MessageConsumer()
                .Handle<TestDiscoveredMessage>(message =>
                {
                    if (message.Test.IsTestCase)
                        tests.Add(GallioTestElementFactory.CreateTestElement(message.Test, location, projectData));
                })
                .Handle<AnnotationDiscoveredMessage>(message =>
                {
                    message.Annotation.Log(logger, true);
                });

            driver.Describe(loader.ReflectionPolicy, new ICodeElementInfo[] { assembly },
                testExplorationOptions, messageConsumer, NullProgressMonitor.CreateInstance());

            return tests;
        }
Exemple #8
0
            private void ExploreOrRun(Type driverType, object[] driverArguments,
                                      string assemblyPath, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
                                      IMessageSink messageSink, IProgressMonitor progressMonitor)
            {
                Assembly assembly = LoadAssembly(assemblyPath);

                using (var queuedMessageSink = new QueuedMessageSink(messageSink))
                {
                    var testDriver = (DotNetTestDriver)Activator.CreateInstance(driverType, driverArguments);

                    if (testExecutionOptions == null)
                    {
                        testDriver.ExploreAssembly(assembly, testExplorationOptions, queuedMessageSink, progressMonitor);
                    }
                    else
                    {
                        testDriver.RunAssembly(assembly, testExplorationOptions, testExecutionOptions, queuedMessageSink, progressMonitor);
                    }
                }
            }
            protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
            {
                using (progressMonitor.BeginTask("Running tests.", 1))
                {
                    if (testIsolationContext.RequiresSingleThreadedExecution && !testExecutionOptions.SingleThreaded)
                    {
                        testExecutionOptions = testExecutionOptions.Copy();
                        testExecutionOptions.SingleThreaded = true;
                    }

                    ForEachDriver(testFrameworkManager.SelectTestFrameworksForFilesImpl(
                                      testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, testPackage.Files),
                                  (driver, items, driverCount) =>
                    {
                        TestPackage testPackageForDriver = CreateTestPackageWithFiles(testPackage, items);
                        driver.Run(testIsolationContext, testPackageForDriver, testExplorationOptions, testExecutionOptions,
                                   messageSink,
                                   progressMonitor.CreateSubProgressMonitor(1.0 / driverCount));
                        return(false);
                    });
                }
            }
Exemple #10
0
 /// <summary>
 /// Runs tests in an assembly.
 /// </summary>
 /// <remarks>
 /// <para>
 /// The default implementation does nothing.  Subclasses may override to enable tests
 /// to be run.  This is required for actually running tests.
 /// </para>
 /// <para>
 /// This method executes within the test host which is most likely a different AppDomain
 /// or Process from the test runner itself.
 /// </para>
 /// </remarks>
 /// <param name="assembly">The test assembly, not null.</param>
 /// <param name="testExplorationOptions">The test exploration options, not null.</param>
 /// <param name="testExecutionOptions">The test execution options, not null.</param>
 /// <param name="messageSink">The message sink, not null.</param>
 /// <param name="progressMonitor">The progress monitor, not null.</param>
 protected virtual void RunAssembly(Assembly assembly, TestExplorationOptions testExplorationOptions,
                                    TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
 }
 /// <summary>
 /// Creates a copy of the options.
 /// </summary>
 /// <returns>The copy.</returns>
 public TestExplorationOptions Copy()
 {
     var copy = new TestExplorationOptions();
     copy.properties.AddAll(properties);
     return copy;
 }
 private static List<TestData> FindTests(TestPackage testPackage, ITestRunner runner)
 {
     var options = new TestExplorationOptions();
     testPackage.AddExcludedTestFrameworkId(MsTestFrameworkHandleId);
     testPackage.AddExcludedTestFrameworkId(NUnitTestFrameworkHandleId);
     var result = runner.Explore(testPackage, options, new ObservableProgressMonitor());
     var tests = result.TestModel.AllTests.Where(t => t.IsTestCase).ToList();
     return tests;
 }
Exemple #13
0
        /// <inheritdoc />
        public void Explore(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
                            IMessageSink messageSink, IProgressMonitor progressMonitor)
        {
            if (testIsolationContext == null)
            {
                throw new ArgumentNullException("testIsolationContext");
            }
            if (testPackage == null)
            {
                throw new ArgumentNullException("testPackage");
            }
            if (testExplorationOptions == null)
            {
                throw new ArgumentNullException("testExplorationOptions");
            }
            if (messageSink == null)
            {
                throw new ArgumentNullException("messageSink");
            }
            if (progressMonitor == null)
            {
                throw new ArgumentNullException("progressMonitor");
            }

            ExploreImpl(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
        }
Exemple #14
0
 /// <summary>
 /// Runs tests from a test package.
 /// </summary>
 /// <remarks>
 /// <para>
 /// The default implementation does nothing.  Subclasses may override to enable tests
 /// to be run.  This is required for actually running tests.
 /// </para>
 /// </remarks>
 /// <param name="testIsolationContext">The test isolation context, not null.</param>
 /// <param name="testPackage">The test package, not null.</param>
 /// <param name="testExplorationOptions">The test exploration options, not null.</param>
 /// <param name="testExecutionOptions">The test execution options, not null.</param>
 /// <param name="messageSink">The message sink to receive test exploration and execution messages, not null.</param>
 /// <param name="progressMonitor">The progress monitor, not null.</param>
 /// <returns>The test report.</returns>
 protected virtual void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
                                TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
                                IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
 }
 /// <inheritdoc />
 protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
     TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
     using (progressMonitor.BeginTask("Exploring unsupported tests.", Math.Max(testPackage.Files.Count, 1)))
     {
         PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor);
     }
 }
            private static void ExploreOrRun(TestPackage testPackage, ScriptRuntimeSetup scriptRuntimeSetup, string testDriverScriptPath,
                TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
                IMessageSink messageSink, IProgressMonitor progressMonitor, ILogger logger)
            {
                using (BufferedLogWriter outputWriter = new BufferedLogWriter(logger, LogSeverity.Info, Encoding.Default),
                    errorWriter = new BufferedLogWriter(logger, LogSeverity.Error, Encoding.Default))
                {
                    using (var queuedMessageSink = new QueuedMessageSink(messageSink))
                    {
                        using (new ConsoleRedirection(outputWriter, errorWriter))
                        {
                            var scriptRuntime = new ScriptRuntime(scriptRuntimeSetup);

                            scriptRuntime.IO.SetInput(Stream.Null, TextReader.Null, Encoding.Default);
                            scriptRuntime.IO.SetOutput(new TextWriterStream(outputWriter), outputWriter);
                            scriptRuntime.IO.SetErrorOutput(new TextWriterStream(errorWriter), errorWriter);

                            try
                            {
                                var scriptParameters = new Dictionary<string, object>();
                                scriptParameters.Add("Verb", testExecutionOptions != null ? "Run" : "Explore");
                                scriptParameters.Add("TestPackage", testPackage);
                                scriptParameters.Add("TestExplorationOptions", testExplorationOptions);
                                scriptParameters.Add("TestExecutionOptions", testExecutionOptions);
                                scriptParameters.Add("MessageSink", queuedMessageSink);
                                scriptParameters.Add("ProgressMonitor", progressMonitor);
                                scriptParameters.Add("Logger", logger);

                                scriptRuntime.Globals.SetVariable(ScriptParametersVariableName, scriptParameters);

                                RunScript(scriptRuntime, testDriverScriptPath);
                            }
                            finally
                            {
                                scriptRuntime.Shutdown();
                            }
                        }
                    }
                }
            }
        private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
            TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
        {
            using (progressMonitor.BeginTask(taskName, 1))
            {
                if (progressMonitor.IsCanceled)
                    return;

                FileInfo testDriverScriptFile = GetTestDriverScriptFile(testPackage);
                if (testDriverScriptFile == null)
                    return;

                HostSetup hostSetup = CreateHostSetup(testPackage);
                ScriptRuntimeSetup scriptRuntimeSetup = CreateScriptRuntimeSetup(testPackage);
                string testDriverScriptPath = testDriverScriptFile.FullName;
                var remoteMessageSink = new RemoteMessageSink(messageSink);
                var remoteLogger = new RemoteLogger(logger);

                using (var remoteProgressMonitor = new RemoteProgressMonitor(progressMonitor.CreateSubProgressMonitor(1)))
                {
                    testIsolationContext.RunIsolatedTask<ExploreOrRunTask>(hostSetup,
                        (statusMessage) => progressMonitor.SetStatus(statusMessage),
                        new object[] { testPackage, scriptRuntimeSetup, testDriverScriptPath, testExplorationOptions, testExecutionOptions,
                            remoteMessageSink, remoteProgressMonitor, remoteLogger });
                }
            }
        }
Exemple #18
0
        private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
                                  TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
        {
            using (progressMonitor.BeginTask(taskName, Math.Max(testPackage.Files.Count, 1)))
            {
                foreach (FileInfo file in testPackage.Files)
                {
                    if (progressMonitor.IsCanceled)
                    {
                        return;
                    }

                    RemoteMessageSink remoteMessageSink = new RemoteMessageSink(messageSink);
                    ExploreOrRunAssembly(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions,
                                         remoteMessageSink, progressMonitor.CreateSubProgressMonitor(1), taskName, file);
                }
            }
        }
Exemple #19
0
        private void ExploreOrRunAssembly(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
                                          TestExecutionOptions testExecutionOptions, RemoteMessageSink remoteMessageSink, IProgressMonitor progressMonitor, string taskName,
                                          FileInfo file)
        {
            using (progressMonitor.BeginTask(taskName, 100))
            {
                if (progressMonitor.IsCanceled)
                {
                    return;
                }

                string assemblyPath = file.FullName;
                progressMonitor.SetStatus("Getting test assembly metadata.");
                AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.RuntimeVersion);
                progressMonitor.Worked(2);

                if (progressMonitor.IsCanceled)
                {
                    return;
                }

                if (assemblyMetadata != null)
                {
                    Type     driverType      = GetType();
                    object[] driverArguments = GetRemoteTestDriverArguments();

                    HostSetup hostSetup = CreateHostSetup(testPackage, assemblyPath, assemblyMetadata);

                    using (var remoteProgressMonitor = new RemoteProgressMonitor(
                               progressMonitor.CreateSubProgressMonitor(97)))
                    {
                        testIsolationContext.RunIsolatedTask <ExploreOrRunTask>(hostSetup,
                                                                                (statusMessage) => progressMonitor.SetStatus(statusMessage),
                                                                                new object[] { driverType, driverArguments, assemblyPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor });
                    }

                    // Record one final work unit after the isolated task has been fully cleaned up.
                    progressMonitor.SetStatus("");
                    progressMonitor.Worked(1);
                }
            }
        }
 /// <summary>
 /// Runs tests from a test package.
 /// </summary>
 /// <remarks>
 /// <para>
 /// The default implementation does nothing.  Subclasses may override to enable tests
 /// to be run.  This is required for actually running tests.
 /// </para>
 /// </remarks>
 /// <param name="testIsolationContext">The test isolation context, not null.</param>
 /// <param name="testPackage">The test package, not null.</param>
 /// <param name="testExplorationOptions">The test exploration options, not null.</param>
 /// <param name="testExecutionOptions">The test execution options, not null.</param>
 /// <param name="messageSink">The message sink to receive test exploration and execution messages, not null.</param>
 /// <param name="progressMonitor">The progress monitor, not null.</param>
 /// <returns>The test report.</returns>
 protected virtual void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
     TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
     IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
 }
 /// <summary>
 /// Describes tests via reflection over code elements.
 /// </summary>
 /// <remarks>
 /// <para>
 /// The default implementation does nothing.  Subclasses may override to enable tests
 /// to be described via reflection only.  This is required for ReSharper integration.
 /// </para>
 /// </remarks>
 /// <param name="reflectionPolicy">The reflection policy, not null.</param>
 /// <param name="codeElements">The enumeration of code elements, usually <see cref="ITypeInfo" />
 /// and <see cref="IAssemblyInfo" /> objects, that might contain tests to be described, not null.</param>
 /// <param name="testExplorationOptions">The test exploration options, not null.</param>
 /// <param name="messageSink">The message sink to receive test exploration messages, not null.</param>
 /// <param name="progressMonitor">The progress monitor, not null.</param>
 protected virtual void DescribeImpl(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements,
     TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
 }
        /// <inheritdoc />
        protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
            TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
            IMessageSink messageSink, IProgressMonitor progressMonitor)
        {
            double workItems = Math.Max(testPackage.Files.Count, 1);
            using (progressMonitor.BeginTask("Running unsupported tests.", workItems * 2))
            {
                TestModel testModel = PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor);

                if (testModel != null)
                    RunTestModel(testModel, messageSink, progressMonitor, workItems);
            }
        }
 protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
     using (progressMonitor.BeginTask("Exploring tests.", 1))
     {
         ForEachDriver(testFrameworkManager.SelectTestFrameworksForFilesImpl(
                           testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, testPackage.Files),
                       (driver, items, driverCount) =>
         {
             TestPackage testPackageForDriver = CreateTestPackageWithFiles(testPackage, items);
             driver.Explore(testIsolationContext, testPackageForDriver, testExplorationOptions, messageSink,
                            progressMonitor.CreateSubProgressMonitor(1.0 / driverCount));
             return(false);
         });
     }
 }
 /// <inheritdoc />
 sealed protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
     ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, null, messageSink, progressMonitor,
         "Exploring tests.");
 }
Exemple #25
0
 /// <summary>
 /// Describes tests via reflection over code elements.
 /// </summary>
 /// <remarks>
 /// <para>
 /// The default implementation does nothing.  Subclasses may override to enable tests
 /// to be described via reflection only.  This is required for ReSharper integration.
 /// </para>
 /// </remarks>
 /// <param name="reflectionPolicy">The reflection policy, not null.</param>
 /// <param name="codeElements">The enumeration of code elements, usually <see cref="ITypeInfo" />
 /// and <see cref="IAssemblyInfo" /> objects, that might contain tests to be described, not null.</param>
 /// <param name="testExplorationOptions">The test exploration options, not null.</param>
 /// <param name="messageSink">The message sink to receive test exploration messages, not null.</param>
 /// <param name="progressMonitor">The progress monitor, not null.</param>
 protected virtual void DescribeImpl(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements,
                                     TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
 }
 /// <summary>
 /// Runs tests in an assembly.
 /// </summary>
 /// <remarks>
 /// <para>
 /// The default implementation does nothing.  Subclasses may override to enable tests
 /// to be run.  This is required for actually running tests.
 /// </para>
 /// <para>
 /// This method executes within the test host which is most likely a different AppDomain
 /// or Process from the test runner itself.
 /// </para>
 /// </remarks>
 /// <param name="assembly">The test assembly, not null.</param>
 /// <param name="testExplorationOptions">The test exploration options, not null.</param>
 /// <param name="testExecutionOptions">The test execution options, not null.</param>
 /// <param name="messageSink">The message sink, not null.</param>
 /// <param name="progressMonitor">The progress monitor, not null.</param>
 protected virtual void RunAssembly(Assembly assembly, TestExplorationOptions testExplorationOptions,
     TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
 }
Exemple #27
0
        /// <inheritdoc />
        public void Describe(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
        {
            if (reflectionPolicy == null)
            {
                throw new ArgumentNullException("reflectionPolicy");
            }
            if (codeElements == null || codeElements.Contains(null))
            {
                throw new ArgumentNullException("codeElements");
            }
            if (testExplorationOptions == null)
            {
                throw new ArgumentNullException("testExplorationOptions");
            }
            if (messageSink == null)
            {
                throw new ArgumentNullException("messageSink");
            }
            if (progressMonitor == null)
            {
                throw new ArgumentNullException("progressMonitor");
            }

            DescribeImpl(reflectionPolicy, codeElements, testExplorationOptions, messageSink, progressMonitor);
        }
        public IEnumerable<AutoTest.TestRunners.Shared.Results.TestResult> Run(RunSettings settings)
        {
            if (!_isInitialized)
                return new AutoTest.TestRunners.Shared.Results.TestResult[] { getNotInitializedResult(settings) };
            var tests = settings.Assembly.Tests.ToList();
            var members = settings.Assembly.Members.ToList();
            var namespaces = settings.Assembly.Namespaces.ToList();

            var runAll = namespaces.Count == 0 && members.Count == 0 && tests.Count == 0;
            var steps = new List<TestStepData>();
            var testResults = new List<AutoTest.TestRunners.Shared.Results.TestResult>();

            // Get a test isolation context.  Here we want to run tests in the same AppDomain.
            var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
            var testIsolationOptions = new TestIsolationOptions();
            ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, _logger);

            var testPackage = new TestPackage();
            testPackage.AddFile(new FileInfo(settings.Assembly.Assembly));
            testPackage.TestFrameworkFallbackMode = TestFrameworkFallbackMode.Strict;

            // Create some test exploration options.  Nothing interesting for you here, probably.
            var testExplorationOptions = new TestExplorationOptions();
            var messageSink = new MessageConsumer()
                .Handle<TestStepStartedMessage>((message) =>
                {
                    steps.Add(message.Step);
                })
                .Handle<TestStepFinishedMessage>(message =>
                {
                    var test = steps.FirstOrDefault(x => x.Id.Equals(message.StepId) && x.IsTestCase);
                    if (test == null)
                        return;
                    var fixture = string.Format("{0}.{1}", test.CodeReference.NamespaceName, steps.First(x => x.Id.Equals(test.ParentId)).Name);
                    testResults.Add(new AutoTest.TestRunners.Shared.Results.TestResult(
                        "MbUnit",
                        settings.Assembly.Assembly,
                        fixture,
                        message.Result.Duration.TotalMilliseconds,
                        string.Format("{0}.{1}", fixture, test.Name),
                        convertState(message.Result.Outcome.Status),
                        message.Result.Outcome.DisplayName));
                });

            // Provide a progress monitor.
            var logProgressMonitorProvider = new LogProgressMonitorProvider(_logger);
            var options = new TestExecutionOptions();
            options.FilterSet = new Gallio.Model.Filters.FilterSet<ITestDescriptor>(new OrFilter<ITestDescriptor>(getTestFilter(namespaces, members, tests)));

            // Run the tests.
            logProgressMonitorProvider.Run((progressMonitor) =>
            {
                _testDriver.Run(testIsolationContext, testPackage, testExplorationOptions, options, messageSink, progressMonitor);
            });

            return testResults;
        }
        /// <inheritdoc />
        public Report Run(TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IProgressMonitor progressMonitor)
        {
            if (testPackage == null)
                throw new ArgumentNullException("testPackageConfig");
            if (testExplorationOptions == null)
                throw new ArgumentNullException("testExplorationOptions");
            if (testExecutionOptions == null)
                throw new ArgumentNullException("testExecutionOptions");
            if (progressMonitor == null)
                throw new ArgumentNullException("progressMonitor");

            ThrowIfDisposed();
            if (state != State.Initialized)
                throw new InvalidOperationException("The test runner must be initialized before this operation is performed.");

            testPackage = testPackage.Copy();
            testExplorationOptions = testExplorationOptions.Copy();
            testExecutionOptions = testExecutionOptions.Copy();
            GenericCollectionUtils.ForEach(testRunnerOptions.Properties, x => testPackage.AddProperty(x.Key, x.Value));

            using (progressMonitor.BeginTask("Running the tests.", 10))
            {
                Stopwatch stopwatch = Stopwatch.StartNew();
                Report report = new Report()
                {
                    TestPackage = new TestPackageData(testPackage),
                    TestModel = new TestModelData(),
                    TestPackageRun = new TestPackageRun()
                    {
                        StartTime = DateTime.Now
                    }
                };
                var reportLockBox = new LockBox<Report>(report);

                eventDispatcher.NotifyRunStarted(new RunStartedEventArgs(testPackage, testExplorationOptions,
                    testExecutionOptions, reportLockBox));

                bool success;
                using (Listener listener = new Listener(eventDispatcher, tappedLogger, reportLockBox))
                {
                    try
                    {
                        ITestDriver testDriver = testFrameworkManager.GetTestDriver(
                            testPackage.CreateTestFrameworkSelector(), tappedLogger);

                        using (testIsolationContext.BeginBatch(progressMonitor.SetStatus))
                        {
                            testDriver.Run(testIsolationContext, testPackage, testExplorationOptions,
                                testExecutionOptions, listener, progressMonitor.CreateSubProgressMonitor(10));
                        }

                        success = true;
                    }
                    catch (Exception ex)
                    {
                        success = false;

                        tappedLogger.Log(LogSeverity.Error,
                            "A fatal exception occurred while running tests.  Possible causes include invalid test runner parameters and stack overflows.",
                            ex);
                        report.TestModel.Annotations.Add(new AnnotationData(AnnotationType.Error,
                            CodeLocation.Unknown, CodeReference.Unknown,
                            "A fatal exception occurred while running tests.  See log for details.", null));
                    }
                    finally
                    {
                        report.TestPackageRun.EndTime = DateTime.Now;
                        report.TestPackageRun.Statistics.Duration = stopwatch.Elapsed.TotalSeconds;
                    }
                }

                eventDispatcher.NotifyRunFinished(new RunFinishedEventArgs(success, report));

                return report;
            }
        }
Exemple #30
0
 /// <inheritdoc />
 sealed protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
                                        TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
     ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, messageSink,
                  progressMonitor, "Running tests.");
 }
        private void RunAllTests(IRunContext runContext)
        {
            ITestRunnerManager runnerManager = RuntimeAccessor.ServiceLocator.Resolve<ITestRunnerManager>();
            var runner = runnerManager.CreateTestRunner(StandardTestRunnerFactoryNames.IsolatedAppDomain);
            runner.RegisterExtension(new RunContextExtension(runContext));

            ILogger logger = new RunContextLogger(runContext);
            TestRunnerOptions testRunnerOptions = new TestRunnerOptions();

            try
            {
                RunWithProgressMonitor(delegate(IProgressMonitor progressMonitor)
                {
                    runner.Initialize(testRunnerOptions, logger, progressMonitor);
                });

                if (isCanceled)
                    return;

                TestPackage testPackage = new TestPackage();
                testPackage.AddExcludedTestFrameworkId("MSTestAdapter.TestFramework");

                foreach (ITestElement testElement in runContext.RunConfig.TestElements)
                {
                    GallioTestElement gallioTestElement = testElement as GallioTestElement;
                    if (gallioTestElement != null)
                    {
                        testPackage.AddFile(new FileInfo(gallioTestElement.AssemblyPath));
                    }
                }

                TestExplorationOptions testExplorationOptions = new TestExplorationOptions();
                TestExecutionOptions testExecutionOptions = new TestExecutionOptions();

                List<Filter<string>> idFilters = new List<Filter<string>>();
                foreach (ITestElement includedTestElement in runContext.RunConfig.TestElements)
                {
                    GallioTestElement gallioTestElement = includedTestElement as GallioTestElement;
                    if (gallioTestElement != null)
                        idFilters.Add(new EqualityFilter<string>(gallioTestElement.GallioTestId));
                }

                testExecutionOptions.FilterSet = new FilterSet<ITestDescriptor>(new IdFilter<ITestDescriptor>(new OrFilter<string>(idFilters)));

                RunWithProgressMonitor(delegate(IProgressMonitor progressMonitor)
                {
                    runner.Run(testPackage, testExplorationOptions, testExecutionOptions, progressMonitor);
                });
            }
            finally
            {
                runner.Dispose(NullProgressMonitor.CreateInstance());
            }
        }
    TestModel PopulateTestTree(Assembly assembly)
    {
      TestModel testModel = new TestModel();

      var testFrameworkManager = RuntimeAccessor.ServiceLocator.Resolve<ITestFrameworkManager>();
      var logger = new MarkupStreamLogger(TestLog.Default);

      var testFrameworkSelector = new TestFrameworkSelector()
      {
        Filter = testFrameworkHandle => testFrameworkHandle.Id == TestFrameworkHandle.Id,
        FallbackMode = TestFrameworkFallbackMode.Strict
      };

      ITestDriver testDriver = testFrameworkManager.GetTestDriver(testFrameworkSelector, logger);

      var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
      var testIsolationOptions = new TestIsolationOptions();
      using (ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, logger))
      {
        var testPackage = new TestPackage();
        testPackage.AddFile(new FileInfo(AssemblyUtils.GetFriendlyAssemblyCodeBase(assembly)));
        var testExplorationOptions = new TestExplorationOptions();

        var messageSink = TestModelSerializer.CreateMessageSinkToPopulateTestModel(testModel);

        new LogProgressMonitorProvider(logger).Run(progressMonitor =>
        {
          testDriver.Explore(testIsolationContext, testPackage, testExplorationOptions,
            messageSink, progressMonitor);
        });
      }

      return testModel;
    }
 /// <inheritdoc />
 protected override sealed void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
     TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
 {
     ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, messageSink,
         progressMonitor, "Running tests.");
 }
 protected override void DescribeImpl(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, Gallio.Runtime.ProgressMonitoring.IProgressMonitor progressMonitor)
 {
     using (progressMonitor.BeginTask("Describing tests.", 1))
     {
         ForEachDriver(testFrameworkManager.SelectTestFrameworksForCodeElementsImpl(
                           testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, codeElements),
                       (driver, items, driverCount) =>
         {
             driver.Describe(reflectionPolicy, items, testExplorationOptions, messageSink,
                             progressMonitor.CreateSubProgressMonitor(1.0 / driverCount));
             return(false);
         });
     }
 }
        private List<Test> getTests(string assembly)
        {
            var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
            var testIsolationOptions = new TestIsolationOptions();
            ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, _logger);

            // Create a test package.
            // You can set a whole bunch of options here.
            var testPackage = new TestPackage();
            testPackage.AddFile(new FileInfo(assembly));
            testPackage.TestFrameworkFallbackMode = TestFrameworkFallbackMode.Strict;

            var testExplorationOptions = new TestExplorationOptions();

            // This query you can answer by exploring tests and looking at their metadata for "TestsOn".
            // You can explore tests using the Explore method of TestDriver. It sends a bunch of
            // messages to an IMessageSink just like Run.  Just scan these messages for tests that
            // match the pattern you like.

            // Alternately, you can build a TestModel once up front and traverse the resulting test tree
            // to find what you need.  The Explore method of testDriver is just like Run, except that
            // it does not run the tests.
            // TestModelSerializer is a little helper we can use to build up a TestModel from test messages on the fly.
            // The TestModel is just a tree of test metadata.  Pretty straightforward.
            TestModel testModel = new TestModel();
            IMessageSink messageSink = TestModelSerializer.CreateMessageSinkToPopulateTestModel(testModel);

            var logProgressMonitorProvider = new LogProgressMonitorProvider(_logger);

            logProgressMonitorProvider.Run((progressMonitor) =>
            {
                _testDriver.Explore(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
            });

            return testModel.AllTests.Where(x => x.IsTestCase).ToList();
        }
        private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
            TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
        {
            using (progressMonitor.BeginTask(taskName, Math.Max(testPackage.Files.Count, 1)))
            {
                foreach (FileInfo file in testPackage.Files)
                {
                    if (progressMonitor.IsCanceled)
                        return;

                    RemoteMessageSink remoteMessageSink = new RemoteMessageSink(messageSink);
                    ExploreOrRunAssembly(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions,
                        remoteMessageSink, progressMonitor.CreateSubProgressMonitor(1), taskName, file);
                }
            }
        }
 private void Describe(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements, ConsumerAdapter consumerAdapter)
 {
     var testDriver = CreateTestDriver();
     var testExplorationOptions = new TestExplorationOptions();
     testDriver.Describe(reflectionPolicy, codeElements, testExplorationOptions, consumerAdapter,
         NullProgressMonitor.CreateInstance());
 }
        private void ExploreOrRunAssembly(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
            TestExecutionOptions testExecutionOptions, RemoteMessageSink remoteMessageSink, IProgressMonitor progressMonitor, string taskName,
            FileInfo file)
        {
            using (progressMonitor.BeginTask(taskName, 100))
            {
                if (progressMonitor.IsCanceled)
                    return;

                string assemblyPath = file.FullName;
                progressMonitor.SetStatus("Getting test assembly metadata.");
                AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.RuntimeVersion);
                progressMonitor.Worked(2);

                if (progressMonitor.IsCanceled)
                    return;

                if (assemblyMetadata != null)
                {
                    Type driverType = GetType();
                    object[] driverArguments = GetRemoteTestDriverArguments();

                    HostSetup hostSetup = CreateHostSetup(testPackage, assemblyPath, assemblyMetadata);

                    using (var remoteProgressMonitor = new RemoteProgressMonitor(
                        progressMonitor.CreateSubProgressMonitor(97)))
                    {
                        testIsolationContext.RunIsolatedTask<ExploreOrRunTask>(hostSetup,
                            (statusMessage) => progressMonitor.SetStatus(statusMessage),
                            new object[] { driverType, driverArguments, assemblyPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor });
                    }

                    // Record one final work unit after the isolated task has been fully cleaned up.
                    progressMonitor.SetStatus("");
                    progressMonitor.Worked(1);
                }
            }
        }
            private void ExploreOrRun(Type driverType, object[] driverArguments,
                string assemblyPath, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
                IMessageSink messageSink, IProgressMonitor progressMonitor)
            {
                Assembly assembly = LoadAssembly(assemblyPath);

                using (var queuedMessageSink = new QueuedMessageSink(messageSink))
                {
                    var testDriver = (DotNetTestDriver) Activator.CreateInstance(driverType, driverArguments);

                    if (testExecutionOptions == null)
                        testDriver.ExploreAssembly(assembly, testExplorationOptions, queuedMessageSink, progressMonitor);
                    else
                        testDriver.RunAssembly(assembly, testExplorationOptions, testExecutionOptions, queuedMessageSink, progressMonitor);
                }
            }
        private FacadeTaskResult RunTests()
        {
            var logger = new FacadeLoggerWrapper(facadeLogger);
            var runner = TestRunnerUtils.CreateTestRunnerByName(StandardTestRunnerFactoryNames.IsolatedAppDomain);

            // Set parameters.
            var testPackage = new TestPackage();
            foreach (var assemblyLocation in assemblyLocations)
                testPackage.AddFile(new FileInfo(assemblyLocation));

            testPackage.ShadowCopy = facadeTaskExecutorConfiguration.ShadowCopy;

            if (facadeTaskExecutorConfiguration.AssemblyFolder != null)
            {
                testPackage.ApplicationBaseDirectory = new DirectoryInfo(facadeTaskExecutorConfiguration.AssemblyFolder);
                testPackage.WorkingDirectory = new DirectoryInfo(facadeTaskExecutorConfiguration.AssemblyFolder);
            }

            var testRunnerOptions = new TestRunnerOptions();

            var testExplorationOptions = new TestExplorationOptions();

            var filters = GenericCollectionUtils.ConvertAllToArray<string, Filter<string>>(explicitTestIds, 
                testId => new EqualityFilter<string>(testId));
            var filterSet = new FilterSet<ITestDescriptor>(new IdFilter<ITestDescriptor>(new OrFilter<string>(filters)));
            var testExecutionOptions = new TestExecutionOptions { FilterSet = filterSet };

            // Install the listeners.
            runner.Events.TestStepStarted += TestStepStarted;
            runner.Events.TestStepFinished += TestStepFinished;
            runner.Events.TestStepLifecyclePhaseChanged += TestStepLifecyclePhaseChanged;

            // Run the tests.
            try
            {
                try
                {
                    runner.Initialize(testRunnerOptions, logger, CreateProgressMonitor());
                    Report report = runner.Run(testPackage, testExplorationOptions, testExecutionOptions, CreateProgressMonitor());

                    if (sessionId != null)
                        SessionCache.SaveSerializedReport(sessionId, report);

                    return FacadeTaskResult.Success;
                }
                catch (Exception ex)
                {
                    if (sessionId != null)
                        SessionCache.ClearSerializedReport(sessionId);

                    logger.Log(LogSeverity.Error, "A fatal exception occurred during test execution.", ex);
                    return FacadeTaskResult.Exception;
                }
                finally
                {
                    SubmitFailureForRemainingPendingTasks();
                }
            }
            finally
            {
                runner.Dispose(CreateProgressMonitor());
            }
        }