/// <summary>
        /// Runs the tests.
        /// </summary>
        /// <param name="tests">Which tests should be run.</param>
        /// <param name="context">Context in which to run tests.</param>
        /// <param param name="framework">Where results should be stored.</param>
        public void RunTests(IEnumerable<TestCase> tests, IRunContext context, IFrameworkHandle framework)
        {
            _state = ExecutorState.Running;

            foreach (var test in tests)
            {
                if (_state == ExecutorState.Cancelling)
                {
                    _state = ExecutorState.Cancelled;
                    return;
                }

                try
                {
                    var reportDocument = RunOrDebugCatchTest(test.Source, test.FullyQualifiedName, context, framework);
                    var result = GetTestResultFromReport(test, reportDocument, framework);
                    framework.RecordResult(result);
                }
                catch (Exception ex)
                {
                    // Log it and move on. It will show up to the user as a test that hasn't been run.
                    framework.SendMessage(TestMessageLevel.Error, "Exception occured when processing test case: " + test.FullyQualifiedName);
                    framework.SendMessage(TestMessageLevel.Informational, "Message: " + ex.Message + "\nStacktrace:" + ex.StackTrace);
                }
            }
        }
        // NOTE: an earlier version of this code had a FilterBuilder
        // property. This seemed to make sense, because we instantiate
        // it in two different places. However, the existence of an
        // NUnitTestFilterBuilder, containing a reference to an engine 
        // service caused our second-level tests of the test executor
        // to throw an exception. So if you consider doing this, beware!

        #endregion

        #region ITestExecutor Implementation

        /// <summary>
        /// Called by the Visual Studio IDE to run all tests. Also called by TFS Build
        /// to run either all or selected tests. In the latter case, a filter is provided
        /// as part of the run context.
        /// </summary>
        /// <param name="sources">Sources to be run.</param>
        /// <param name="runContext">Context to use when executing the tests.</param>
        /// <param name="frameworkHandle">Test log to send results and messages through</param>
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
#if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
                Debugger.Launch();
#endif
            Initialize(runContext, frameworkHandle);

            try
            {
                foreach (var source in sources)
                {
                    var assemblyName = source;
                    if (!Path.IsPathRooted(assemblyName))
                        assemblyName = Path.Combine(Environment.CurrentDirectory, assemblyName);

                    TestLog.Info("Running all tests in " + assemblyName);

                    RunAssembly(assemblyName, TestFilter.Empty);
                }
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                    ex = ex.InnerException;
                TestLog.Error("Exception thrown executing tests", ex);
            }
            finally
            {
                TestLog.Info(string.Format("NUnit Adapter {0}: Test execution complete", AdapterVersion));
                Unload();
            }

        }
        bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression filter)
        {
            filter = null;

            try
            {
                // In Microsoft.VisualStudio.TestPlatform.ObjectModel V11 IRunContext provides a TestCaseFilter property
                // GetTestCaseFilter only exists in V12+
#if PLATFORM_DOTNET
                var getTestCaseFilterMethod = runContext.GetType().GetRuntimeMethod("GetTestCaseFilter", new[] { typeof(IEnumerable<string>), typeof(Func<string, TestProperty>) });
#else
                var getTestCaseFilterMethod = runContext.GetType().GetMethod("GetTestCaseFilter");
#endif
                if (getTestCaseFilterMethod != null)
                    filter = (ITestCaseFilterExpression)getTestCaseFilterMethod.Invoke(runContext, new object[] { supportedPropertyNames, null });

                return true;
            }
            catch (TargetInvocationException e)
            {
                var innerExceptionType = e.InnerException.GetType();
                if (innerExceptionType.FullName.EndsWith("TestPlatformFormatException", StringComparison.OrdinalIgnoreCase))
                {
                    logger.LogError("{0}: Exception discovering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.InnerException.Message);
                    return false;
                }

                throw;
            }
        }
Esempio n. 4
0
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            Guard.ArgumentNotNull("sources", sources);
            Guard.ArgumentNotNull("runContext", runContext);
            Guard.ArgumentNotNull("frameworkHandle", frameworkHandle);

            var cleanupList = new List<ExecutorWrapper>();

            try
            {
                RemotingUtility.CleanUpRegisteredChannels();

                cancelled = false;

                foreach (string source in sources)
                    if (VsTestRunner.IsXunitTestAssembly(source))
                        RunTestsInAssembly(cleanupList, source, frameworkHandle);
            }
            finally
            {
                Thread.Sleep(1000);

                foreach (var executorWrapper in cleanupList)
                    executorWrapper.Dispose();
            }
        }
 public void RunTests(IEnumerable<string> sources, IRunContext runContext,
     IFrameworkHandle frameworkHandle)
 {
     SetupExecutionPolicy();
     IEnumerable<TestCase> tests = PowerShellTestDiscoverer.GetTests(sources, null);
     RunTests(tests, runContext, frameworkHandle);
 }
        public TestCaseFilter(IRunContext runContext, ISet<string> traitNames, TestEnvironment testEnvironment)
        {
            _runContext = runContext;
            _testEnvironment = testEnvironment;

            InitProperties(traitNames);
        }
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            //Debugger.Launch();

            frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING);

            Settings settings = GetSettings(runContext);

            foreach (string currentAsssembly in sources.Distinct())
            {
                try
                {
                    if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(currentAsssembly)),"Machine.Specifications.dll")))
                    {
                        frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Machine.Specifications.dll not found for {0}", currentAsssembly));
                        continue;
                    }

                    frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly));

                    this.executor.RunAssembly(currentAsssembly, settings, uri, frameworkHandle);
                }
                catch (Exception ex)
                {
                    frameworkHandle.SendMessage(TestMessageLevel.Error, String.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message));
                }
            }

            frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Complete on {0} assemblies ", sources.Count()));
            
        }
        public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            //Debugger.Launch();

            frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING);

            int executedSpecCount = 0;

            Settings settings = GetSettings(runContext);

            string currentAsssembly = string.Empty;
            try
            {

                foreach (IGrouping<string, TestCase> grouping in tests.GroupBy(x => x.Source)) {
                    currentAsssembly = grouping.Key;
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly));

                    List<VisualStudioTestIdentifier> testsToRun = grouping.Select(test => test.ToVisualStudioTestIdentifier()).ToList();

                    this.executor.RunAssemblySpecifications(currentAsssembly, testsToRun, settings, uri, frameworkHandle);
                    executedSpecCount += grouping.Count();
                }

                frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_COMPLETE, executedSpecCount, tests.GroupBy(x => x.Source).Count()));
            } catch (Exception ex)
            {
                frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message));
            }
            finally
            {
            }
        }
Esempio n. 9
0
        public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            this.frameworkHandle = frameworkHandle;

            var testLogger = new TestLogger(frameworkHandle);

            testLogger.SendMainMessage("Execution started");

            foreach (var group in tests.GroupBy(t => t.Source))
            {
                testLogger.SendInformationalMessage(String.Format("Running selected: '{0}'", group.Key));

                try
                {
                    using (var sandbox = new Sandbox<Executor>(group.Key))
                    {
                        var assemblyDirectory = new DirectoryInfo(Path.GetDirectoryName(group.Key));
                        Directory.SetCurrentDirectory(assemblyDirectory.FullName);
                        sandbox.Content.Execute(this, group.Select(t => t.FullyQualifiedName).ToArray());
                    }
                }
                catch (Exception ex)
                {
                    testLogger.SendErrorMessage(ex, String.Format("Exception found while executing tests in group '{0}'", group.Key));

                    // just go on with the next
                }
            }

            testLogger.SendMainMessage("Execution finished");
        }
Esempio n. 10
0
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            this.frameworkHandle = frameworkHandle;

            var testLogger = new TestLogger(frameworkHandle);

            testLogger.SendMainMessage("Execution started");

            foreach (var source in sources)
            {
                try
                {
                    using (var sandbox = new Sandbox<Executor>(source))
                    {
                        testLogger.SendInformationalMessage(String.Format("Running: '{0}'", source));

                        var assemblyDirectory = new DirectoryInfo(Path.GetDirectoryName(source));
                        Directory.SetCurrentDirectory(assemblyDirectory.FullName);
                        sandbox.Content.Execute(this);
                    }
                }
                catch (Exception ex)
                {
                    testLogger.SendErrorMessage(ex, String.Format("Exception found while executing tests in source '{0}'", source));

                    // just go on with the next
                }
            }

            testLogger.SendMainMessage("Execution finished");
        }
Esempio n. 11
0
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            IMessageLogger log = frameworkHandle;

            log.Version();

            HandlePoorVisualStudioImplementationDetails(runContext, frameworkHandle);

            foreach (var assemblyPath in sources)
            {
                try
                {
                    if (AssemblyDirectoryContainsFixie(assemblyPath))
                    {
                        log.Info("Processing " + assemblyPath);

                        var listener = new VisualStudioListener(frameworkHandle, assemblyPath);

                        using (var environment = new ExecutionEnvironment(assemblyPath))
                        {
                            environment.RunAssembly(new Options(), listener);
                        }
                    }
                    else
                    {
                        log.Info("Skipping " + assemblyPath + " because it is not a test assembly.");
                    }
                }
                catch (Exception exception)
                {
                    log.Error(exception);
                }
            }
        }
        /// <summary>
        /// Called by the Visual Studio IDE to run all tests. Also called by TFS Build
        /// to run either all or selected tests. In the latter case, a filter is provided
        /// as part of the run context.
        /// </summary>
        /// <param name="sources">Sources to be run.</param>
        /// <param name="runContext">Context to use when executing the tests.</param>
        /// <param name="frameworkHandle">Test log to send results and messages through</param>
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            testLog.Initialize(frameworkHandle);
            Info("executing tests", "started");

            try
            {
                // Ensure any channels registered by other adapters are unregistered
                CleanUpRegisteredChannels();

                var tfsfilter = new TFSTestFilter(runContext);
                testLog.SendDebugMessage("Keepalive:" + runContext.KeepAlive);
                if (!tfsfilter.HasTfsFilterValue && runContext.KeepAlive)
                    frameworkHandle.EnableShutdownAfterTestRun = true;

                foreach (var source in sources)
                {
                    using (currentRunner = new AssemblyRunner(testLog, source, tfsfilter))
                    {
                        currentRunner.RunAssembly(frameworkHandle);
                    }

                    currentRunner = null;
                }
            }
            catch (Exception ex)
            {
                testLog.SendErrorMessage("Exception " + ex);
            }
            finally
            {
                Info("executing tests", "finished");
            }
        }
        /// <summary>
        /// Runs the tests.
        /// </summary>
        /// <param name="tests">Tests to be run.</param>
        /// <param name="runContext">Context to use when executing the tests.</param>
        /// <param param name="frameworkHandle">Handle to the framework to record results and to do framework operations.</param>
        public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            m_cancelled = false;
            try
            {
                foreach (TestCase test in tests)
                {
                    if (m_cancelled)
                    {
                        break;
                    }
                    frameworkHandle.RecordStart(test);
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, "Starting external test for " + test.DisplayName);
                    var testOutcome = RunExternalTest(test, runContext, frameworkHandle, test);
                    frameworkHandle.RecordResult(testOutcome);
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, "Test result:" + testOutcome.Outcome.ToString());


                }
            }
            catch(Exception e)
            {
                frameworkHandle.SendMessage(TestMessageLevel.Error, "Exception during test execution: " +e.Message);
            }
}
        public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING);
            int executedSpecCount = 0;
            string currentAsssembly = string.Empty;
            try
            {
                ISpecificationExecutor specificationExecutor = this.adapterFactory.CreateExecutor();
                IEnumerable<IGrouping<string, TestCase>> groupBySource = tests.GroupBy(x => x.Source);
                foreach (IGrouping<string, TestCase> grouping in groupBySource)
                {
                    currentAsssembly = grouping.Key;
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly));
                    specificationExecutor.RunAssemblySpecifications(currentAsssembly, MSpecTestAdapter.uri, runContext, frameworkHandle, grouping);
                    executedSpecCount += grouping.Count();
                }

                frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_COMPLETE, executedSpecCount, groupBySource.Count()));
            }
            catch (Exception ex)
            {
                frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message));
            }
            finally
            {
            }
        }
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            ChutzpahTracer.TraceInformation("Begin Test Adapter Run Tests");

            var settingsProvider = runContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider;
            var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings();

            ChutzpahTracingHelper.Toggle(settings.EnabledTracing);

            var testOptions = new TestOptions
                {
                    TestLaunchMode =
                        runContext.IsBeingDebugged ? TestLaunchMode.Custom:
                        settings.OpenInBrowser ? TestLaunchMode.FullBrowser:
                        TestLaunchMode.HeadlessBrowser,
                    CustomTestLauncher     = runContext.IsBeingDebugged ? new VsDebuggerTestLauncher() : null,
                    MaxDegreeOfParallelism = runContext.IsBeingDebugged ? 1 : settings.MaxDegreeOfParallelism,
                    ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments)
                };

            testOptions.CoverageOptions.Enabled = runContext.IsDataCollectionEnabled;

            var callback = new ParallelRunnerCallbackAdapter(new ExecutionCallback(frameworkHandle, runContext));
            testRunner.RunTests(sources, testOptions, callback);

            ChutzpahTracer.TraceInformation("End Test Adapter Run Tests");

        }
 public void Cleanup()
 {
     _runContext = null;
     foreach (var adapter in GetAdapters()) {
         adapter.Cleanup();
     }
     _adapters.Clear();
 }
Esempio n. 17
0
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            List<TestCase> tests = new List<TestCase>();

            TestDiscoverer.VisitTests(sources, t => tests.Add(t));

            InternalRunTests(tests, runContext, frameworkHandle, null);
        }
		public ServersTree(IRunContext runContext)
		{
			Context = runContext;
			Children = new ObservableCollection<Node>
			{
				new LocalHostNode(this),
			};
		}
Esempio n. 19
0
        public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) {
            ValidateArg.NotNull(tests, "tests");
            ValidateArg.NotNull(runContext, "runContext");
            ValidateArg.NotNull(frameworkHandle, "frameworkHandle");

            _cancelRequested.Reset();

            RunTestCases(tests, runContext, frameworkHandle);
        }
Esempio n. 20
0
 private void RunTests(string source, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     foreach (var result in ExternalTestExecutor.GetTestResults(source, null).Select(c => CreateTestResult(source, c)))
     {
         frameworkHandle.RecordStart(result.TestCase);
         frameworkHandle.RecordResult(result);
         frameworkHandle.RecordEnd(result.TestCase, result.Outcome);
     }
 }
Esempio n. 21
0
 public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     RunTests(sources.Select(x => new TestCase(x, ExecutorDetails.Uri, x)
     {
         CodeFilePath = x,
         LineNumber = 2,
         DisplayName = "Abra cadabra"
     }), runContext, frameworkHandle);
 }
Esempio n. 22
0
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder)
        {
            Guard.ArgumentNotNull("sources", sources);
            Guard.ArgumentNotNull("runContext", runContext);
            Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder);

            foreach (string source in sources)
                if (VsTestRunner.IsXunitTestAssembly(source))
                    RunTestsInAssembly(source, runContext, testExecutionRecorder);
        }
Esempio n. 23
0
        public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder)
        {
            Guard.ArgumentNotNull("tests", tests);
            Guard.ArgumentNotNull("runContext", runContext);
            Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder);

            foreach (var testCaseGroup in tests.GroupBy(tc => tc.Source))
                if (VsTestRunner.IsXunitTestAssembly(testCaseGroup.Key))
                    RunTestsInAssembly(testCaseGroup.Key, runContext, testExecutionRecorder, testCaseGroup);
        }
        private void RunTests(IRunContext runContext, IFrameworkHandle testExecutionRecorder)
        {
            if (runContext.InIsolation)
                launcher.TestProject.TestRunnerFactoryName = StandardTestRunnerFactoryNames.IsolatedAppDomain;

            var extension = new VSTestWindowExtension(testExecutionRecorder, testCaseFactory, testResultFactory);

            launcher.TestProject.AddTestRunnerExtension(extension);
            launcher.Run();
        }
 public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     //Debugger.Launch();
     // this a temporary hack until I can figure out why running the specs per assembly directly using mspec does not work with a large number of specifications
     // when they are run diectly the first 100 or so specs run fine and then an error occurs saying it has taken more than 10 seconds and is being stopped
     // for now we just rediscover and run them like that, makes no sense
     TestCaseCollector collector = new TestCaseCollector();
     this.DiscoverTests(sources, runContext, frameworkHandle, collector);
     this.RunTests(collector.TestCases, runContext, frameworkHandle);
 }
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            var validSources = from source in sources
                               where source.EndsWith(StringHelper.GetSearchExpression(), StringComparison.CurrentCultureIgnoreCase)
                               select source;

            foreach (var source in validSources) {
                RunTest(frameworkHandle, source);
            }
        }
Esempio n. 27
0
 public TestGeneratorAdapterTests()
 {
     testGeneratorDiscoverer = Substitute.For<ITestGeneratorDiscoverer>();
     discoveryContext = Substitute.For<IDiscoveryContext>();
     discoverySink = Substitute.For<ITestCaseDiscoverySink>();
     logger = Substitute.For<IMessageLogger>();
     frameworkHandle = Substitute.For<IFrameworkHandle>();
     runContext = Substitute.For<IRunContext>();
     testGenerator = Substitute.For<ITestGenerator>();
 }
        public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle testExecutionRecorder)
        {
            launcher = new TestLauncher();

            foreach (var source in sources)
            {
                launcher.AddFilePattern(source);
            }

            RunTests(runContext, testExecutionRecorder);
        }
Esempio n. 29
0
        private static IEnumerable<KarmaTestResult> GetTestResults(IRunContext runContext)
        {
            var firstResultFile = Directory.EnumerateFiles(Path.Combine(@"C:\temp", "test-results"), "*.xml").First();

            string outputResults = File.ReadAllText(firstResultFile);

            var karmaResultsParser = new JUnitResultsParser();

            IEnumerable<KarmaTestResult> testResult = karmaResultsParser.Parse(outputResults);

            return testResult;
        }
        void ITestAdapter.Initialize(IRunContext runContext)
        {
            // We delay inner TAs initialization until Run method because we don't know which test type this is going to be.
            m_runContext = runContext;

            TestRunConfiguration runConfig = m_runContext.RunConfig.TestRun.RunConfiguration;
            Debug.Assert(runConfig != null);
            VsIdeHostRunConfigData runConfigHostData = runConfig.HostData[VsIdeHostAdapter.Name] as VsIdeHostRunConfigData;
            if (runConfigHostData != null)
            {
                VsIdeTestHostContext.AdditionalTestData = runConfigHostData.AdditionalTestData;
            }
        }
 private static XDocument RunOrDebugCatchTest(string testBinary, string testSpec, IRunContext runContext, IFrameworkHandle framework)
 {
     if (runContext.IsBeingDebugged)
     {
         return(XDocument.Parse(DebugCatchTest(testBinary, testSpec, framework)));
     }
     else
     {
         return(XDocument.Parse(RunCatchTests(testBinary, testSpec)));
     }
 }
Esempio n. 32
0
        public void RunTests(IEnumerable <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            // We'll just punt and run everything in each file that contains the selected tests
            var sources = tests.Select(test => test.Source).Distinct();

            RunTests(sources, runContext, frameworkHandle);
        }
 public void Initialize(IRunContext runContext)
 {
     _runContext = runContext;
     _runId      = runContext.RunConfig.TestRun.Id;
 }
Esempio n. 34
0
 public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     RunTestsWithSourcesCallback?.Invoke(sources, runContext, frameworkHandle);
 }
Esempio n. 35
0
 public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     // HACK: For now, just run them all - we can fix this later :)
     RunTests(tests.Select(t => t.Source).Distinct(), runContext, frameworkHandle);
 }
Esempio n. 36
0
        void RunTests(IRunContext runContext, IFrameworkHandle frameworkHandle, Stopwatch stopwatch, Func <XunitVisualStudioSettings, IEnumerable <IGrouping <string, TestCase> > > testCaseAccessor)
        {
            Guard.ArgumentNotNull("runContext", runContext);
            Guard.ArgumentNotNull("frameworkHandle", frameworkHandle);

            var settings     = SettingsProvider.Load();
            var shuttingDown = !runContext.KeepAlive || settings.ShutdownAfterRun;

            if (runContext.KeepAlive && settings.ShutdownAfterRun)
            {
                frameworkHandle.EnableShutdownAfterTestRun = true;
            }

            var toDispose = new List <IDisposable>();

            if (settings.MessageDisplay == MessageDisplay.Diagnostic)
            {
                lock (stopwatch)
                {
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution started", stopwatch.Elapsed));
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Settings: MaxParallelThreads = {1}, NameDisplay = {2}, ParallelizeAssemblies = {3}, ParallelizeTestCollections = {4}, ShutdownAfterRun = {5}",
                                                                                              stopwatch.Elapsed,
                                                                                              settings.MaxParallelThreads,
                                                                                              settings.NameDisplay,
                                                                                              settings.ParallelizeAssemblies,
                                                                                              settings.ParallelizeTestCollections,
                                                                                              settings.ShutdownAfterRun));
                }
            }

            try
            {
                RemotingUtility.CleanUpRegisteredChannels();

                cancelled = false;

                using (AssemblyHelper.SubscribeResolve())
                    if (settings.ParallelizeAssemblies)
                    {
                        testCaseAccessor(settings)
                        .Select(testCaseGroup => RunTestsInAssemblyAsync(runContext, frameworkHandle, toDispose, testCaseGroup.Key, testCaseGroup, settings, stopwatch))
                        .ToList()
                        .ForEach(@event => @event.WaitOne());
                    }
                    else
                    {
                        testCaseAccessor(settings)
                        .ToList()
                        .ForEach(testCaseGroup => RunTestsInAssembly(runContext, frameworkHandle, toDispose, testCaseGroup.Key, testCaseGroup, settings, stopwatch));
                    }
            }
            finally
            {
                if (!shuttingDown)
                {
                    toDispose.ForEach(disposable => disposable.Dispose());
                }
            }

            if (settings.MessageDisplay == MessageDisplay.Diagnostic)
            {
                lock (stopwatch)
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution complete", stopwatch.Elapsed));
            }
        }
Esempio n. 37
0
 public bool ShouldAttachToTestHost(IEnumerable <VsTestCase> tests, IRunContext runContext)
 {
     // TODO: expose setting in runContext to attach to testhost if needed?
     return(false);
 }
Esempio n. 38
0
        private void RunTestCase(
            IFrameworkHandle frameworkHandle,
            IRunContext runContext,
            TestCase test,
            Dictionary <string, PythonProjectSettings> sourceToSettings
            )
        {
            var testResult = new TestResult(test);

            frameworkHandle.RecordStart(test);
            testResult.StartTime = DateTimeOffset.Now;

            PythonProjectSettings settings;

            if (!sourceToSettings.TryGetValue(test.Source, out settings))
            {
                sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source, _interpreterService);
            }
            if (settings == null)
            {
                frameworkHandle.SendMessage(
                    TestMessageLevel.Error,
                    "Unable to determine interpreter to use for " + test.Source);
                RecordEnd(
                    frameworkHandle,
                    test,
                    testResult,
                    null,
                    "Unable to determine interpreter to use for " + test.Source,
                    TestOutcome.Failed);
                return;
            }

            var debugMode = PythonDebugMode.None;

            if (runContext.IsBeingDebugged && _app != null)
            {
                debugMode = settings.EnableNativeCodeDebugging ? PythonDebugMode.PythonAndNative : PythonDebugMode.PythonOnly;
            }

            var testCase = new PythonTestCase(settings, test, debugMode);

            var dte = _app.GetDTE();

            if (debugMode != PythonDebugMode.None)
            {
                dte.Debugger.DetachAll();
            }

            if (!File.Exists(settings.Factory.Configuration.InterpreterPath))
            {
                frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.Factory.Configuration.InterpreterPath);
                return;
            }

            var env           = new Dictionary <string, string>();
            var pythonPathVar = settings.Factory.Configuration.PathEnvironmentVariable;
            var pythonPath    = testCase.SearchPaths;

            if (!string.IsNullOrWhiteSpace(pythonPathVar))
            {
                if (_app != null)
                {
                    var settingsManager = SettingsManagerCreator.GetSettingsManager(dte);
                    if (settingsManager != null)
                    {
                        var store = settingsManager.GetReadOnlySettingsStore(SettingsScope.UserSettings);
                        if (store != null && store.CollectionExists(@"PythonTools\Options\General"))
                        {
                            var  settingStr = store.GetString(@"PythonTools\Options\General", "ClearGlobalPythonPath", "True");
                            bool settingBool;
                            if (bool.TryParse(settingStr, out settingBool) && !settingBool)
                            {
                                pythonPath += ";" + Environment.GetEnvironmentVariable(pythonPathVar);
                            }
                        }
                    }
                }
                env[pythonPathVar] = pythonPath;
            }

            foreach (var envVar in testCase.Environment)
            {
                env[envVar.Key] = envVar.Value;
            }

            using (var proc = ProcessOutput.Run(
                       !settings.IsWindowsApplication ?
                       settings.Factory.Configuration.InterpreterPath :
                       settings.Factory.Configuration.WindowsInterpreterPath,
                       testCase.Arguments,
                       testCase.WorkingDirectory,
                       env,
                       false,
                       null
                       )) {
                bool killed = false;

#if DEBUG
                frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + testCase.WorkingDirectory);
                frameworkHandle.SendMessage(TestMessageLevel.Informational, "set " + (pythonPathVar ?? "") + "=" + (pythonPath ?? ""));
                frameworkHandle.SendMessage(TestMessageLevel.Informational, proc.Arguments);
#endif

                proc.Wait(TimeSpan.FromMilliseconds(500));
                if (debugMode != PythonDebugMode.None)
                {
                    if (proc.ExitCode.HasValue)
                    {
                        // Process has already exited
                        frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to attach debugger because the process has already exited.");
                        if (proc.StandardErrorLines.Any())
                        {
                            frameworkHandle.SendMessage(TestMessageLevel.Error, "Standard error from Python:");
                            foreach (var line in proc.StandardErrorLines)
                            {
                                frameworkHandle.SendMessage(TestMessageLevel.Error, line);
                            }
                        }
                    }

                    try {
                        if (debugMode == PythonDebugMode.PythonOnly)
                        {
                            string qualifierUri = string.Format("tcp://{0}@localhost:{1}", testCase.DebugSecret, testCase.DebugPort);
                            while (!_app.AttachToProcess(proc, PythonRemoteDebugPortSupplierUnsecuredId, qualifierUri))
                            {
                                if (proc.Wait(TimeSpan.FromMilliseconds(500)))
                                {
                                    break;
                                }
                            }
                        }
                        else
                        {
                            var engines = new[] { PythonDebugEngineGuid, VSConstants.DebugEnginesGuids.NativeOnly_guid };
                            while (!_app.AttachToProcess(proc, engines))
                            {
                                if (proc.Wait(TimeSpan.FromMilliseconds(500)))
                                {
                                    break;
                                }
                            }
                        }

#if DEBUG
                    } catch (COMException ex) {
                        frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee.");
                        frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString());
                        try {
                            proc.Kill();
                        } catch (InvalidOperationException) {
                            // Process has already exited
                        }
                        killed = true;
                    }
#else
                    } catch (COMException) {
Esempio n. 39
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                TestPlatformContext testPlatformContext,
                                RunSettings runSettings,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assemblyDisplayName = "(unknown assembly)";

            try
            {
                var assembly = new XunitProjectAssembly {
                    AssemblyFilename = runInfo.AssemblyFileName
                };
                var assemblyFileName = runInfo.AssemblyFileName;
                assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
                var configuration = runInfo.Configuration;
                var shadowCopy    = configuration.ShadowCopyOrDefault;

                var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
                var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

                if (runSettings.DisableAppDomain)
                {
                    appDomain = AppDomainSupport.Denied;
                }

#if WINDOWS_UAP
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink        = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                {
                    var testCasesMap = new Dictionary <string, TestCase>();
                    var testCases    = new List <ITestCase>();
                    if (runInfo.TestCases == null || !runInfo.TestCases.Any())
                    {
                        // Discover tests
                        AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
                        DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings,
                                              (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
                                              (source, discoverer, discoveryOptions, visitor) =>
                        {
                            if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
                            {
                                foreach (var testCase in visitor.TestCases)
                                {
                                    logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
                                               testCase.DisplayName, testCase.UniqueID);
                                }
                            }

                            assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
                            {
                                AssemblyFileName    = source,
                                DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
                            };
                        },
                                              assemblyFileName,
                                              shadowCopy,
                                              configuration
                                              );

                        if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
                        {
                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
                            }

                            return;
                        }

                        // Filter tests
                        var traitNames        = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
                        var filter            = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
                        var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();

                        foreach (var filteredTestCase in filteredTestCases)
                        {
                            var uniqueID = filteredTestCase.UniqueID;
                            if (testCasesMap.ContainsKey(uniqueID))
                            {
                                logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
                            }
                            else
                            {
                                testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
                                testCases.Add(filteredTestCase.TestCase);
                            }
                        }
                    }
                    else
                    {
                        try
                        {
                            var serializations = runInfo.TestCases
                                                 .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
                                                 .ToList();

                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}",
                                                     serializations.Count,
                                                     Environment.NewLine,
                                                     string.Join(Environment.NewLine, serializations.Select(x => $"  {x}")));
                            }

                            var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);

                            if (deserializedTestCasesByUniqueId == null)
                            {
                                logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
                            }
                            else
                            {
                                for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
                                {
                                    try
                                    {
                                        var kvp        = deserializedTestCasesByUniqueId[idx];
                                        var vsTestCase = runInfo.TestCases[idx];

                                        if (kvp.Value == null)
                                        {
                                            logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
                                        }
                                        else
                                        {
                                            testCasesMap.Add(kvp.Key, vsTestCase);
                                            testCases.Add(kvp.Value);
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
                        }
                    }

                    // Execute tests
                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (runSettings.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(testCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }
Esempio n. 40
0
 public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     Debugger.Launch();
 }
Esempio n. 41
0
 public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     Debugger.Launch();
 }
Esempio n. 42
0
        /// <summary>
        /// Execute the parameter tests present in parameter source
        /// </summary>
        /// <param name="tests">Tests to execute.</param>
        /// <param name="runContext">The run context.</param>
        /// <param name="frameworkHandle">Handle to record test start/end/results.</param>
        /// <param name="source">The test container for the tests.</param>
        /// <param name="isDeploymentDone">Indicates if deployment is done.</param>
        private void ExecuteTestsInSource(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone)
        {
            Debug.Assert(!string.IsNullOrEmpty(source), "Source cannot be empty");

            if (isDeploymentDone)
            {
                source = Path.Combine(PlatformServiceProvider.Instance.TestDeployment.GetDeploymentDirectory(), Path.GetFileName(source));
            }

            using (var isolationHost = PlatformServiceProvider.Instance.CreateTestSourceHost(source, runContext?.RunSettings, frameworkHandle))
            {
                var testRunner = isolationHost.CreateInstanceForType(
                    typeof(UnitTestRunner),
                    new object[] { MSTestSettings.CurrentSettings }) as UnitTestRunner;
                PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Created unit-test runner {0}", source);

                // Default test set is filtered tests based on user provided filter criteria
                IEnumerable <TestCase> testsToRun = Enumerable.Empty <TestCase>();
                var filterExpression = this.TestMethodFilter.GetFilterExpression(runContext, frameworkHandle, out var filterHasError);
                if (filterHasError)
                {
                    // Bail out without processing everything else below.
                    return;
                }

                testsToRun = tests.Where(t => MatchTestFilter(filterExpression, t, this.TestMethodFilter));

                // this is done so that appropriate values of testcontext properties are set at source level
                // and are merged with session level parameters
                var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(source);

                if (this.sessionParameters != null && this.sessionParameters.Count > 0)
                {
                    sourceLevelParameters = sourceLevelParameters.Concat(this.sessionParameters).ToDictionary(x => x.Key, x => x.Value);
                }

                var sourceSettingsProvider = isolationHost.CreateInstanceForType(
                    typeof(TestAssemblySettingsProvider),
                    null) as TestAssemblySettingsProvider;

                var sourceSettings  = sourceSettingsProvider.GetSettings(source);
                var parallelWorkers = sourceSettings.Workers;
                var parallelScope   = sourceSettings.Scope;

                if (MSTestSettings.CurrentSettings.ParallelizationWorkers.HasValue)
                {
                    // The runsettings value takes precedence over an assembly level setting. Reset the level.
                    parallelWorkers = MSTestSettings.CurrentSettings.ParallelizationWorkers.Value;
                }

                if (MSTestSettings.CurrentSettings.ParallelizationScope.HasValue)
                {
                    // The runsettings value takes precedence over an assembly level setting. Reset the level.
                    parallelScope = MSTestSettings.CurrentSettings.ParallelizationScope.Value;
                }

                if (!MSTestSettings.CurrentSettings.DisableParallelization && sourceSettings.CanParallelizeAssembly && parallelWorkers > 0)
                {
                    // Parallelization is enabled. Let's do further classification for sets.
                    var logger = (IMessageLogger)frameworkHandle;
                    logger.SendMessage(
                        TestMessageLevel.Informational,
                        string.Format(CultureInfo.CurrentCulture, Resource.TestParallelizationBanner, source, parallelWorkers, parallelScope));

                    // Create test sets for execution, we can execute them in parallel based on parallel settings
                    IEnumerable <IGrouping <bool, TestCase> > testsets = Enumerable.Empty <IGrouping <bool, TestCase> >();

                    // Parallel and not parallel sets.
                    testsets = testsToRun.GroupBy(t => t.GetPropertyValue <bool>(TestAdapter.Constants.DoNotParallelizeProperty, false));

                    var parallelizableTestSet    = testsets.FirstOrDefault(g => g.Key == false);
                    var nonparallelizableTestSet = testsets.FirstOrDefault(g => g.Key == true);

                    if (parallelizableTestSet != null)
                    {
                        ConcurrentQueue <IEnumerable <TestCase> > queue = null;

                        // Chunk the sets into further groups based on parallel level
                        switch (parallelScope)
                        {
                        case ExecutionScope.MethodLevel:
                            queue = new ConcurrentQueue <IEnumerable <TestCase> >(parallelizableTestSet.Select(t => new[] { t }));
                            break;

                        case ExecutionScope.ClassLevel:
                            queue = new ConcurrentQueue <IEnumerable <TestCase> >(parallelizableTestSet.GroupBy(t => t.GetPropertyValue(TestAdapter.Constants.TestClassNameProperty) as string));
                            break;
                        }

                        var tasks = new List <Task>();

                        for (int i = 0; i < parallelWorkers; i++)
                        {
                            tasks.Add(Task.Factory.StartNew(
                                          () =>
                            {
                                while (!queue.IsEmpty)
                                {
                                    if (this.cancellationToken != null && this.cancellationToken.Canceled)
                                    {
                                        // if a cancellation has been requested, do not queue any more test runs.
                                        break;
                                    }

                                    if (queue.TryDequeue(out IEnumerable <TestCase> testSet))
                                    {
                                        this.ExecuteTestsWithTestRunner(testSet, runContext, frameworkHandle, source, sourceLevelParameters, testRunner);
                                    }
                                }
                            },
                                          CancellationToken.None,
                                          TaskCreationOptions.LongRunning,
                                          TaskScheduler.Default));
                        }

                        Task.WaitAll(tasks.ToArray());
                    }

                    // Queue the non parallel set
                    if (nonparallelizableTestSet != null)
                    {
                        this.ExecuteTestsWithTestRunner(nonparallelizableTestSet, runContext, frameworkHandle, source, sourceLevelParameters, testRunner);
                    }
                }
                else
                {
                    this.ExecuteTestsWithTestRunner(testsToRun, runContext, frameworkHandle, source, sourceLevelParameters, testRunner);
                }

                this.RunCleanup(frameworkHandle, testRunner);

                PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
                    "Executed tests belonging to source {0}",
                    source);
            }
        }
Esempio n. 43
0
        List <AssemblyRunInfo> GetTests(IEnumerable <string> sources, LoggerHelper logger, IRunContext runContext)
        {
            // For store apps, the files are copied to the AppX dir, we need to load it from there
#if PLATFORM_DOTNET
            sources = sources.Select(s => Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(s)));
#endif

            var result          = new List <AssemblyRunInfo>();
            var knownTraitNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);

            DiscoverTests(
                sources,
                logger,
                (source, discoverer, discoveryOptions) => new VsExecutionDiscoveryVisitor(),
                (source, discoverer, discoveryOptions, visitor) =>
            {
                var vsFilteredTestCases = visitor.TestCases.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(source, discoverer, testCase, false, logger: logger, knownTraitNames: knownTraitNames)).ToList();

                // Apply any filtering
                var filterHelper    = new TestCaseFilterHelper(knownTraitNames);
                vsFilteredTestCases = filterHelper.GetFilteredTestList(vsFilteredTestCases, runContext, logger, source).ToList();

                // Re-create testcases with unique names if there is more than 1
                var testCases = visitor.TestCases.Where(tc => vsFilteredTestCases.Any(vsTc => vsTc.DisplayName == tc.DisplayName)).GroupBy(tc => $"{tc.TestMethod.TestClass.Class.Name}.{tc.TestMethod.Method.Name}")
                                .SelectMany(group => group.Select(testCase =>
                                                                  VsDiscoveryVisitor.CreateVsTestCase(
                                                                      source,
                                                                      discoverer,
                                                                      testCase,
                                                                      forceUniqueNames: group.Count() > 1,
                                                                      logger: logger,
                                                                      knownTraitNames: knownTraitNames))
                                            .Where(vsTestCase => vsTestCase != null)).ToList();             // pre-enumerate these as it populates the known trait names collection

                var runInfo = new AssemblyRunInfo
                {
                    AssemblyFileName = source,
                    Configuration    = LoadConfiguration(source),
                    TestCases        = testCases
                };
                result.Add(runInfo);
            }
                );

            return(result);
        }
Esempio n. 44
0
 /// <summary>
 /// Gets filter expression from run context.
 /// </summary>
 /// <param name="context">Run context</param>
 /// <returns>Filter expression.</returns>
 private ITestCaseFilterExpression GetTestCaseFilterFromRunContext(IRunContext context)
 {
     return(context.GetTestCaseFilter(this.supportedProperties.Keys, this.PropertyProvider));
 }
        /// <summary>
        /// Implements initialization. This is called by ITestAdapter.Initialize
        /// (a synchronous function) which will block until this function is
        /// completed.
        /// </summary>
        /// <param name="runContext">
        /// The context for the current test run.
        /// </param>
        private async Task InitializeWorker(
            TestProperties vars,
            IRunContext runContext,
            ITestElement currentTest = null
            )
        {
            string  application, executable, versionString, hive;
            Version version;
            string  launchTimeoutInSecondsString;
            int     launchTimeoutInSeconds;
            string  reuseInstanceString;

            // VSApplication is the registry key name like 'VisualStudio'
            application = vars[VSTestProperties.VSApplication.Key] ?? VSTestProperties.VSApplication.VisualStudio;
            // VSExecutableName is the executable name like 'devenv'
            if (!vars.TryGetValue(VSTestProperties.VSExecutable.Key, out executable))
            {
                executable = VSTestProperties.VSExecutable.DevEnv;
            }
            if (!string.IsNullOrEmpty(executable) &&
                string.IsNullOrEmpty(Path.GetExtension(executable)))
            {
                executable = Path.ChangeExtension(executable, ".exe");
            }

            // VSVersion is the version like '12.0'
            if (!vars.TryGetValue(VSTestProperties.VSVersion.Key, out versionString) ||
                !Version.TryParse(versionString, out version))
            {
                version = new Version(int.Parse(AssemblyVersionInfo.VSVersion), 0);
            }

            // VSHive is the optional hive like 'Exp'
            hive = vars[VSTestProperties.VSHive.Key];

            if (!vars.TryGetValue(VSTestProperties.VSLaunchTimeoutInSeconds.Key, out launchTimeoutInSecondsString) ||
                !int.TryParse(launchTimeoutInSecondsString, out launchTimeoutInSeconds))
            {
                launchTimeoutInSeconds = 30;
            }

            if (string.IsNullOrEmpty(application) || string.IsNullOrEmpty(executable) || version == null)
            {
                throw new ArgumentException(string.Format(
                                                Resources.MissingConfigurationValues,
                                                application ?? "(null)",
                                                executable ?? "(null)",
                                                version != null ? version.ToString() : "(null)",
                                                hive ?? "(default)"
                                                ));
            }

            _mockVs = (application == VSTestProperties.VSApplication.Mock);
            if (_mockVs)
            {
                _remote = new TesteeTestAdapter();
                _remote.Initialize(_runContext);
                // In the mock case tester and testee are the same process, therefore
                // VSTestContext is in our process too.  So we can just set this value
                // directly here.
                VSTestContext.IsMock = true;
                return;
            }

            if (!vars.TryGetValue(VSTestProperties.VSReuseInstance.Key, out reuseInstanceString) ||
                !bool.TryParse(reuseInstanceString, out _reuseInstance))
            {
                // the default behavior is to reuse the current instance when possible
                _reuseInstance = true;
            }

            // TODO: Detect and perform first run of VS if necessary.
            // The first time a VS hive is run, the user sees a dialog allowing
            // them to select settings such as the theme. We can avoid this by
            // running devenv.exe /resetSettings <path to profile.settings>
            // first, though it is not trivial to detect when this is necessary.
            // Without having done this, all tests will time out. For now, the
            // user is responsible for running VS at least once before
            // attempting to execute tests.

            var cts = new CancellationTokenSource(TimeSpan.FromSeconds(launchTimeoutInSeconds));

            try {
                await Connect(application, executable, version, hive, runContext, currentTest, cts.Token);
            } catch (OperationCanceledException ex) {
                throw new TimeoutException(string.Format(Resources.VSLaunchTimeout, launchTimeoutInSeconds), ex);
            } catch (Exception ex) {
                throw new InvalidOperationException(
                          string.Format(Resources.VSFailedToLaunch, application, executable, version, hive),
                          ex
                          );
            }
        }
Esempio n. 46
0
 public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
 }
Esempio n. 47
0
 public FeatureRunner(IStringStepRunner stringStepRunner, IRunContext context)
 {
     this.context          = context;
     this.stringStepRunner = stringStepRunner;
 }
Esempio n. 48
0
        private void RunTestsCore(IEnumerable <ITestScript> tests, Predicate <ITestFunction> filter, ILogger logger, IRunContext runContext, ITestExecutionRecorder recorder)
        {
            var settingsService = runContext.RunSettings.GetSettings(PSycheTestRunSettings.SettingsProviderName) as IPSycheTestSettingsService ?? new PSycheTestSettingsService();

            var testCaseMap = new Dictionary <ITestFunction, TestCase>();
            var executor    = _executorFactory(logger);

            executor.OutputDirectory = new DirectoryInfo(runContext.TestRunDirectory);
            logger.Info("Test output directory: {0}", executor.OutputDirectory.FullName);

            foreach (var module in settingsService.Settings.Modules)
            {
                logger.Info("Adding module '{0}'", module);
                executor.InitialModules.Add(module);
            }

            executor.TestStarting += (o, e) =>
            {
                var testCase = _mapper.Map(e.Test);
                testCaseMap[e.Test] = testCase;
                recorder.RecordStart(testCase);
            };
            executor.TestEnded += (o, e) =>
            {
                var testCase = testCaseMap[e.Test];
                recorder.RecordEnd(testCase, _mapper.Map(e.Result.Status));
                recorder.RecordResult(_mapper.Map(testCase, e.Result));
            };

            using (_cancellationTokenSource = new CancellationTokenSource())
            {
                executor.ExecuteAsync(tests, filter, _cancellationTokenSource.Token).Wait();                    // Awaiting seems to cause odd behavior, just block instead.
            }
            _cancellationTokenSource = null;
        }
Esempio n. 49
0
        void RunTestsInAssembly(IRunContext runContext,
                                IFrameworkHandle frameworkHandle,
                                LoggerHelper logger,
                                TestPlatformContext testPlatformContext,
                                IMessageSinkWithTypes reporterMessageHandler,
                                AssemblyRunInfo runInfo)
        {
            if (cancelled)
            {
                return;
            }

            var assembly = new XunitProjectAssembly {
                AssemblyFilename = runInfo.AssemblyFileName
            };
            var assemblyFileName    = runInfo.AssemblyFileName;
            var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
            var configuration       = runInfo.Configuration;
            var shadowCopy          = configuration.ShadowCopyOrDefault;

            var appDomain          = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
            var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;

            if (RunSettingsHelper.DisableAppDomain)
            {
                appDomain = AppDomainSupport.Denied;
            }

            try
            {
#if WINDOWS_UAP
                // For AppX Apps, use the package location
                assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif

                var diagnosticSink        = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
                var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
                using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
                {
                    var testCasesMap = new Dictionary <string, TestCase>();
                    var testCases    = new List <ITestCase>();
                    if (runInfo.TestCases == null || !runInfo.TestCases.Any())
                    {
                        // Discover tests
                        AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
                        DiscoverTestsInSource(controller, logger, testPlatformContext,
                                              (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
                                              (source, discoverer, discoveryOptions, visitor) =>
                        {
                            if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
                            {
                                foreach (var testCase in visitor.TestCases)
                                {
                                    logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
                                               testCase.DisplayName, testCase.UniqueID);
                                }
                            }

                            assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
                            {
                                AssemblyFileName    = source,
                                DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList()
                            };
                        },
                                              assemblyFileName,
                                              shadowCopy,
                                              configuration
                                              );

                        if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
                        {
                            if (configuration.InternalDiagnosticMessagesOrDefault)
                            {
                                logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
                            }

                            return;
                        }

                        // Filter tests
                        var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));

                        // Apply any filtering
                        var filter            = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
                        var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();

                        // Force unique names if there is more than 1 testcase with the same name
                        foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1))
                        {
                            foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames)
                            {
                                discoveredTestCaseWithDuplicateName.ForceUniqueName();
                            }
                        }

                        foreach (var filteredTestCase in filteredTestCases)
                        {
                            var uniqueID = filteredTestCase.UniqueID;
                            if (testCasesMap.ContainsKey(uniqueID))
                            {
                                logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
                            }
                            else
                            {
                                testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
                                testCases.Add(filteredTestCase.TestCase);
                            }
                        }
                    }
                    else
                    {
                        // We are in Run Specific tests scenario, the `TestCase` objects are available.
                        // Query the `TestCase` objects to find XunitTestCase objects.
                        foreach (var vstestCase in runInfo.TestCases)
                        {
                            var xunitTestCase = Deserialize(logger, controller, vstestCase);
                            if (xunitTestCase != null)
                            {
                                testCasesMap.Add(xunitTestCase.UniqueID, vstestCase);
                                testCases.Add(xunitTestCase);
                            }
                        }
                    }

                    // Execute tests
                    var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
                    if (RunSettingsHelper.DisableParallelization)
                    {
                        executionOptions.SetSynchronousMessageReporting(true);
                        executionOptions.SetDisableParallelization(true);
                    }

                    reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));

                    using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
                    {
                        IExecutionSink resultsSink = vsExecutionSink;
                        if (longRunningSeconds > 0)
                        {
                            resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
                        }

                        controller.RunTests(testCases, resultsSink, executionOptions);
                        resultsSink.Finished.WaitOne();

                        reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
            }
        }
Esempio n. 50
0
        private FeatureResults Run(IFeatureRunner featureRunner, IEnumerable <Feature> features, IRunContext context)
        {
            var result = new FeatureResults();

            foreach (Feature feature in features)
            {
                FeatureResult featureResult = null;
                try
                {
                    context.FeatureStarted(feature);
                    featureResult = featureRunner.Run(feature);
                    result.Add(featureResult);
                }
                finally
                {
                    context.FeatureFinished(featureResult);
                }
            }
            return(result);
        }
Esempio n. 51
0
 public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     RunTestsWithTestsCallback?.Invoke(tests, runContext, frameworkHandle);
 }
Esempio n. 52
0
        public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            _mCancelled = false;
            SetupExecutionPolicy();

            var testSets = new List <TestCaseSet>();

            foreach (var testCase in tests)
            {
                var describe = testCase.FullyQualifiedName.Split('.').First();
                var codeFile = testCase.CodeFilePath;

                var testSet = testSets.FirstOrDefault(m => m.Describe.Equals(describe, StringComparison.OrdinalIgnoreCase) &&
                                                      m.File.Equals(codeFile, StringComparison.OrdinalIgnoreCase));

                if (testSet == null)
                {
                    testSet = new TestCaseSet(codeFile, describe);
                    testSets.Add(testSet);
                }

                testSet.TestCases.Add(testCase);
            }

            foreach (var testSet in testSets)
            {
                if (_mCancelled)
                {
                    break;
                }

                var testOutput = new StringBuilder();

                try
                {
                    var testAdapter = new TestAdapterHost();
                    testAdapter.HostUi.OutputString = s =>
                    {
                        if (!string.IsNullOrEmpty(s))
                        {
                            testOutput.Append(s);
                        }
                    };

                    var runpsace = RunspaceFactory.CreateRunspace(testAdapter);
                    runpsace.Open();

                    using (var ps = PowerShell.Create())
                    {
                        ps.Runspace = runpsace;
                        RunTestSet(ps, testSet, runContext);

                        foreach (var testResult in testSet.TestResults)
                        {
                            frameworkHandle.RecordResult(testResult);
                        }
                    }
                }
                catch (Exception ex)
                {
                    foreach (var testCase in testSet.TestCases)
                    {
                        var testResult = new TestResult(testCase);
                        testResult.Outcome         = TestOutcome.Failed;
                        testResult.ErrorMessage    = ex.Message;
                        testResult.ErrorStackTrace = ex.StackTrace;
                        frameworkHandle.RecordResult(testResult);
                    }
                }

                if (testOutput.Length > 0)
                {
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, testOutput.ToString());
                }
            }
        }
Esempio n. 53
0
 /// <summary>
 /// ITestAdapter method: called just before the test run finishes and
 /// gives the adapter a chance to do any clean-up.
 /// </summary>
 /// <param name="runContext">The run context for this run</param>
 void ITestAdapter.PreTestRunFinished(IRunContext runContext)
 {
     HostSide.PreTestRunFinished(runContext);
 }
 public void PreTestRunFinished(IRunContext runContext)
 {
     RemoteCall(r => r.PreTestRunFinished(runContext));
 }
Esempio n. 55
0
 public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     throw new NotImplementedException();
 }
        private async Task Connect(
            string application,
            string executable,
            Version version,
            string hive,
            IRunContext runContext,
            ITestElement currentTest,
            CancellationToken cancel
            )
        {
            var hiveOption = string.IsNullOrEmpty(hive) ? "" : (" /rootSuffix " + hive);

            if (_reuseInstance &&
                _ide != null &&
                _remote != null &&
                application == _currentApplication &&
                executable == _currentExecutable &&
                version == _currentVersion &&
                hive == _currentHive)
            {
                if (runContext != null)
                {
                    SendMessage(
                        runContext,
                        string.Format(Resources.VSReuseMessage, application, executable, version, hiveOption),
                        currentTest
                        );
                }
                return;
            }

            Close();

            if (runContext != null)
            {
                SendMessage(
                    runContext,
                    string.Format(Resources.VSLaunchMessage, application, executable, version, hiveOption),
                    currentTest
                    );
            }

            Internal.VisualStudio ide = null;
            try {
                ide = await Internal.VisualStudio.LaunchAsync(application, executable, version, hive, cancel);

                var p   = Process.GetProcessById(ide.ProcessId);
                var url = string.Format("ipc://{0}/{1}", VSTestHostPackage.GetChannelName(p), TesteeTestAdapter.Url);

                try {
                    _remote = (TesteeTestAdapter)RemotingServices.Connect(typeof(TesteeTestAdapter), url);
                } catch (RemotingException ex) {
                    throw new InvalidOperationException(Resources.FailedToConnect, ex);
                }

                _currentApplication = application;
                _currentExecutable  = executable;
                _currentVersion     = version;
                _currentHive        = hive;
                _ide = ide;
                ide  = null;
            } finally {
                if (ide != null)
                {
                    ide.Dispose();
                }
            }
        }
Esempio n. 57
0
 public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
     throw new NotImplementedException();
 }
        private string RunProtractor(TestCase test, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            var resultFile = Path.GetFileNameWithoutExtension(test.CodeFilePath);

            resultFile += ".result.json";

            resultFile = AppConfig.ResultsPath ?? Path.GetTempPath() + Path.DirectorySeparatorChar + resultFile;
            frameworkHandle.SendMessage(TestMessageLevel.Informational, "Framework: Using result file: " + resultFile);
            // Can't use test.Source as it's going to be in lowercase
            var cwd = Helper.FindInDirectoryTree(test.CodeFilePath, "package.json");
            var exe = Helper.FindExePath(AppConfig.Program);

            //frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: Exe " + exe);
            //frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: Cwd " + cwd);
            //frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: ResultFile " + resultFile);

            ProcessStartInfo info = new ProcessStartInfo()
            {
                Arguments              = string.Format("{0} --resultJsonOutputFile \"{1}\" --specs \"{2}\"", AppConfig.Arguments, resultFile, test.CodeFilePath),
                FileName               = exe,
                WorkingDirectory       = cwd,//runContext.SolutionDirectory,
                UseShellExecute        = false,
                CreateNoWindow         = true,
                RedirectStandardError  = true,
                RedirectStandardOutput = true,
            };

            frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Framework: Starting {exe} on '{cwd}' with arguments: {info.Arguments}");


            Process p = new Process
            {
                StartInfo = info
            };

            using (AutoResetEvent outputWaitHandle = new AutoResetEvent(false))
                using (AutoResetEvent errorWaitHandle = new AutoResetEvent(false))
                {
                    p.OutputDataReceived += (sender, e) => {
                        if (e.Data == null)
                        {
                            outputWaitHandle.Set();
                        }
                        else if (!String.IsNullOrWhiteSpace(e.Data))
                        {
                            // Console.WriteLine(e.Data);
                            frameworkHandle.SendMessage(TestMessageLevel.Informational, e.Data);
                        }
                    };
                    p.ErrorDataReceived += (sender, e) =>
                    {
                        if (e.Data == null)
                        {
                            errorWaitHandle.Set();
                        }
                        else if (!string.IsNullOrWhiteSpace(e.Data))
                        {
                            // Console.Error.WriteLine(e.Data);
                            frameworkHandle.SendMessage(TestMessageLevel.Error, e.Data);
                        }
                    };

                    p.Start();
                    p.BeginOutputReadLine();
                    p.BeginErrorReadLine();
                    p.WaitForExit();
                    outputWaitHandle.WaitOne(10000);
                    errorWaitHandle.WaitOne(10000);
                    // Process completed. Check process.ExitCode here.
                    frameworkHandle.SendMessage(TestMessageLevel.Informational, "Framework: Complete. Exit code: " + p.ExitCode.ToString());
                }

            return(resultFile);
        }
Esempio n. 59
0
 public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
 {
 }
Esempio n. 60
0
        public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
        {
            var explicitlyInclude = CommonStringPrefix.Of(tests.Select(t => t.FullyQualifiedName));

            InternalRunTests(tests, runContext, frameworkHandle, explicitlyInclude);
        }