public void SetCustomLauncher(ITestHostLauncher customLauncher)
 {
     throw new NotImplementedException();
 }
Exemplo n.º 2
0
 /// <inheritdoc/>
 public void RunTestsWithCustomTestHost(IEnumerable <TestCase> testCases, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
 {
     this.EnsureInitialized();
     this.requestSender.StartTestRunWithCustomHost(testCases, runSettings, testRunEventsHandler, customTestHostLauncher);
 }
Exemplo n.º 3
0
        private void SendMessageAndListenAndReportTestResults(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher)
        {
            try
            {
                this.communicationManager.SendMessage(messageType, payload, this.protocolVersion);
                var isTestRunComplete = false;

                // Cycle through the messages that the testhost sends.
                // Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification.
                while (!isTestRunComplete)
                {
                    var message = this.TryReceiveMessage();

                    if (string.Equals(MessageType.TestRunStatsChange, message.MessageType))
                    {
                        var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(
                            message);
                        eventHandler.HandleTestRunStatsChange(testRunChangedArgs);
                    }
                    else if (string.Equals(MessageType.ExecutionComplete, message.MessageType))
                    {
                        var testRunCompletePayload =
                            this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message);

                        eventHandler.HandleTestRunComplete(
                            testRunCompletePayload.TestRunCompleteArgs,
                            testRunCompletePayload.LastRunTests,
                            testRunCompletePayload.RunAttachments,
                            testRunCompletePayload.ExecutorUris);
                        isTestRunComplete = true;
                    }
                    else if (string.Equals(MessageType.TestMessage, message.MessageType))
                    {
                        var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message);
                        eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message);
                    }
                    else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType))
                    {
                        HandleCustomHostLaunch(customHostLauncher, message);
                    }
                }
            }
            catch (Exception exception)
            {
                EqtTrace.Error("Aborting Test Run Operation: {0}", exception);
                eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun);
                var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero);
                eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
                this.CleanupCommunicationIfProcessExit();
            }

            this.testPlatformEventSource.TranslationLayerExecutionStop();
        }
Exemplo n.º 4
0
        /// <inheritdoc/>
        public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, TestPlatformOptions options, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
        {
            var sourceList = sources.ToList();

            this.testPlatformEventSource.TranslationLayerExecutionStart(1, sourceList.Count, 0, runSettings ?? string.Empty);

            this.EnsureInitialized();
            this.requestSender.StartTestRunWithCustomHost(sourceList, runSettings, options, testRunEventsHandler, customTestHostLauncher);
        }
Exemplo n.º 5
0
        /// <summary>
        /// Run Tests with given a set of test cases.
        /// </summary>
        /// <param name="testRunRequestPayload">TestRun request Payload</param>
        /// <param name="testHostLauncher">TestHost Launcher for the run</param>
        /// <param name="testRunEventsRegistrar">event registrar for run events</param>
        /// <param name="protocolConfig">Protocol related information</param>
        /// <returns>True, if successful</returns>
        public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
        {
            EqtTrace.Info("TestRequestManager.RunTests: run tests started.");

            TestRunCriteria runCriteria = null;
            var             runsettings = testRunRequestPayload.RunSettings;

            if (testRunRequestPayload.TestPlatformOptions != null)
            {
                this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
            }

            var requestData = this.GetRequestData(protocolConfig);

            // Get sources to auto detect fx and arch for both run selected or run all scenario.
            var sources = GetSources(testRunRequestPayload);

            if (this.UpdateRunSettingsIfRequired(runsettings, sources, out string updatedRunsettings))
            {
                runsettings = updatedRunsettings;
            }

            if (InferRunSettingsHelper.AreRunSettingsCollectorsInCompatibleWithTestSettings(runsettings))
            {
                throw new SettingsException(string.Format(Resources.RunsettingsWithDCErrorMessage, runsettings));
            }

            var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
            var batchSize        = runConfiguration.BatchSize;

            if (requestData.IsTelemetryOptedIn)
            {
                // Collect Metrics
                this.CollectMetrics(requestData, runConfiguration);

                // Collect Commands
                this.LogCommandsTelemetryPoints(requestData);
            }

            if (!commandLineOptions.IsDesignMode)
            {
                // Generate fakes settings only for command line scenarios. In case of
                // Editors/IDEs, this responsibility is with the caller.
                GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings);
            }

            if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
            {
                runCriteria = new TestRunCriteria(
                    testRunRequestPayload.Sources,
                    batchSize,
                    testRunRequestPayload.KeepAlive,
                    runsettings,
                    this.commandLineOptions.TestStatsEventTimeout,
                    testHostLauncher);
                runCriteria.TestCaseFilter = testRunRequestPayload.TestPlatformOptions?.TestCaseFilter;
                runCriteria.FilterOptions  = testRunRequestPayload.TestPlatformOptions?.FilterOptions;
            }
            else
            {
                runCriteria = new TestRunCriteria(
                    testRunRequestPayload.TestCases,
                    batchSize,
                    testRunRequestPayload.KeepAlive,
                    runsettings,
                    this.commandLineOptions.TestStatsEventTimeout,
                    testHostLauncher);
            }

            var success = this.RunTests(requestData, runCriteria, testRunEventsRegistrar);

            EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
            this.testPlatformEventSource.ExecutionRequestStop();

            // Post the run complete event
            this.metricsPublisher.Result.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics);

            return(success);
        }
Exemplo n.º 6
0
        private async Task SendMessageAndListenAndReportTestResultsAsync(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher)
        {
            try
            {
                this.communicationManager.SendMessage(messageType, payload, this.protocolVersion);
                var isTestRunComplete = false;

                // Cycle through the messages that the testhost sends.
                // Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification.
                while (!isTestRunComplete)
                {
                    var message = await this.TryReceiveMessageAsync();

                    if (string.Equals(MessageType.TestRunStatsChange, message.MessageType))
                    {
                        var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(
                            message);
                        eventHandler.HandleTestRunStatsChange(testRunChangedArgs);
                    }
                    else if (string.Equals(MessageType.ExecutionComplete, message.MessageType))
                    {
                        if (EqtTrace.IsInfoEnabled)
                        {
                            EqtTrace.Info("VsTestConsoleRequestSender.SendMessageAndListenAndReportTestResultsAsync: Execution complete.");
                        }

                        var testRunCompletePayload =
                            this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message);

                        eventHandler.HandleTestRunComplete(
                            testRunCompletePayload.TestRunCompleteArgs,
                            testRunCompletePayload.LastRunTests,
                            testRunCompletePayload.RunAttachments,
                            testRunCompletePayload.ExecutorUris);
                        isTestRunComplete = true;
                    }
                    else if (string.Equals(MessageType.TestMessage, message.MessageType))
                    {
                        var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message);
                        eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message);
                    }
                    else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType))
                    {
                        HandleCustomHostLaunch(customHostLauncher, message);
                    }
                    else if (string.Equals(MessageType.EditorAttachDebugger, message.MessageType))
                    {
                        AttachDebuggerToProcess(customHostLauncher, message);
                    }
                }
            }
            catch (Exception exception)
            {
                EqtTrace.Error("Aborting Test Run Operation: {0}", exception);
                eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun);
                var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero);
                eventHandler.HandleTestRunComplete(completeArgs, null, null, null);

                // Earlier we were closing the connection with vstest.console in case of exceptions
                // Removing that code because vstest.console might be in a healthy state and letting the client
                // know of the error, so that the TL can wait for the next instruction from the client itself.
                // Also, connection termination might not kill the process which could result in files being locked by testhost.
            }

            this.testPlatformEventSource.TranslationLayerExecutionStop();
        }
Exemplo n.º 7
0
        /// <inheritdoc/>
        public void StartTestSession(
            StartTestSessionPayload payload,
            ITestHostLauncher testHostLauncher,
            ITestSessionEventsHandler eventsHandler,
            ProtocolConfig protocolConfig)
        {
            EqtTrace.Info("TestRequestManager.StartTestSession: Starting test session.");

            if (payload.TestPlatformOptions != null)
            {
                this.telemetryOptedIn = payload.TestPlatformOptions.CollectMetrics;
            }

            var requestData = this.GetRequestData(protocolConfig);

            if (this.UpdateRunSettingsIfRequired(
                    payload.RunSettings,
                    payload.Sources,
                    null,
                    out string updatedRunsettings))
            {
                payload.RunSettings = updatedRunsettings;
            }

            if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(payload.RunSettings))
            {
                throw new SettingsException(
                          string.Format(
                              Resources.RunsettingsWithDCErrorMessage,
                              payload.RunSettings));
            }

            // TODO (copoiena): Collect metrics ?

            lock (this.syncObject)
            {
                try
                {
                    EqtTrace.Info("TestRequestManager.StartTestRunner: Synchronization context taken.");
                    this.TestPlatformEventSourceInstance.StartTestSessionStart();

                    var criteria = new StartTestSessionCriteria()
                    {
                        Sources          = payload.Sources,
                        RunSettings      = payload.RunSettings,
                        TestHostLauncher = testHostLauncher
                    };

                    this.testPlatform.StartTestSession(requestData, criteria, eventsHandler);
                }
                finally
                {
                    EqtTrace.Info("TestRequestManager.StartTestSession: Starting test session completed.");
                    this.TestPlatformEventSourceInstance.StartTestSessionStop();

                    // Post the attachments processing complete event.
                    this.metricsPublisher.Result.PublishMetrics(
                        TelemetryDataConstants.StartTestSessionCompleteEvent,
                        requestData.MetricsCollection.Metrics);
                }
            }
        }
Exemplo n.º 8
0
 public void SetCustomLauncher(ITestHostLauncher customLauncher)
 {
 }
Exemplo n.º 9
0
        /// <summary>
        /// Initializes a new instance of the <see cref="TestRunCriteria"/> class.
        /// </summary>
        /// <param name="tests">
        /// Sources which contains tests that should be executed
        /// </param>
        /// <param name="frequencyOfRunStatsChangeEvent">
        /// Frequency of run stats event
        /// </param>
        /// <param name="keepAlive">
        /// Whether or not to keep the test executor process alive after run completion
        /// </param>
        /// <param name="testSettings">
        /// Settings used for this run.
        /// </param>
        /// <param name="runStatsChangeEventTimeout">
        /// Timeout that triggers sending results regardless of cache size.
        /// </param>
        /// <param name="testHostLauncher">
        /// Test host launcher. If null then default will be used.
        /// </param>
        public TestRunCriteria(IEnumerable <TestCase> tests, long frequencyOfRunStatsChangeEvent, bool keepAlive, string testSettings, TimeSpan runStatsChangeEventTimeout, ITestHostLauncher testHostLauncher)
            : base(frequencyOfRunStatsChangeEvent, keepAlive, testSettings, runStatsChangeEventTimeout, testHostLauncher)
        {
            var testCases = tests as IList <TestCase> ?? tests.ToList();

            ValidateArg.NotNullOrEmpty(testCases, "tests");

            this.Tests = testCases;
        }
Exemplo n.º 10
0
 public TestableDotnetTestHostManager(ITestHostLauncher testHostLauncher, IProcessHelper processHelper, IFileHelper fileHelper)
     : base(testHostLauncher, processHelper, fileHelper)
 {
 }
Exemplo n.º 11
0
 /// <inheritdoc/>
 public async Task RunTestsWithCustomTestHostAsync(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
 {
     await RunTestsWithCustomTestHostAsync(sources, runSettings, null, testRunEventsHandler, customTestHostLauncher);
 }
Exemplo n.º 12
0
 public void RunTestsWithCustomTestHost(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
 => inner.RunTestsWithCustomTestHost(testCases, runSettings, options, testRunEventsHandler, customTestHostLauncher);
Exemplo n.º 13
0
 public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
 => inner.RunTestsWithCustomTestHost(sources, runSettings, testRunEventsHandler, customTestHostLauncher);
Exemplo n.º 14
0
 /// <inheritdoc/>
 public void StartTestRunWithCustomHost(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler runEventsHandler, ITestHostLauncher customHostLauncher)
 {
     this.SendMessageAndListenAndReportTestResults(
         MessageType.GetTestRunnerProcessStartInfoForRunSelected,
         new TestRunRequestPayload
     {
         TestCases           = testCases.ToList(),
         RunSettings         = runSettings,
         DebuggingEnabled    = customHostLauncher.IsDebug,
         TestPlatformOptions = options
     },
         runEventsHandler,
         customHostLauncher);
 }
Exemplo n.º 15
0
        /// <inheritdoc/>
        public async Task RunTestsWithCustomTestHostAsync(IEnumerable <TestCase> testCases, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
        {
            var testCaseList = testCases.ToList();

            this.testPlatformEventSource.TranslationLayerExecutionStart(1, 0, testCaseList.Count, runSettings ?? string.Empty);

            await this.EnsureInitializedAsync();

            await this.requestSender.StartTestRunWithCustomHostAsync(testCaseList, runSettings, testRunEventsHandler, customTestHostLauncher);
        }
Exemplo n.º 16
0
        /// <summary>
        /// Initializes a new instance of the <see cref="BaseTestRunCriteria"/> class.
        /// </summary>
        /// <param name="frequencyOfRunStatsChangeEvent">
        /// Frequency of <c>TestRunChangedEvent</c>.
        /// </param>
        /// <param name="keepAlive">
        /// Specify if the test host process should be stay alive after run.
        /// </param>
        /// <param name="testSettings">
        /// Test run settings.
        /// </param>
        /// <param name="runStatsChangeEventTimeout">
        /// Timeout for a <c>TestRunChangedEvent</c>.
        /// </param>
        /// <param name="testHostLauncher">
        /// Test host launcher.
        /// </param>
        public BaseTestRunCriteria(long frequencyOfRunStatsChangeEvent, bool keepAlive, string testSettings, TimeSpan runStatsChangeEventTimeout, ITestHostLauncher testHostLauncher)
        {
            if (frequencyOfRunStatsChangeEvent <= 0)
            {
                throw new ArgumentOutOfRangeException(nameof(frequencyOfRunStatsChangeEvent), Resources.NotificationFrequencyIsNotPositive);
            }

            if (runStatsChangeEventTimeout <= TimeSpan.MinValue)
            {
                throw new ArgumentOutOfRangeException(nameof(runStatsChangeEventTimeout), Resources.NotificationTimeoutIsZero);
            }

            this.FrequencyOfRunStatsChangeEvent = frequencyOfRunStatsChangeEvent;
            this.KeepAlive                  = keepAlive;
            this.TestRunSettings            = testSettings;
            this.RunStatsChangeEventTimeout = runStatsChangeEventTimeout;
            this.TestHostLauncher           = testHostLauncher;
        }
Exemplo n.º 17
0
        /// <inheritdoc/>
        public async Task StartTestRunWithCustomHostAsync(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler runEventsHandler, ITestHostLauncher customHostLauncher)
        {
            if (EqtTrace.IsInfoEnabled)
            {
                EqtTrace.Info("VsTestConsoleRequestSender.StartTestRunWithCustomHostAsync: Starting test run.");
            }

            await this.SendMessageAndListenAndReportTestResultsAsync(
                MessageType.GetTestRunnerProcessStartInfoForRunSelected,
                new TestRunRequestPayload()
            {
                TestCases           = testCases.ToList(),
                RunSettings         = runSettings,
                DebuggingEnabled    = customHostLauncher.IsDebug,
                TestPlatformOptions = options
            },
                runEventsHandler,
                customHostLauncher);
        }
Exemplo n.º 18
0
 /// <inheritdoc/>
 public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
 {
     this.RunTestsWithCustomTestHost(sources, runSettings, options: null, testRunEventsHandler: testRunEventsHandler, customTestHostLauncher: customTestHostLauncher);
 }
Exemplo n.º 19
0
 /// <inheritdoc/>
 public void SetCustomLauncher(ITestHostLauncher customLauncher)
 {
     this.customTestHostLauncher = customLauncher;
 }
Exemplo n.º 20
0
        /// <inheritdoc />
        public void RunTests(
            TestRunRequestPayload testRunRequestPayload,
            ITestHostLauncher testHostLauncher,
            ITestRunEventsRegistrar testRunEventsRegistrar,
            ProtocolConfig protocolConfig)
        {
            EqtTrace.Info("TestRequestManager.RunTests: run tests started.");

            TestRunCriteria runCriteria = null;
            var             runsettings = testRunRequestPayload.RunSettings;

            if (testRunRequestPayload.TestPlatformOptions != null)
            {
                this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
            }

            var requestData = this.GetRequestData(protocolConfig);

            // Get sources to auto detect fx and arch for both run selected or run all scenario.
            var sources = GetSources(testRunRequestPayload);

            if (this.UpdateRunSettingsIfRequired(
                    runsettings,
                    sources,
                    testRunEventsRegistrar,
                    out string updatedRunsettings))
            {
                runsettings = updatedRunsettings;
            }

            if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(runsettings))
            {
                throw new SettingsException(
                          string.Format(
                              Resources.RunsettingsWithDCErrorMessage,
                              runsettings));
            }

            var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
            var batchSize        = runConfiguration.BatchSize;

            if (requestData.IsTelemetryOptedIn)
            {
                // Collect metrics.
                this.CollectMetrics(requestData, runConfiguration);

                // Collect commands.
                this.LogCommandsTelemetryPoints(requestData);

                // Collect data for legacy settings.
                this.LogTelemetryForLegacySettings(requestData, runsettings);
            }

            // Get Fakes data collector settings.
            if (!string.Equals(Environment.GetEnvironmentVariable("VSTEST_SKIP_FAKES_CONFIGURATION"), "1"))
            {
                // The commandline options do not have sources in design time mode,
                // and so we fall back to using sources instead.
                if (this.commandLineOptions.Sources.Any())
                {
                    GenerateFakesUtilities.GenerateFakesSettings(
                        this.commandLineOptions,
                        this.commandLineOptions.Sources.ToList(),
                        ref runsettings);
                }
                else if (sources.Any())
                {
                    GenerateFakesUtilities.GenerateFakesSettings(
                        this.commandLineOptions,
                        sources,
                        ref runsettings);
                }
            }

            if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
            {
                runCriteria = new TestRunCriteria(
                    testRunRequestPayload.Sources,
                    batchSize,
                    testRunRequestPayload.KeepAlive,
                    runsettings,
                    this.commandLineOptions.TestStatsEventTimeout,
                    testHostLauncher,
                    testRunRequestPayload.TestPlatformOptions?.TestCaseFilter,
                    testRunRequestPayload.TestPlatformOptions?.FilterOptions,
                    testRunRequestPayload.TestSessionInfo,
                    debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null &&
                    testRunRequestPayload.DebuggingEnabled);
            }
            else
            {
                runCriteria = new TestRunCriteria(
                    testRunRequestPayload.TestCases,
                    batchSize,
                    testRunRequestPayload.KeepAlive,
                    runsettings,
                    this.commandLineOptions.TestStatsEventTimeout,
                    testHostLauncher,
                    testRunRequestPayload.TestSessionInfo,
                    debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null &&
                    testRunRequestPayload.DebuggingEnabled);
            }

            // Run tests.
            try
            {
                this.RunTests(
                    requestData,
                    runCriteria,
                    testRunEventsRegistrar,
                    testRunRequestPayload.TestPlatformOptions);
                EqtTrace.Info("TestRequestManager.RunTests: run tests completed.");
            }
            finally
            {
                this.TestPlatformEventSourceInstance.ExecutionRequestStop();

                // Post the run complete event
                this.metricsPublisher.Result.PublishMetrics(
                    TelemetryDataConstants.TestExecutionCompleteEvent,
                    requestData.MetricsCollection.Metrics);
            }
        }