public void SetCustomLauncher(ITestHostLauncher customLauncher) { throw new NotImplementedException(); }
/// <inheritdoc/> public void RunTestsWithCustomTestHost(IEnumerable <TestCase> testCases, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) { this.EnsureInitialized(); this.requestSender.StartTestRunWithCustomHost(testCases, runSettings, testRunEventsHandler, customTestHostLauncher); }
private void SendMessageAndListenAndReportTestResults(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher) { try { this.communicationManager.SendMessage(messageType, payload, this.protocolVersion); var isTestRunComplete = false; // Cycle through the messages that the testhost sends. // Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification. while (!isTestRunComplete) { var message = this.TryReceiveMessage(); if (string.Equals(MessageType.TestRunStatsChange, message.MessageType)) { var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>( message); eventHandler.HandleTestRunStatsChange(testRunChangedArgs); } else if (string.Equals(MessageType.ExecutionComplete, message.MessageType)) { var testRunCompletePayload = this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message); eventHandler.HandleTestRunComplete( testRunCompletePayload.TestRunCompleteArgs, testRunCompletePayload.LastRunTests, testRunCompletePayload.RunAttachments, testRunCompletePayload.ExecutorUris); isTestRunComplete = true; } else if (string.Equals(MessageType.TestMessage, message.MessageType)) { var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message); eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message); } else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType)) { HandleCustomHostLaunch(customHostLauncher, message); } } } catch (Exception exception) { EqtTrace.Error("Aborting Test Run Operation: {0}", exception); eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun); var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); this.CleanupCommunicationIfProcessExit(); } this.testPlatformEventSource.TranslationLayerExecutionStop(); }
/// <inheritdoc/> public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, TestPlatformOptions options, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) { var sourceList = sources.ToList(); this.testPlatformEventSource.TranslationLayerExecutionStart(1, sourceList.Count, 0, runSettings ?? string.Empty); this.EnsureInitialized(); this.requestSender.StartTestRunWithCustomHost(sourceList, runSettings, options, testRunEventsHandler, customTestHostLauncher); }
/// <summary> /// Run Tests with given a set of test cases. /// </summary> /// <param name="testRunRequestPayload">TestRun request Payload</param> /// <param name="testHostLauncher">TestHost Launcher for the run</param> /// <param name="testRunEventsRegistrar">event registrar for run events</param> /// <param name="protocolConfig">Protocol related information</param> /// <returns>True, if successful</returns> public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; if (testRunRequestPayload.TestPlatformOptions != null) { this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics; } var requestData = this.GetRequestData(protocolConfig); // Get sources to auto detect fx and arch for both run selected or run all scenario. var sources = GetSources(testRunRequestPayload); if (this.UpdateRunSettingsIfRequired(runsettings, sources, out string updatedRunsettings)) { runsettings = updatedRunsettings; } if (InferRunSettingsHelper.AreRunSettingsCollectorsInCompatibleWithTestSettings(runsettings)) { throw new SettingsException(string.Format(Resources.RunsettingsWithDCErrorMessage, runsettings)); } var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings); var batchSize = runConfiguration.BatchSize; if (requestData.IsTelemetryOptedIn) { // Collect Metrics this.CollectMetrics(requestData, runConfiguration); // Collect Commands this.LogCommandsTelemetryPoints(requestData); } if (!commandLineOptions.IsDesignMode) { // Generate fakes settings only for command line scenarios. In case of // Editors/IDEs, this responsibility is with the caller. GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings); } if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); runCriteria.TestCaseFilter = testRunRequestPayload.TestPlatformOptions?.TestCaseFilter; runCriteria.FilterOptions = testRunRequestPayload.TestPlatformOptions?.FilterOptions; } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); } var success = this.RunTests(requestData, runCriteria, testRunEventsRegistrar); EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success); this.testPlatformEventSource.ExecutionRequestStop(); // Post the run complete event this.metricsPublisher.Result.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics); return(success); }
private async Task SendMessageAndListenAndReportTestResultsAsync(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher) { try { this.communicationManager.SendMessage(messageType, payload, this.protocolVersion); var isTestRunComplete = false; // Cycle through the messages that the testhost sends. // Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification. while (!isTestRunComplete) { var message = await this.TryReceiveMessageAsync(); if (string.Equals(MessageType.TestRunStatsChange, message.MessageType)) { var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>( message); eventHandler.HandleTestRunStatsChange(testRunChangedArgs); } else if (string.Equals(MessageType.ExecutionComplete, message.MessageType)) { if (EqtTrace.IsInfoEnabled) { EqtTrace.Info("VsTestConsoleRequestSender.SendMessageAndListenAndReportTestResultsAsync: Execution complete."); } var testRunCompletePayload = this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message); eventHandler.HandleTestRunComplete( testRunCompletePayload.TestRunCompleteArgs, testRunCompletePayload.LastRunTests, testRunCompletePayload.RunAttachments, testRunCompletePayload.ExecutorUris); isTestRunComplete = true; } else if (string.Equals(MessageType.TestMessage, message.MessageType)) { var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message); eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message); } else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType)) { HandleCustomHostLaunch(customHostLauncher, message); } else if (string.Equals(MessageType.EditorAttachDebugger, message.MessageType)) { AttachDebuggerToProcess(customHostLauncher, message); } } } catch (Exception exception) { EqtTrace.Error("Aborting Test Run Operation: {0}", exception); eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun); var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); // Earlier we were closing the connection with vstest.console in case of exceptions // Removing that code because vstest.console might be in a healthy state and letting the client // know of the error, so that the TL can wait for the next instruction from the client itself. // Also, connection termination might not kill the process which could result in files being locked by testhost. } this.testPlatformEventSource.TranslationLayerExecutionStop(); }
/// <inheritdoc/> public void StartTestSession( StartTestSessionPayload payload, ITestHostLauncher testHostLauncher, ITestSessionEventsHandler eventsHandler, ProtocolConfig protocolConfig) { EqtTrace.Info("TestRequestManager.StartTestSession: Starting test session."); if (payload.TestPlatformOptions != null) { this.telemetryOptedIn = payload.TestPlatformOptions.CollectMetrics; } var requestData = this.GetRequestData(protocolConfig); if (this.UpdateRunSettingsIfRequired( payload.RunSettings, payload.Sources, null, out string updatedRunsettings)) { payload.RunSettings = updatedRunsettings; } if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(payload.RunSettings)) { throw new SettingsException( string.Format( Resources.RunsettingsWithDCErrorMessage, payload.RunSettings)); } // TODO (copoiena): Collect metrics ? lock (this.syncObject) { try { EqtTrace.Info("TestRequestManager.StartTestRunner: Synchronization context taken."); this.TestPlatformEventSourceInstance.StartTestSessionStart(); var criteria = new StartTestSessionCriteria() { Sources = payload.Sources, RunSettings = payload.RunSettings, TestHostLauncher = testHostLauncher }; this.testPlatform.StartTestSession(requestData, criteria, eventsHandler); } finally { EqtTrace.Info("TestRequestManager.StartTestSession: Starting test session completed."); this.TestPlatformEventSourceInstance.StartTestSessionStop(); // Post the attachments processing complete event. this.metricsPublisher.Result.PublishMetrics( TelemetryDataConstants.StartTestSessionCompleteEvent, requestData.MetricsCollection.Metrics); } } }
public void SetCustomLauncher(ITestHostLauncher customLauncher) { }
/// <summary> /// Initializes a new instance of the <see cref="TestRunCriteria"/> class. /// </summary> /// <param name="tests"> /// Sources which contains tests that should be executed /// </param> /// <param name="frequencyOfRunStatsChangeEvent"> /// Frequency of run stats event /// </param> /// <param name="keepAlive"> /// Whether or not to keep the test executor process alive after run completion /// </param> /// <param name="testSettings"> /// Settings used for this run. /// </param> /// <param name="runStatsChangeEventTimeout"> /// Timeout that triggers sending results regardless of cache size. /// </param> /// <param name="testHostLauncher"> /// Test host launcher. If null then default will be used. /// </param> public TestRunCriteria(IEnumerable <TestCase> tests, long frequencyOfRunStatsChangeEvent, bool keepAlive, string testSettings, TimeSpan runStatsChangeEventTimeout, ITestHostLauncher testHostLauncher) : base(frequencyOfRunStatsChangeEvent, keepAlive, testSettings, runStatsChangeEventTimeout, testHostLauncher) { var testCases = tests as IList <TestCase> ?? tests.ToList(); ValidateArg.NotNullOrEmpty(testCases, "tests"); this.Tests = testCases; }
public TestableDotnetTestHostManager(ITestHostLauncher testHostLauncher, IProcessHelper processHelper, IFileHelper fileHelper) : base(testHostLauncher, processHelper, fileHelper) { }
/// <inheritdoc/> public async Task RunTestsWithCustomTestHostAsync(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) { await RunTestsWithCustomTestHostAsync(sources, runSettings, null, testRunEventsHandler, customTestHostLauncher); }
public void RunTestsWithCustomTestHost(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) => inner.RunTestsWithCustomTestHost(testCases, runSettings, options, testRunEventsHandler, customTestHostLauncher);
public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) => inner.RunTestsWithCustomTestHost(sources, runSettings, testRunEventsHandler, customTestHostLauncher);
/// <inheritdoc/> public void StartTestRunWithCustomHost(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler runEventsHandler, ITestHostLauncher customHostLauncher) { this.SendMessageAndListenAndReportTestResults( MessageType.GetTestRunnerProcessStartInfoForRunSelected, new TestRunRequestPayload { TestCases = testCases.ToList(), RunSettings = runSettings, DebuggingEnabled = customHostLauncher.IsDebug, TestPlatformOptions = options }, runEventsHandler, customHostLauncher); }
/// <inheritdoc/> public async Task RunTestsWithCustomTestHostAsync(IEnumerable <TestCase> testCases, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) { var testCaseList = testCases.ToList(); this.testPlatformEventSource.TranslationLayerExecutionStart(1, 0, testCaseList.Count, runSettings ?? string.Empty); await this.EnsureInitializedAsync(); await this.requestSender.StartTestRunWithCustomHostAsync(testCaseList, runSettings, testRunEventsHandler, customTestHostLauncher); }
/// <summary> /// Initializes a new instance of the <see cref="BaseTestRunCriteria"/> class. /// </summary> /// <param name="frequencyOfRunStatsChangeEvent"> /// Frequency of <c>TestRunChangedEvent</c>. /// </param> /// <param name="keepAlive"> /// Specify if the test host process should be stay alive after run. /// </param> /// <param name="testSettings"> /// Test run settings. /// </param> /// <param name="runStatsChangeEventTimeout"> /// Timeout for a <c>TestRunChangedEvent</c>. /// </param> /// <param name="testHostLauncher"> /// Test host launcher. /// </param> public BaseTestRunCriteria(long frequencyOfRunStatsChangeEvent, bool keepAlive, string testSettings, TimeSpan runStatsChangeEventTimeout, ITestHostLauncher testHostLauncher) { if (frequencyOfRunStatsChangeEvent <= 0) { throw new ArgumentOutOfRangeException(nameof(frequencyOfRunStatsChangeEvent), Resources.NotificationFrequencyIsNotPositive); } if (runStatsChangeEventTimeout <= TimeSpan.MinValue) { throw new ArgumentOutOfRangeException(nameof(runStatsChangeEventTimeout), Resources.NotificationTimeoutIsZero); } this.FrequencyOfRunStatsChangeEvent = frequencyOfRunStatsChangeEvent; this.KeepAlive = keepAlive; this.TestRunSettings = testSettings; this.RunStatsChangeEventTimeout = runStatsChangeEventTimeout; this.TestHostLauncher = testHostLauncher; }
/// <inheritdoc/> public async Task StartTestRunWithCustomHostAsync(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler runEventsHandler, ITestHostLauncher customHostLauncher) { if (EqtTrace.IsInfoEnabled) { EqtTrace.Info("VsTestConsoleRequestSender.StartTestRunWithCustomHostAsync: Starting test run."); } await this.SendMessageAndListenAndReportTestResultsAsync( MessageType.GetTestRunnerProcessStartInfoForRunSelected, new TestRunRequestPayload() { TestCases = testCases.ToList(), RunSettings = runSettings, DebuggingEnabled = customHostLauncher.IsDebug, TestPlatformOptions = options }, runEventsHandler, customHostLauncher); }
/// <inheritdoc/> public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) { this.RunTestsWithCustomTestHost(sources, runSettings, options: null, testRunEventsHandler: testRunEventsHandler, customTestHostLauncher: customTestHostLauncher); }
/// <inheritdoc/> public void SetCustomLauncher(ITestHostLauncher customLauncher) { this.customTestHostLauncher = customLauncher; }
/// <inheritdoc /> public void RunTests( TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; if (testRunRequestPayload.TestPlatformOptions != null) { this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics; } var requestData = this.GetRequestData(protocolConfig); // Get sources to auto detect fx and arch for both run selected or run all scenario. var sources = GetSources(testRunRequestPayload); if (this.UpdateRunSettingsIfRequired( runsettings, sources, testRunEventsRegistrar, out string updatedRunsettings)) { runsettings = updatedRunsettings; } if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(runsettings)) { throw new SettingsException( string.Format( Resources.RunsettingsWithDCErrorMessage, runsettings)); } var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings); var batchSize = runConfiguration.BatchSize; if (requestData.IsTelemetryOptedIn) { // Collect metrics. this.CollectMetrics(requestData, runConfiguration); // Collect commands. this.LogCommandsTelemetryPoints(requestData); // Collect data for legacy settings. this.LogTelemetryForLegacySettings(requestData, runsettings); } // Get Fakes data collector settings. if (!string.Equals(Environment.GetEnvironmentVariable("VSTEST_SKIP_FAKES_CONFIGURATION"), "1")) { // The commandline options do not have sources in design time mode, // and so we fall back to using sources instead. if (this.commandLineOptions.Sources.Any()) { GenerateFakesUtilities.GenerateFakesSettings( this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings); } else if (sources.Any()) { GenerateFakesUtilities.GenerateFakesSettings( this.commandLineOptions, sources, ref runsettings); } } if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher, testRunRequestPayload.TestPlatformOptions?.TestCaseFilter, testRunRequestPayload.TestPlatformOptions?.FilterOptions, testRunRequestPayload.TestSessionInfo, debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null && testRunRequestPayload.DebuggingEnabled); } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher, testRunRequestPayload.TestSessionInfo, debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null && testRunRequestPayload.DebuggingEnabled); } // Run tests. try { this.RunTests( requestData, runCriteria, testRunEventsRegistrar, testRunRequestPayload.TestPlatformOptions); EqtTrace.Info("TestRequestManager.RunTests: run tests completed."); } finally { this.TestPlatformEventSourceInstance.ExecutionRequestStop(); // Post the run complete event this.metricsPublisher.Result.PublishMetrics( TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics); } }