private async Task SendMessageAndListenAndReportTestResultsAsync(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher) { try { this.communicationManager.SendMessage(messageType, payload, this.protocolVersion); var isTestRunComplete = false; // Cycle through the messages that the testhost sends. // Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification. while (!isTestRunComplete) { var message = await this.TryReceiveMessageAsync(); if (string.Equals(MessageType.TestRunStatsChange, message.MessageType)) { var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>( message); eventHandler.HandleTestRunStatsChange(testRunChangedArgs); } else if (string.Equals(MessageType.ExecutionComplete, message.MessageType)) { var testRunCompletePayload = this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message); eventHandler.HandleTestRunComplete( testRunCompletePayload.TestRunCompleteArgs, testRunCompletePayload.LastRunTests, testRunCompletePayload.RunAttachments, testRunCompletePayload.ExecutorUris); isTestRunComplete = true; } else if (string.Equals(MessageType.TestMessage, message.MessageType)) { var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message); eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message); } else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType)) { HandleCustomHostLaunch(customHostLauncher, message); } } } catch (Exception exception) { EqtTrace.Error("Aborting Test Run Operation: {0}", exception); eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun); var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); // Earlier we were closing the connection with vstest.console in case of exceptions // Removing that code because vstest.console might be in a healthy state and letting the client // know of the error, so that the TL can wait for the next instruction from the client itself. // Also, connection termination might not kill the process which could result in files being locked by testhost. } this.testPlatformEventSource.TranslationLayerExecutionStop(); }
private async Task SendMessageAndListenAndReportTestResultsAsync(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher) { try { this.communicationManager.SendMessage(messageType, payload, this.protocolVersion); var isTestRunComplete = false; // Cycle through the messages that the testhost sends. // Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification. while (!isTestRunComplete) { var message = await this.TryReceiveMessageAsync(); if (string.Equals(MessageType.TestRunStatsChange, message.MessageType)) { var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>( message); eventHandler.HandleTestRunStatsChange(testRunChangedArgs); } else if (string.Equals(MessageType.ExecutionComplete, message.MessageType)) { var testRunCompletePayload = this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message); eventHandler.HandleTestRunComplete( testRunCompletePayload.TestRunCompleteArgs, testRunCompletePayload.LastRunTests, testRunCompletePayload.RunAttachments, testRunCompletePayload.ExecutorUris); isTestRunComplete = true; } else if (string.Equals(MessageType.TestMessage, message.MessageType)) { var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message); eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message); } else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType)) { HandleCustomHostLaunch(customHostLauncher, message); } } } catch (Exception exception) { EqtTrace.Error("Aborting Test Run Operation: {0}", exception); eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun); var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); this.CleanupCommunicationIfProcessExit(); } this.testPlatformEventSource.TranslationLayerExecutionStop(); }
private void OnTestRunAbort(ITestRunEventsHandler testRunEventsHandler, Exception exception, bool getClientError) { if (this.IsOperationComplete()) { EqtTrace.Verbose("TestRequestSender: OnTestRunAbort: Operation is already complete. Skip error message."); return; } EqtTrace.Verbose("TestRequestSender: OnTestRunAbort: Set operation complete."); this.SetOperationComplete(); var reason = this.GetAbortErrorMessage(exception, getClientError); EqtTrace.Error("TestRequestSender: Aborting test run because {0}", reason); this.LogErrorMessage(string.Format(CommonResources.AbortedTestRun, reason)); // notify test run abort to vstest console wrapper. var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero); var payload = new TestRunCompletePayload { TestRunCompleteArgs = completeArgs }; var rawMessage = this.dataSerializer.SerializePayload(MessageType.ExecutionComplete, payload); testRunEventsHandler.HandleRawMessage(rawMessage); // notify of a test run complete and bail out. testRunEventsHandler.HandleTestRunComplete(completeArgs, null, null, null); }
// mock a VsTest run. Provides test result one by one at 10 ms intervals // note: a lot of information is still missing (vs real VsTest). You will have to add them if your test requires them private void MoqTestRun(ITestRunEventsHandler testRunEvents, IReadOnlyList <TestResult> testResults, TestCase timeOutTest = null) { Task.Run(() => { var timer = new Stopwatch(); testRunEvents.HandleTestRunStatsChange( new TestRunChangedEventArgs(new TestRunStatistics(0, null), null, timeOutTest == null ? null : new[] { timeOutTest })); for (var i = 0; i < testResults.Count; i++) { Thread.Sleep(10); testResults[i].EndTime = DateTimeOffset.Now; testRunEvents.HandleTestRunStatsChange(new TestRunChangedEventArgs( new TestRunStatistics(i + 1, null), new[] { testResults[i] }, null)); } if (timeOutTest != null) { testRunEvents.HandleTestRunStatsChange(new TestRunChangedEventArgs( new TestRunStatistics(testResults.Count, null), null, new[] { timeOutTest })); } Thread.Sleep(10); testRunEvents.HandleTestRunComplete( new TestRunCompleteEventArgs(new TestRunStatistics(testResults.Count, null), false, false, null, null, timer.Elapsed), null, null, null); }); }
private void OnTestRunAbort(ITestRunEventsHandler testRunEventsHandler, Exception exception) { EqtTrace.Error("Server: TestExecution: Aborting test run because {0}", exception); // log console message to vstest console testRunEventsHandler.HandleLogMessage(TestMessageLevel.Error, CommonResources.AbortedTestRun); // log console message to vstest console wrapper var testMessagePayload = new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = CommonResources.AbortedTestRun }; var rawMessage = this.dataSerializer.SerializePayload(MessageType.TestMessage, testMessagePayload); testRunEventsHandler.HandleRawMessage(rawMessage); // notify test run abort to vstest console wrapper var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero); var payload = new TestRunCompletePayload { TestRunCompleteArgs = completeArgs }; rawMessage = this.dataSerializer.SerializePayload(MessageType.ExecutionComplete, payload); testRunEventsHandler.HandleRawMessage(rawMessage); // notify of a test run complete and bail out. testRunEventsHandler.HandleTestRunComplete(completeArgs, null, null, null); this.CleanupCommunicationIfProcessExit(); }
/// <inheritdoc/> public int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { try { var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings); var testPackages = new List <string>(testRunCriteria.HasSpecificSources ? testRunCriteria.Sources : // If the test execution is with a test filter, group them by sources testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key)); // This code should be in sync with ProxyExecutionManager.StartTestRun executionContext var executionContext = new TestExecutionContext( testRunCriteria.FrequencyOfRunStatsChangeEvent, testRunCriteria.RunStatsChangeEventTimeout, inIsolation: runConfiguration.InIsolation, keepAlive: testRunCriteria.KeepAlive, isDataCollectionEnabled: false, areTestCaseLevelEventsRequired: false, hasTestRun: true, isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug), testCaseFilter: testRunCriteria.TestCaseFilter, filterOptions: testRunCriteria.FilterOptions); // Initialize extension before execution this.InitializeExtensions(testPackages); if (testRunCriteria.HasSpecificSources) { var runRequest = testRunCriteria.CreateTestRunCriteriaForSources(testHostManager, testRunCriteria.TestRunSettings, executionContext, testPackages); Task.Run(() => executionManager.StartTestRun(runRequest.AdapterSourceMap, runRequest.Package, runRequest.RunSettings, runRequest.TestExecutionContext, null, eventHandler)); } else { var runRequest = testRunCriteria.CreateTestRunCriteriaForTests(testHostManager, testRunCriteria.TestRunSettings, executionContext, testPackages); Task.Run(() => executionManager.StartTestRun(runRequest.Tests, runRequest.Package, runRequest.RunSettings, runRequest.TestExecutionContext, null, eventHandler)); } } catch (Exception exception) { EqtTrace.Error("InProcessProxyexecutionManager.StartTestRun: Failed to start test run: {0}", exception); // Send exception message. eventHandler.HandleLogMessage(TestMessageLevel.Error, exception.ToString()); // Send a run complete to caller. var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, new Collection <AttachmentSet>(), TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); } return(0); }
private void OnExecutionMessageReceived(object sender, MessageReceivedEventArgs messageReceived, ITestRunEventsHandler testRunEventsHandler) { try { var rawMessage = messageReceived.Data; // Send raw message first to unblock handlers waiting to send message to IDEs testRunEventsHandler.HandleRawMessage(rawMessage); var message = this.dataSerializer.DeserializeMessage(rawMessage); switch (message.MessageType) { case MessageType.TestRunStatsChange: var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(message); testRunEventsHandler.HandleTestRunStatsChange(testRunChangedArgs); break; case MessageType.ExecutionComplete: var testRunCompletePayload = this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message); testRunEventsHandler.HandleTestRunComplete( testRunCompletePayload.TestRunCompleteArgs, testRunCompletePayload.LastRunTests, testRunCompletePayload.RunAttachments, testRunCompletePayload.ExecutorUris); this.SetOperationComplete(); break; case MessageType.TestMessage: var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message); testRunEventsHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message); break; case MessageType.LaunchAdapterProcessWithDebuggerAttached: var testProcessStartInfo = this.dataSerializer.DeserializePayload <TestProcessStartInfo>(message); int processId = testRunEventsHandler.LaunchProcessWithDebuggerAttached(testProcessStartInfo); var data = this.dataSerializer.SerializePayload( MessageType.LaunchAdapterProcessWithDebuggerAttachedCallback, processId, this.protocolVersion); this.channel.Send(data); break; } } catch (Exception exception) { this.OnTestRunAbort(testRunEventsHandler, exception, false); } }
/// <summary> /// Aborts the test execution. /// </summary> public void Abort(ITestRunEventsHandler testRunEventsHandler) { if (this.activeTestRun == null) { var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero); testRunEventsHandler.HandleTestRunComplete(testRunCompleteEventArgs, null, null, null); } else { this.activeTestRun.Abort(); } }
private void ListenAndReportTestResults(ITestRunEventsHandler testRunEventsHandler) { var isTestRunComplete = false; // Cycle through the messages that the testhost sends. // Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification. while (!isTestRunComplete) { try { var rawMessage = this.TryReceiveRawMessage(); // Send raw message first to unblock handlers waiting to send message to IDEs testRunEventsHandler.HandleRawMessage(rawMessage); var message = this.dataSerializer.DeserializeMessage(rawMessage); if (string.Equals(MessageType.TestRunStatsChange, message.MessageType)) { var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>( message); testRunEventsHandler.HandleTestRunStatsChange(testRunChangedArgs); } else if (string.Equals(MessageType.ExecutionComplete, message.MessageType)) { var testRunCompletePayload = this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message); testRunEventsHandler.HandleTestRunComplete( testRunCompletePayload.TestRunCompleteArgs, testRunCompletePayload.LastRunTests, testRunCompletePayload.RunAttachments, testRunCompletePayload.ExecutorUris); isTestRunComplete = true; } else if (string.Equals(MessageType.TestMessage, message.MessageType)) { var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message); testRunEventsHandler.HandleLogMessage( testMessagePayload.MessageLevel, testMessagePayload.Message); } else if (string.Equals(MessageType.LaunchAdapterProcessWithDebuggerAttached, message.MessageType)) { var testProcessStartInfo = this.dataSerializer.DeserializePayload <TestProcessStartInfo>(message); int processId = testRunEventsHandler.LaunchProcessWithDebuggerAttached(testProcessStartInfo); this.communicationManager.SendMessage( MessageType.LaunchAdapterProcessWithDebuggerAttachedCallback, processId, version: this.protocolVersion); } } catch (IOException exception) { // To avoid further communication with remote host this.sendMessagesToRemoteHost = false; this.OnTestRunAbort(testRunEventsHandler, exception); isTestRunComplete = true; } catch (Exception exception) { this.OnTestRunAbort(testRunEventsHandler, exception); isTestRunComplete = true; } } }
/// <summary> /// Starts the test run /// </summary> /// <param name="testRunCriteria"> The settings/options for the test run. </param> /// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param> /// <returns> The process id of the runner executing tests. </returns> public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { try { var executionEngineStartTime = DateTime.UtcNow; if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("ProxyExecutionManager: Test host is always Lazy initialize."); } var testPackages = new List <string>(testRunCriteria.HasSpecificSources ? testRunCriteria.Sources : // If the test execution is with a test filter, group them by sources testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key)); this.isCommunicationEstablished = this.SetupChannel(testPackages, this.cancellationTokenSource.Token); if (this.isCommunicationEstablished) { if (this.cancellationTokenSource.IsCancellationRequested) { if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("ProxyExecutionManager.StartTestRun: Canceling the current run after getting cancelation request."); } throw new TestPlatformException(Resources.Resources.CancelationRequested); } this.InitializeExtensions(testPackages); // Collecting Time Taken to Start Discovery Engine var executionEngineTotalTime = DateTime.UtcNow - executionEngineStartTime; // Collecting Data Point for Time taken to start Execution Engine. In case of Parallel, it will be maximum time taken. this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenToStartExecutionEngineExe, executionEngineTotalTime.TotalSeconds.ToString()); // This code should be in sync with InProcessProxyExecutionManager.StartTestRun executionContext var executionContext = new TestExecutionContext( testRunCriteria.FrequencyOfRunStatsChangeEvent, testRunCriteria.RunStatsChangeEventTimeout, inIsolation: false, keepAlive: testRunCriteria.KeepAlive, isDataCollectionEnabled: false, areTestCaseLevelEventsRequired: false, hasTestRun: true, isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug), testCaseFilter: testRunCriteria.TestCaseFilter); // This is workaround for the bug https://github.com/Microsoft/vstest/issues/970 var runsettings = this.RemoveNodesFromRunsettingsIfRequired(testRunCriteria.TestRunSettings, (testMessageLevel, message) => { this.LogMessage(testMessageLevel, message, eventHandler); }); if (testRunCriteria.HasSpecificSources) { var runRequest = testRunCriteria.CreateTestRunCriteriaForSources(testHostManager, runsettings, executionContext, testPackages); this.RequestSender.StartTestRun(runRequest, eventHandler); } else { var runRequest = testRunCriteria.CreateTestRunCriteriaForTests(testHostManager, runsettings, executionContext, testPackages); this.RequestSender.StartTestRun(runRequest, eventHandler); } } } catch (Exception exception) { EqtTrace.Error("ProxyExecutionManager.StartTestRun: Failed to start test run: {0}", exception); this.LogMessage(TestMessageLevel.Error, exception.Message, eventHandler); // Send a run complete to caller. Similar logic is also used in ParallelProxyExecutionManager.StartTestRunOnConcurrentManager // Aborted is `true`: in case of parallel run (or non shared host), an aborted message ensures another execution manager // created to replace the current one. This will help if the current execution manager is aborted due to irreparable error // and the test host is lost as well. var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, new Collection <AttachmentSet>(), TimeSpan.Zero, null); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); } return(0); }
/// <summary> /// Starts the test run /// </summary> /// <param name="testRunCriteria"> The settings/options for the test run. </param> /// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param> /// <returns> The process id of the runner executing tests. </returns> public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { try { if (!this.testHostManager.Shared) { // Non shared test host requires test source information to launch. Provide the sources // information and create the channel. EqtTrace.Verbose("ProxyExecutionManager: Test host is non shared. Lazy initialize."); var testSources = testRunCriteria.Sources; // If the test execution is with a test filter, group them by sources if (testRunCriteria.HasSpecificTests) { testSources = testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key); } this.InitializeExtensions(testSources); } this.SetupChannel(testRunCriteria.Sources); var executionContext = new TestExecutionContext( testRunCriteria.FrequencyOfRunStatsChangeEvent, testRunCriteria.RunStatsChangeEventTimeout, inIsolation: false, keepAlive: testRunCriteria.KeepAlive, isDataCollectionEnabled: false, areTestCaseLevelEventsRequired: false, hasTestRun: true, isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug), testCaseFilter: testRunCriteria.TestCaseFilter); if (testRunCriteria.HasSpecificSources) { var runRequest = new TestRunCriteriaWithSources( testRunCriteria.AdapterSourceMap, testRunCriteria.TestRunSettings, executionContext); this.RequestSender.StartTestRun(runRequest, eventHandler); } else { var runRequest = new TestRunCriteriaWithTests( testRunCriteria.Tests, testRunCriteria.TestRunSettings, executionContext); this.RequestSender.StartTestRun(runRequest, eventHandler); } } catch (Exception exception) { EqtTrace.Error("ProxyExecutionManager.StartTestRun: Failed to start test run: {0}", exception); var completeArgs = new TestRunCompleteEventArgs(null, false, false, exception, new Collection <AttachmentSet>(), TimeSpan.Zero); eventHandler.HandleLogMessage(TestMessageLevel.Error, exception.Message); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); } return(0); }