private void StartTestRunErrorTestsTemplate(string errorMessage, Action <string> onClientProcessExitCallback) { var mockHandler = new Mock <ITestRunEventsHandler>(); var runCriteria = new TestRunCriteriaWithSources(null, null, null); this.mockCommunicationManager.Setup(mc => mc.ReceiveRawMessageAsync(It.IsAny <CancellationToken>())) .Callback(() => onClientProcessExitCallback(errorMessage)).Returns(Task.FromResult((string)null)); string testCompleteRawMessage = "{\"MessageType\":\"TestExecution.Completed\",\"Payload\":{\"TestRunCompleteArgs\":{\"TestRunStatistics\":null,\"IsCanceled\":false,\"IsAborted\":true,\"Error\":{\"ClassName\":\"System.IO.IOException\",\"Message\":\"Unable to read data from the transport connection: An existing connection was forcibly closed by the remote host.\",\"Data\":null,\"InnerException\":null},\"AttachmentSets\":null,\"ElapsedTimeInRunningTests\":\"00:00:00\"},\"LastRunTests\":null,\"RunAttachments\":null,\"ExecutorUris\":null}}"; this.mockDataSerializer.Setup( md => md.SerializePayload(MessageType.ExecutionComplete, It.IsAny <TestRunCompletePayload>())) .Returns(testCompleteRawMessage); var waitHandle = new AutoResetEvent(false); mockHandler.Setup(mh => mh.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(), null, null, null)).Callback(() => waitHandle.Set()); this.testRequestSender.InitializeCommunication(); this.testRequestSender.StartTestRun(runCriteria, mockHandler.Object); waitHandle.WaitOne(); this.testRequestSender.EndSession(); mockHandler.Verify(mh => mh.HandleLogMessage(TestMessageLevel.Error, string.Format(CommunicationUtilitiesResources.AbortedTestRun, errorMessage)), Times.Once); mockHandler.Verify(mh => mh.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(), null, null, null), Times.Once); mockHandler.Verify(mh => mh.HandleRawMessage(testCompleteRawMessage), Times.Once); this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.SessionEnd), Times.Never); }
public void StartTestRunShouldCallLaunchProcessWithDebuggerAndWaitForCallback() { var mockHandler = new Mock <ITestRunEventsHandler>(); var runCriteria = new TestRunCriteriaWithSources(null, null, null); var rawMessage = "LaunchProcessWithDebugger"; var message = new Message() { MessageType = MessageType.LaunchAdapterProcessWithDebuggerAttached, Payload = null }; var payload = new TestProcessStartInfo(); this.SetupReceiveRawMessageAsyncAndDeserializeMessageAndInitialize(rawMessage, message); this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestProcessStartInfo>(message)).Returns(payload); var completePayload = new TestRunCompletePayload() { ExecutorUris = null, LastRunTests = null, RunAttachments = null, TestRunCompleteArgs = null }; var completeMessage = new Message() { MessageType = MessageType.ExecutionComplete, Payload = null }; mockHandler.Setup(mh => mh.LaunchProcessWithDebuggerAttached(payload)).Callback( () => { this.mockDataSerializer.Setup(ds => ds.DeserializeMessage(It.IsAny <string>())).Returns(completeMessage); this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestRunCompletePayload>(completeMessage)).Returns(completePayload); }); var waitHandle = new AutoResetEvent(false); mockHandler.Setup(mh => mh.HandleTestRunComplete( It.IsAny <TestRunCompleteEventArgs>(), It.IsAny <TestRunChangedEventArgs>(), It.IsAny <ICollection <AttachmentSet> >(), It.IsAny <ICollection <string> >())).Callback(() => waitHandle.Set()); this.testRequestSender.StartTestRun(runCriteria, mockHandler.Object); waitHandle.WaitOne(); this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.StartTestExecutionWithSources, runCriteria, this.protocolConfig.Version), Times.Once); this.mockDataSerializer.Verify(ds => ds.DeserializeMessage(It.IsAny <string>()), Times.Exactly(2)); mockHandler.Verify(mh => mh.LaunchProcessWithDebuggerAttached(payload), Times.Once); mockHandler.Verify(mh => mh.HandleRawMessage(rawMessage), Times.Exactly(2)); this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.LaunchAdapterProcessWithDebuggerAttachedCallback, It.IsAny <object>(), this.protocolConfig.Version), Times.Once); }
public void StartTestRunWithSourcesShouldCallHandleTestRunStatsChange() { var mockHandler = new Mock <ITestRunEventsHandler>(); var runCriteria = new TestRunCriteriaWithSources(null, null, null); var testRunChangedArgs = new TestRunChangedEventArgs(null, null, null); var rawMessage = "OnTestRunStatsChange"; var message = new Message() { MessageType = MessageType.TestRunStatsChange, Payload = null }; this.SetupReceiveRawMessageAsyncAndDeserializeMessageAndInitialize(rawMessage, message); this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestRunChangedEventArgs>(message)).Returns(testRunChangedArgs); var completePayload = new TestRunCompletePayload() { ExecutorUris = null, LastRunTests = null, RunAttachments = null, TestRunCompleteArgs = null }; var completeMessage = new Message() { MessageType = MessageType.ExecutionComplete, Payload = null }; mockHandler.Setup(mh => mh.HandleTestRunStatsChange(testRunChangedArgs)).Callback( () => { this.mockDataSerializer.Setup(ds => ds.DeserializeMessage(It.IsAny <string>())).Returns(completeMessage); this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestRunCompletePayload>(completeMessage)).Returns(completePayload); }); var waitHandle = new AutoResetEvent(false); mockHandler.Setup(mh => mh.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(), It.IsAny <TestRunChangedEventArgs>(), It.IsAny <ICollection <AttachmentSet> >(), It.IsAny <ICollection <string> >())).Callback (() => waitHandle.Set()); this.testRequestSender.StartTestRun(runCriteria, mockHandler.Object); waitHandle.WaitOne(); this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.StartTestExecutionWithSources, runCriteria), Times.Once); // One for run stats and another for runcomplete this.mockDataSerializer.Verify(ds => ds.DeserializeMessage(rawMessage), Times.Exactly(2)); mockHandler.Verify(mh => mh.HandleTestRunStatsChange(testRunChangedArgs), Times.Once); mockHandler.Verify(mh => mh.HandleRawMessage(rawMessage), Times.Exactly(2)); }
public TestRequestSenderTests2() { this.mockChannel = new Mock <ICommunicationChannel>(); this.mockServer = new Mock <ICommunicationServer>(); this.mockDataSerializer = new Mock <IDataSerializer>(); this.testRequestSender = new TestableTestRequestSender(this.mockServer.Object, this.mockDataSerializer.Object, new ProtocolConfig { Version = DUMMYPROTOCOLVERSION }); this.connectedEventArgs = new ConnectedEventArgs(this.mockChannel.Object); this.mockDiscoveryEventsHandler = new Mock <ITestDiscoveryEventsHandler2>(); this.mockExecutionEventsHandler = new Mock <ITestRunEventsHandler>(); this.testRunCriteriaWithSources = new TestRunCriteriaWithSources(new Dictionary <string, IEnumerable <string> >(), null, "runsettings", null); }
/// <inheritdoc /> public void StartTestRun(TestRunCriteriaWithSources runCriteria, ITestRunEventsHandler eventHandler) { this.messageEventHandler = eventHandler; this.onDisconnected = (disconnectedEventArgs) => { this.OnTestRunAbort(eventHandler, disconnectedEventArgs.Error, true); }; this.onMessageReceived = (sender, args) => this.OnExecutionMessageReceived(sender, args, eventHandler); this.channel.MessageReceived += this.onMessageReceived; // This code section is needed because we altered the old testhost launch process for // the debugging workflow. Now we don't ask VS to launch and attach to the testhost // process for us as we previously did, instead we launch it as a standalone process // and rely on the testhost to ask VS to attach the debugger to itself. // // In order to avoid breaking compatibility with previous testhost versions because of // those changes (older testhosts won't know to request VS to attach to themselves // thinking instead VS launched and attached to them already), we request VS to attach // to the testhost here before starting the test run. if (runCriteria.TestExecutionContext != null && runCriteria.TestExecutionContext.IsDebug && this.runtimeProvider is ITestRuntimeProvider2 convertedRuntimeProvider && this.protocolVersion < ObjectModelConstants.MinimumProtocolVersionWithDebugSupport) { var handler = (ITestRunEventsHandler2)eventHandler; if (!convertedRuntimeProvider.AttachDebuggerToTestHost()) { EqtTrace.Warning( string.Format( CultureInfo.CurrentUICulture, CommonResources.AttachDebuggerToDefaultTestHostFailure)); } } var message = this.dataSerializer.SerializePayload( MessageType.StartTestExecutionWithSources, runCriteria, this.protocolVersion); if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("TestRequestSender.StartTestRun: Sending test run with message: {0}", message); } this.channel.Send(message); }
public void StartTestRunShouldCallHandleLogMessageOnTestMessage() { var mockHandler = new Mock <ITestRunEventsHandler>(); var runCriteria = new TestRunCriteriaWithSources(null, null, null, null); var rawMessage = "OnLogMessage"; var message = new Message() { MessageType = MessageType.TestMessage, Payload = null }; var payload = new TestMessagePayload() { MessageLevel = TestMessageLevel.Error, Message = rawMessage }; this.SetupReceiveRawMessageAsyncAndDeserializeMessageAndInitialize(rawMessage, message); this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestMessagePayload>(message)).Returns(payload); var completePayload = new TestRunCompletePayload() { ExecutorUris = null, LastRunTests = null, RunAttachments = null, TestRunCompleteArgs = null }; var completeMessage = new Message() { MessageType = MessageType.ExecutionComplete, Payload = null }; var waitHandle = new AutoResetEvent(false); mockHandler.Setup(mh => mh.HandleLogMessage(TestMessageLevel.Error, rawMessage)).Callback( () => { this.mockDataSerializer.Setup(ds => ds.DeserializeMessage(It.IsAny <string>())).Returns(completeMessage); this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestRunCompletePayload>(completeMessage)).Callback(() => { waitHandle.Set(); }) .Returns(completePayload); }); this.testRequestSender.StartTestRun(runCriteria, mockHandler.Object); waitHandle.WaitOne(); this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.StartTestExecutionWithSources, runCriteria, this.protocolConfig.Version), Times.Once); this.mockDataSerializer.Verify(ds => ds.DeserializeMessage(It.IsAny <string>()), Times.AtLeast(2)); // Asserting that 'StartTestRun" should have been completed, & invoked only once this.mockDataSerializer.Verify(ds => ds.DeserializePayload <TestRunCompletePayload>(completeMessage), Times.Exactly(1)); mockHandler.Verify(mh => mh.HandleLogMessage(payload.MessageLevel, payload.Message), Times.Once); mockHandler.Verify(mh => mh.HandleRawMessage(rawMessage), Times.AtLeastOnce); }
/// <inheritdoc /> public void StartTestRun(TestRunCriteriaWithSources runCriteria, ITestRunEventsHandler eventHandler) { this.messageEventHandler = eventHandler; this.onDisconnected = (disconnectedEventArgs) => { this.OnTestRunAbort(eventHandler, disconnectedEventArgs.Error, true); }; this.onMessageReceived = (sender, args) => this.OnExecutionMessageReceived(sender, args, eventHandler); this.channel.MessageReceived += this.onMessageReceived; var message = this.dataSerializer.SerializePayload( MessageType.StartTestExecutionWithSources, runCriteria, this.protocolVersion); this.channel.Send(message); }
public TestRequestSenderTests() { this.connectionInfo = new TestHostConnectionInfo { Endpoint = IPAddress.Loopback + ":123", Role = ConnectionRole.Client, Transport = Transport.Sockets }; this.mockChannel = new Mock <ICommunicationChannel>(); this.mockServer = new Mock <ICommunicationEndPoint>(); this.mockDataSerializer = new Mock <IDataSerializer>(); this.testRequestSender = new TestableTestRequestSender(this.mockServer.Object, this.connectionInfo, this.mockDataSerializer.Object, new ProtocolConfig { Version = DUMMYPROTOCOLVERSION }); this.connectedEventArgs = new ConnectedEventArgs(this.mockChannel.Object); this.mockDiscoveryEventsHandler = new Mock <ITestDiscoveryEventsHandler2>(); this.mockExecutionEventsHandler = new Mock <ITestRunEventsHandler>(); this.testRunCriteriaWithSources = new TestRunCriteriaWithSources(new Dictionary <string, IEnumerable <string> >(), "runsettings", null, null); }
public void ProcessRequestsExecutionStartShouldStartExecutionWithGivenSources() { var asm = new Dictionary <string, IEnumerable <string> >(); asm["mstestv2"] = new[] { "test1.dll", "test2.dll" }; var testRunCriteriaWithSources = new TestRunCriteriaWithSources(asm, "runsettings", null, null); var message = this.dataSerializer.SerializePayload(MessageType.StartTestExecutionWithSources, testRunCriteriaWithSources); this.ProcessRequestsAsync(this.mockTestHostManagerFactory.Object); this.SendMessageOnChannel(message); this.jobQueue.Flush(); mockExecutionManager.Verify(e => e.StartTestRun( It.Is <Dictionary <string, IEnumerable <string> > >(d => d.ContainsKey("mstestv2")), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <TestExecutionContext>(), It.IsAny <ITestCaseEventsHandler>(), It.IsAny <ITestRunEventsHandler>())); this.SendSessionEnd(); }
public void StartTestRunShouldInitiateTestRunForSourcesThroughTheServer() { TestRunCriteriaWithSources testRunCriteriaPassed = null; this.mockRequestSender.Setup(s => s.WaitForRequestHandlerConnection(It.IsAny <int>())).Returns(true); this.mockRequestSender.Setup(s => s.StartTestRun(It.IsAny <TestRunCriteriaWithSources>(), null)) .Callback( (TestRunCriteriaWithSources criteria, ITestRunEventsHandler sink) => { testRunCriteriaPassed = criteria; }); this.testExecutionManager.StartTestRun(this.mockTestRunCriteria.Object, null); Assert.IsNotNull(testRunCriteriaPassed); CollectionAssert.AreEqual(this.mockTestRunCriteria.Object.AdapterSourceMap.Keys, testRunCriteriaPassed.AdapterSourceMap.Keys); CollectionAssert.AreEqual(this.mockTestRunCriteria.Object.AdapterSourceMap.Values, testRunCriteriaPassed.AdapterSourceMap.Values); Assert.AreEqual(this.mockTestRunCriteria.Object.FrequencyOfRunStatsChangeEvent, testRunCriteriaPassed.TestExecutionContext.FrequencyOfRunStatsChangeEvent); Assert.AreEqual(this.mockTestRunCriteria.Object.RunStatsChangeEventTimeout, testRunCriteriaPassed.TestExecutionContext.RunStatsChangeEventTimeout); Assert.AreEqual(this.mockTestRunCriteria.Object.TestRunSettings, testRunCriteriaPassed.RunSettings); }
/// <inheritdoc /> public void StartTestRun(TestRunCriteriaWithSources runCriteria, ITestRunEventsHandler eventHandler) { this.messageEventHandler = eventHandler; this.onDisconnected = (disconnectedEventArgs) => { this.OnTestRunAbort(eventHandler, disconnectedEventArgs.Error, true); }; this.onMessageReceived = (sender, args) => this.OnExecutionMessageReceived(sender, args, eventHandler); this.channel.MessageReceived += this.onMessageReceived; var message = this.dataSerializer.SerializePayload( MessageType.StartTestExecutionWithSources, runCriteria, this.protocolVersion); if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("TestRequestSender.StartTestRun: Sending test run with message: {0}", message); } this.channel.Send(message); }
/// <summary> /// Executes tests on the sources specified with the criteria mentioned. /// </summary> /// <param name="runCriteria">The test run criteria.</param> /// <param name="eventHandler">The handler for execution events from the test host.</param> public void StartTestRun(TestRunCriteriaWithSources runCriteria, ITestRunEventsHandler eventHandler) { this.StartTestRunAndListenAndReportTestResults(MessageType.StartTestExecutionWithSources, runCriteria, eventHandler); }
/// <summary> /// Starts the test run /// </summary> /// <param name="testRunCriteria"> The settings/options for the test run. </param> /// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param> /// <returns> The process id of the runner executing tests. </returns> public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { try { if (!this.testHostManager.Shared) { // Non shared test host requires test source information to launch. Provide the sources // information and create the channel. EqtTrace.Verbose("ProxyExecutionManager: Test host is non shared. Lazy initialize."); var testSources = testRunCriteria.Sources; // If the test execution is with a test filter, group them by sources if (testRunCriteria.HasSpecificTests) { testSources = testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key); } this.InitializeExtensions(testSources); } this.SetupChannel(testRunCriteria.Sources); var executionContext = new TestExecutionContext( testRunCriteria.FrequencyOfRunStatsChangeEvent, testRunCriteria.RunStatsChangeEventTimeout, inIsolation: false, keepAlive: testRunCriteria.KeepAlive, isDataCollectionEnabled: false, areTestCaseLevelEventsRequired: false, hasTestRun: true, isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug), testCaseFilter: testRunCriteria.TestCaseFilter); if (testRunCriteria.HasSpecificSources) { var runRequest = new TestRunCriteriaWithSources( testRunCriteria.AdapterSourceMap, testRunCriteria.TestRunSettings, executionContext); this.RequestSender.StartTestRun(runRequest, eventHandler); } else { var runRequest = new TestRunCriteriaWithTests( testRunCriteria.Tests, testRunCriteria.TestRunSettings, executionContext); this.RequestSender.StartTestRun(runRequest, eventHandler); } } catch (Exception exception) { EqtTrace.Error("ProxyExecutionManager.StartTestRun: Failed to start test run: {0}", exception); var completeArgs = new TestRunCompleteEventArgs(null, false, false, exception, new Collection <AttachmentSet>(), TimeSpan.Zero); eventHandler.HandleLogMessage(TestMessageLevel.Error, exception.Message); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); } return(0); }
/// <summary> /// Starts the test run /// </summary> /// <param name="testRunCriteria"> The settings/options for the test run. </param> /// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param> /// <returns> The process id of the runner executing tests. </returns> public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { try { EqtTrace.Verbose("ProxyExecutionManager: Test host is always Lazy initialize."); var testSources = testRunCriteria.Sources; // If the test execution is with a test filter, group them by sources if (testRunCriteria.HasSpecificTests) { testSources = testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key); } this.isCommunicationEstablished = this.SetupChannel(testSources, this.cancellationTokenSource.Token); if (this.isCommunicationEstablished) { if (this.cancellationTokenSource.IsCancellationRequested) { if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("ProxyExecutionManager.StartTestRun: Canceling the current run after getting cancelation request."); } throw new TestPlatformException(Resources.Resources.CancelationRequested); } this.InitializeExtensions(testSources); var executionContext = new TestExecutionContext( testRunCriteria.FrequencyOfRunStatsChangeEvent, testRunCriteria.RunStatsChangeEventTimeout, inIsolation: false, keepAlive: testRunCriteria.KeepAlive, isDataCollectionEnabled: false, areTestCaseLevelEventsRequired: false, hasTestRun: true, isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug), testCaseFilter: testRunCriteria.TestCaseFilter); // This is workaround for the bug https://github.com/Microsoft/vstest/issues/970 var runsettings = this.RemoveNodesFromRunsettingsIfRequired(testRunCriteria.TestRunSettings, (testMessageLevel, message) => { this.LogMessage(testMessageLevel, message, eventHandler); }); if (testRunCriteria.HasSpecificSources) { var runRequest = new TestRunCriteriaWithSources( testRunCriteria.AdapterSourceMap, runsettings, executionContext); this.RequestSender.StartTestRun(runRequest, eventHandler); } else { var runRequest = new TestRunCriteriaWithTests( testRunCriteria.Tests, runsettings, executionContext); this.RequestSender.StartTestRun(runRequest, eventHandler); } } } catch (Exception exception) { EqtTrace.Error("ProxyExecutionManager.StartTestRun: Failed to start test run: {0}", exception); this.LogMessage(TestMessageLevel.Error, exception.Message, eventHandler); // Send a run complete to caller. Similar logic is also used in ParallelProxyExecutionManager.StartTestRunOnConcurrentManager // Aborted is `true`: in case of parallel run (or non shared host), an aborted message ensures another execution manager // created to replace the current one. This will help if the current execution manager is aborted due to irreparable error // and the test host is lost as well. var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, new Collection <AttachmentSet>(), TimeSpan.Zero); eventHandler.HandleTestRunComplete(completeArgs, null, null, null); } return(0); }