private void StartTestRunErrorTestsTemplate(string errorMessage, Action <string> onClientProcessExitCallback)
{
var mockHandler = new Mock <ITestRunEventsHandler>();
var runCriteria = new TestRunCriteriaWithSources(null, null, null);
this.mockCommunicationManager.Setup(mc => mc.ReceiveRawMessageAsync(It.IsAny <CancellationToken>()))
.Callback(() => onClientProcessExitCallback(errorMessage)).Returns(Task.FromResult((string)null));
string testCompleteRawMessage =
"{\"MessageType\":\"TestExecution.Completed\",\"Payload\":{\"TestRunCompleteArgs\":{\"TestRunStatistics\":null,\"IsCanceled\":false,\"IsAborted\":true,\"Error\":{\"ClassName\":\"System.IO.IOException\",\"Message\":\"Unable to read data from the transport connection: An existing connection was forcibly closed by the remote host.\",\"Data\":null,\"InnerException\":null},\"AttachmentSets\":null,\"ElapsedTimeInRunningTests\":\"00:00:00\"},\"LastRunTests\":null,\"RunAttachments\":null,\"ExecutorUris\":null}}";
this.mockDataSerializer.Setup(
md => md.SerializePayload(MessageType.ExecutionComplete, It.IsAny <TestRunCompletePayload>()))
.Returns(testCompleteRawMessage);
var waitHandle = new AutoResetEvent(false);
mockHandler.Setup(mh => mh.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(), null, null, null)).Callback(() => waitHandle.Set());
this.testRequestSender.InitializeCommunication();
this.testRequestSender.StartTestRun(runCriteria, mockHandler.Object);
waitHandle.WaitOne();
this.testRequestSender.EndSession();
mockHandler.Verify(mh => mh.HandleLogMessage(TestMessageLevel.Error, string.Format(CommunicationUtilitiesResources.AbortedTestRun, errorMessage)), Times.Once);
mockHandler.Verify(mh => mh.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(), null, null, null), Times.Once);
mockHandler.Verify(mh => mh.HandleRawMessage(testCompleteRawMessage), Times.Once);
this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.SessionEnd), Times.Never);
}
public void StartTestRunShouldCallLaunchProcessWithDebuggerAndWaitForCallback()
{
var mockHandler = new Mock <ITestRunEventsHandler>();
var runCriteria = new TestRunCriteriaWithSources(null, null, null);
var rawMessage = "LaunchProcessWithDebugger";
var message = new Message()
{
MessageType = MessageType.LaunchAdapterProcessWithDebuggerAttached, Payload = null
};
var payload = new TestProcessStartInfo();
this.SetupReceiveRawMessageAsyncAndDeserializeMessageAndInitialize(rawMessage, message);
this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestProcessStartInfo>(message)).Returns(payload);
var completePayload = new TestRunCompletePayload()
{
ExecutorUris = null,
LastRunTests = null,
RunAttachments = null,
TestRunCompleteArgs = null
};
var completeMessage = new Message()
{
MessageType = MessageType.ExecutionComplete, Payload = null
};
mockHandler.Setup(mh => mh.LaunchProcessWithDebuggerAttached(payload)).Callback(
() =>
{
this.mockDataSerializer.Setup(ds => ds.DeserializeMessage(It.IsAny <string>())).Returns(completeMessage);
this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestRunCompletePayload>(completeMessage)).Returns(completePayload);
});
var waitHandle = new AutoResetEvent(false);
mockHandler.Setup(mh => mh.HandleTestRunComplete(
It.IsAny <TestRunCompleteEventArgs>(),
It.IsAny <TestRunChangedEventArgs>(),
It.IsAny <ICollection <AttachmentSet> >(),
It.IsAny <ICollection <string> >())).Callback(() => waitHandle.Set());
this.testRequestSender.StartTestRun(runCriteria, mockHandler.Object);
waitHandle.WaitOne();
this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.StartTestExecutionWithSources, runCriteria, this.protocolConfig.Version), Times.Once);
this.mockDataSerializer.Verify(ds => ds.DeserializeMessage(It.IsAny <string>()), Times.Exactly(2));
mockHandler.Verify(mh => mh.LaunchProcessWithDebuggerAttached(payload), Times.Once);
mockHandler.Verify(mh => mh.HandleRawMessage(rawMessage), Times.Exactly(2));
this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.LaunchAdapterProcessWithDebuggerAttachedCallback, It.IsAny <object>(), this.protocolConfig.Version), Times.Once);
}
public void StartTestRunWithSourcesShouldCallHandleTestRunStatsChange()
{
var mockHandler = new Mock <ITestRunEventsHandler>();
var runCriteria = new TestRunCriteriaWithSources(null, null, null);
var testRunChangedArgs = new TestRunChangedEventArgs(null, null, null);
var rawMessage = "OnTestRunStatsChange";
var message = new Message()
{
MessageType = MessageType.TestRunStatsChange, Payload = null
};
this.SetupReceiveRawMessageAsyncAndDeserializeMessageAndInitialize(rawMessage, message);
this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestRunChangedEventArgs>(message)).Returns(testRunChangedArgs);
var completePayload = new TestRunCompletePayload()
{
ExecutorUris = null,
LastRunTests = null,
RunAttachments = null,
TestRunCompleteArgs = null
};
var completeMessage = new Message()
{
MessageType = MessageType.ExecutionComplete, Payload = null
};
mockHandler.Setup(mh => mh.HandleTestRunStatsChange(testRunChangedArgs)).Callback(
() =>
{
this.mockDataSerializer.Setup(ds => ds.DeserializeMessage(It.IsAny <string>())).Returns(completeMessage);
this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestRunCompletePayload>(completeMessage)).Returns(completePayload);
});
var waitHandle = new AutoResetEvent(false);
mockHandler.Setup(mh => mh.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(),
It.IsAny <TestRunChangedEventArgs>(), It.IsAny <ICollection <AttachmentSet> >(), It.IsAny <ICollection <string> >())).Callback
(() => waitHandle.Set());
this.testRequestSender.StartTestRun(runCriteria, mockHandler.Object);
waitHandle.WaitOne();
this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.StartTestExecutionWithSources, runCriteria), Times.Once);
// One for run stats and another for runcomplete
this.mockDataSerializer.Verify(ds => ds.DeserializeMessage(rawMessage), Times.Exactly(2));
mockHandler.Verify(mh => mh.HandleTestRunStatsChange(testRunChangedArgs), Times.Once);
mockHandler.Verify(mh => mh.HandleRawMessage(rawMessage), Times.Exactly(2));
}
public TestRequestSenderTests2()
{
this.mockChannel = new Mock <ICommunicationChannel>();
this.mockServer = new Mock <ICommunicationServer>();
this.mockDataSerializer = new Mock <IDataSerializer>();
this.testRequestSender = new TestableTestRequestSender(this.mockServer.Object, this.mockDataSerializer.Object, new ProtocolConfig {
Version = DUMMYPROTOCOLVERSION
});
this.connectedEventArgs = new ConnectedEventArgs(this.mockChannel.Object);
this.mockDiscoveryEventsHandler = new Mock <ITestDiscoveryEventsHandler2>();
this.mockExecutionEventsHandler = new Mock <ITestRunEventsHandler>();
this.testRunCriteriaWithSources = new TestRunCriteriaWithSources(new Dictionary <string, IEnumerable <string> >(), null, "runsettings", null);
}
/// <inheritdoc />
public void StartTestRun(TestRunCriteriaWithSources runCriteria, ITestRunEventsHandler eventHandler)
{
this.messageEventHandler = eventHandler;
this.onDisconnected = (disconnectedEventArgs) =>
{
this.OnTestRunAbort(eventHandler, disconnectedEventArgs.Error, true);
};
this.onMessageReceived = (sender, args) => this.OnExecutionMessageReceived(sender, args, eventHandler);
this.channel.MessageReceived += this.onMessageReceived;
// This code section is needed because we altered the old testhost launch process for
// the debugging workflow. Now we don't ask VS to launch and attach to the testhost
// process for us as we previously did, instead we launch it as a standalone process
// and rely on the testhost to ask VS to attach the debugger to itself.
//
// In order to avoid breaking compatibility with previous testhost versions because of
// those changes (older testhosts won't know to request VS to attach to themselves
// thinking instead VS launched and attached to them already), we request VS to attach
// to the testhost here before starting the test run.
if (runCriteria.TestExecutionContext != null &&
runCriteria.TestExecutionContext.IsDebug &&
this.runtimeProvider is ITestRuntimeProvider2 convertedRuntimeProvider &&
this.protocolVersion < ObjectModelConstants.MinimumProtocolVersionWithDebugSupport)
{
var handler = (ITestRunEventsHandler2)eventHandler;
if (!convertedRuntimeProvider.AttachDebuggerToTestHost())
{
EqtTrace.Warning(
string.Format(
CultureInfo.CurrentUICulture,
CommonResources.AttachDebuggerToDefaultTestHostFailure));
}
}
var message = this.dataSerializer.SerializePayload(
MessageType.StartTestExecutionWithSources,
runCriteria,
this.protocolVersion);
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("TestRequestSender.StartTestRun: Sending test run with message: {0}", message);
}
this.channel.Send(message);
}
public void StartTestRunShouldCallHandleLogMessageOnTestMessage()
{
var mockHandler = new Mock <ITestRunEventsHandler>();
var runCriteria = new TestRunCriteriaWithSources(null, null, null, null);
var rawMessage = "OnLogMessage";
var message = new Message()
{
MessageType = MessageType.TestMessage, Payload = null
};
var payload = new TestMessagePayload()
{
MessageLevel = TestMessageLevel.Error, Message = rawMessage
};
this.SetupReceiveRawMessageAsyncAndDeserializeMessageAndInitialize(rawMessage, message);
this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestMessagePayload>(message)).Returns(payload);
var completePayload = new TestRunCompletePayload()
{
ExecutorUris = null, LastRunTests = null, RunAttachments = null, TestRunCompleteArgs = null
};
var completeMessage = new Message()
{
MessageType = MessageType.ExecutionComplete, Payload = null
};
var waitHandle = new AutoResetEvent(false);
mockHandler.Setup(mh => mh.HandleLogMessage(TestMessageLevel.Error, rawMessage)).Callback(
() =>
{
this.mockDataSerializer.Setup(ds => ds.DeserializeMessage(It.IsAny <string>())).Returns(completeMessage);
this.mockDataSerializer.Setup(ds => ds.DeserializePayload <TestRunCompletePayload>(completeMessage)).Callback(() => { waitHandle.Set(); })
.Returns(completePayload);
});
this.testRequestSender.StartTestRun(runCriteria, mockHandler.Object);
waitHandle.WaitOne();
this.mockCommunicationManager.Verify(mc => mc.SendMessage(MessageType.StartTestExecutionWithSources, runCriteria, this.protocolConfig.Version), Times.Once);
this.mockDataSerializer.Verify(ds => ds.DeserializeMessage(It.IsAny <string>()), Times.AtLeast(2));
// Asserting that 'StartTestRun" should have been completed, & invoked only once
this.mockDataSerializer.Verify(ds => ds.DeserializePayload <TestRunCompletePayload>(completeMessage), Times.Exactly(1));
mockHandler.Verify(mh => mh.HandleLogMessage(payload.MessageLevel, payload.Message), Times.Once);
mockHandler.Verify(mh => mh.HandleRawMessage(rawMessage), Times.AtLeastOnce);
}
/// <inheritdoc />
public void StartTestRun(TestRunCriteriaWithSources runCriteria, ITestRunEventsHandler eventHandler)
{
this.messageEventHandler = eventHandler;
this.onDisconnected = (disconnectedEventArgs) =>
{
this.OnTestRunAbort(eventHandler, disconnectedEventArgs.Error, true);
};
this.onMessageReceived = (sender, args) => this.OnExecutionMessageReceived(sender, args, eventHandler);
this.channel.MessageReceived += this.onMessageReceived;
var message = this.dataSerializer.SerializePayload(
MessageType.StartTestExecutionWithSources,
runCriteria,
this.protocolVersion);
this.channel.Send(message);
}
public TestRequestSenderTests()
{
this.connectionInfo = new TestHostConnectionInfo
{
Endpoint = IPAddress.Loopback + ":123",
Role = ConnectionRole.Client,
Transport = Transport.Sockets
};
this.mockChannel = new Mock <ICommunicationChannel>();
this.mockServer = new Mock <ICommunicationEndPoint>();
this.mockDataSerializer = new Mock <IDataSerializer>();
this.testRequestSender = new TestableTestRequestSender(this.mockServer.Object, this.connectionInfo, this.mockDataSerializer.Object, new ProtocolConfig {
Version = DUMMYPROTOCOLVERSION
});
this.connectedEventArgs = new ConnectedEventArgs(this.mockChannel.Object);
this.mockDiscoveryEventsHandler = new Mock <ITestDiscoveryEventsHandler2>();
this.mockExecutionEventsHandler = new Mock <ITestRunEventsHandler>();
this.testRunCriteriaWithSources = new TestRunCriteriaWithSources(new Dictionary <string, IEnumerable <string> >(), "runsettings", null, null);
}
public void ProcessRequestsExecutionStartShouldStartExecutionWithGivenSources()
{
var asm = new Dictionary <string, IEnumerable <string> >();
asm["mstestv2"] = new[] { "test1.dll", "test2.dll" };
var testRunCriteriaWithSources = new TestRunCriteriaWithSources(asm, "runsettings", null, null);
var message = this.dataSerializer.SerializePayload(MessageType.StartTestExecutionWithSources, testRunCriteriaWithSources);
this.ProcessRequestsAsync(this.mockTestHostManagerFactory.Object);
this.SendMessageOnChannel(message);
this.jobQueue.Flush();
mockExecutionManager.Verify(e =>
e.StartTestRun(
It.Is <Dictionary <string, IEnumerable <string> > >(d => d.ContainsKey("mstestv2")), It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <TestExecutionContext>(), It.IsAny <ITestCaseEventsHandler>(),
It.IsAny <ITestRunEventsHandler>()));
this.SendSessionEnd();
}
public void StartTestRunShouldInitiateTestRunForSourcesThroughTheServer()
{
TestRunCriteriaWithSources testRunCriteriaPassed = null;
this.mockRequestSender.Setup(s => s.WaitForRequestHandlerConnection(It.IsAny <int>())).Returns(true);
this.mockRequestSender.Setup(s => s.StartTestRun(It.IsAny <TestRunCriteriaWithSources>(), null))
.Callback(
(TestRunCriteriaWithSources criteria, ITestRunEventsHandler sink) =>
{
testRunCriteriaPassed = criteria;
});
this.testExecutionManager.StartTestRun(this.mockTestRunCriteria.Object, null);
Assert.IsNotNull(testRunCriteriaPassed);
CollectionAssert.AreEqual(this.mockTestRunCriteria.Object.AdapterSourceMap.Keys, testRunCriteriaPassed.AdapterSourceMap.Keys);
CollectionAssert.AreEqual(this.mockTestRunCriteria.Object.AdapterSourceMap.Values, testRunCriteriaPassed.AdapterSourceMap.Values);
Assert.AreEqual(this.mockTestRunCriteria.Object.FrequencyOfRunStatsChangeEvent, testRunCriteriaPassed.TestExecutionContext.FrequencyOfRunStatsChangeEvent);
Assert.AreEqual(this.mockTestRunCriteria.Object.RunStatsChangeEventTimeout, testRunCriteriaPassed.TestExecutionContext.RunStatsChangeEventTimeout);
Assert.AreEqual(this.mockTestRunCriteria.Object.TestRunSettings, testRunCriteriaPassed.RunSettings);
}
/// <inheritdoc />
public void StartTestRun(TestRunCriteriaWithSources runCriteria, ITestRunEventsHandler eventHandler)
{
this.messageEventHandler = eventHandler;
this.onDisconnected = (disconnectedEventArgs) =>
{
this.OnTestRunAbort(eventHandler, disconnectedEventArgs.Error, true);
};
this.onMessageReceived = (sender, args) => this.OnExecutionMessageReceived(sender, args, eventHandler);
this.channel.MessageReceived += this.onMessageReceived;
var message = this.dataSerializer.SerializePayload(
MessageType.StartTestExecutionWithSources,
runCriteria,
this.protocolVersion);
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("TestRequestSender.StartTestRun: Sending test run with message: {0}", message);
}
this.channel.Send(message);
}
/// <summary>
/// Executes tests on the sources specified with the criteria mentioned.
/// </summary>
/// <param name="runCriteria">The test run criteria.</param>
/// <param name="eventHandler">The handler for execution events from the test host.</param>
public void StartTestRun(TestRunCriteriaWithSources runCriteria, ITestRunEventsHandler eventHandler)
{
this.StartTestRunAndListenAndReportTestResults(MessageType.StartTestExecutionWithSources, runCriteria, eventHandler);
}
/// <summary>
/// Starts the test run
/// </summary>
/// <param name="testRunCriteria"> The settings/options for the test run. </param>
/// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param>
/// <returns> The process id of the runner executing tests. </returns>
public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler)
{
try
{
if (!this.testHostManager.Shared)
{
// Non shared test host requires test source information to launch. Provide the sources
// information and create the channel.
EqtTrace.Verbose("ProxyExecutionManager: Test host is non shared. Lazy initialize.");
var testSources = testRunCriteria.Sources;
// If the test execution is with a test filter, group them by sources
if (testRunCriteria.HasSpecificTests)
{
testSources = testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key);
}
this.InitializeExtensions(testSources);
}
this.SetupChannel(testRunCriteria.Sources);
var executionContext = new TestExecutionContext(
testRunCriteria.FrequencyOfRunStatsChangeEvent,
testRunCriteria.RunStatsChangeEventTimeout,
inIsolation: false,
keepAlive: testRunCriteria.KeepAlive,
isDataCollectionEnabled: false,
areTestCaseLevelEventsRequired: false,
hasTestRun: true,
isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug),
testCaseFilter: testRunCriteria.TestCaseFilter);
if (testRunCriteria.HasSpecificSources)
{
var runRequest = new TestRunCriteriaWithSources(
testRunCriteria.AdapterSourceMap,
testRunCriteria.TestRunSettings,
executionContext);
this.RequestSender.StartTestRun(runRequest, eventHandler);
}
else
{
var runRequest = new TestRunCriteriaWithTests(
testRunCriteria.Tests,
testRunCriteria.TestRunSettings,
executionContext);
this.RequestSender.StartTestRun(runRequest, eventHandler);
}
}
catch (Exception exception)
{
EqtTrace.Error("ProxyExecutionManager.StartTestRun: Failed to start test run: {0}", exception);
var completeArgs = new TestRunCompleteEventArgs(null, false, false, exception, new Collection <AttachmentSet>(), TimeSpan.Zero);
eventHandler.HandleLogMessage(TestMessageLevel.Error, exception.Message);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
}
return(0);
}
/// <summary>
/// Starts the test run
/// </summary>
/// <param name="testRunCriteria"> The settings/options for the test run. </param>
/// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param>
/// <returns> The process id of the runner executing tests. </returns>
public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler)
{
try
{
EqtTrace.Verbose("ProxyExecutionManager: Test host is always Lazy initialize.");
var testSources = testRunCriteria.Sources;
// If the test execution is with a test filter, group them by sources
if (testRunCriteria.HasSpecificTests)
{
testSources = testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key);
}
this.isCommunicationEstablished = this.SetupChannel(testSources, this.cancellationTokenSource.Token);
if (this.isCommunicationEstablished)
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("ProxyExecutionManager.StartTestRun: Canceling the current run after getting cancelation request.");
}
throw new TestPlatformException(Resources.Resources.CancelationRequested);
}
this.InitializeExtensions(testSources);
var executionContext = new TestExecutionContext(
testRunCriteria.FrequencyOfRunStatsChangeEvent,
testRunCriteria.RunStatsChangeEventTimeout,
inIsolation: false,
keepAlive: testRunCriteria.KeepAlive,
isDataCollectionEnabled: false,
areTestCaseLevelEventsRequired: false,
hasTestRun: true,
isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug),
testCaseFilter: testRunCriteria.TestCaseFilter);
// This is workaround for the bug https://github.com/Microsoft/vstest/issues/970
var runsettings = this.RemoveNodesFromRunsettingsIfRequired(testRunCriteria.TestRunSettings, (testMessageLevel, message) => { this.LogMessage(testMessageLevel, message, eventHandler); });
if (testRunCriteria.HasSpecificSources)
{
var runRequest = new TestRunCriteriaWithSources(
testRunCriteria.AdapterSourceMap,
runsettings,
executionContext);
this.RequestSender.StartTestRun(runRequest, eventHandler);
}
else
{
var runRequest = new TestRunCriteriaWithTests(
testRunCriteria.Tests,
runsettings,
executionContext);
this.RequestSender.StartTestRun(runRequest, eventHandler);
}
}
}
catch (Exception exception)
{
EqtTrace.Error("ProxyExecutionManager.StartTestRun: Failed to start test run: {0}", exception);
this.LogMessage(TestMessageLevel.Error, exception.Message, eventHandler);
// Send a run complete to caller. Similar logic is also used in ParallelProxyExecutionManager.StartTestRunOnConcurrentManager
// Aborted is `true`: in case of parallel run (or non shared host), an aborted message ensures another execution manager
// created to replace the current one. This will help if the current execution manager is aborted due to irreparable error
// and the test host is lost as well.
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, new Collection <AttachmentSet>(), TimeSpan.Zero);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
}
return(0);
}