private async Task SendMessageAndListenAndReportTestResultsAsync(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher)
{
try
{
this.communicationManager.SendMessage(messageType, payload, this.protocolVersion);
var isTestRunComplete = false;
// Cycle through the messages that the testhost sends.
// Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification.
while (!isTestRunComplete)
{
var message = await this.TryReceiveMessageAsync();
if (string.Equals(MessageType.TestRunStatsChange, message.MessageType))
{
var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(
message);
eventHandler.HandleTestRunStatsChange(testRunChangedArgs);
}
else if (string.Equals(MessageType.ExecutionComplete, message.MessageType))
{
var testRunCompletePayload =
this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message);
eventHandler.HandleTestRunComplete(
testRunCompletePayload.TestRunCompleteArgs,
testRunCompletePayload.LastRunTests,
testRunCompletePayload.RunAttachments,
testRunCompletePayload.ExecutorUris);
isTestRunComplete = true;
}
else if (string.Equals(MessageType.TestMessage, message.MessageType))
{
var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message);
eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message);
}
else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType))
{
HandleCustomHostLaunch(customHostLauncher, message);
}
}
}
catch (Exception exception)
{
EqtTrace.Error("Aborting Test Run Operation: {0}", exception);
eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun);
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
// Earlier we were closing the connection with vstest.console in case of exceptions
// Removing that code because vstest.console might be in a healthy state and letting the client
// know of the error, so that the TL can wait for the next instruction from the client itself.
// Also, connection termination might not kill the process which could result in files being locked by testhost.
}
this.testPlatformEventSource.TranslationLayerExecutionStop();
}
private async Task SendMessageAndListenAndReportTestResultsAsync(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher)
{
try
{
this.communicationManager.SendMessage(messageType, payload, this.protocolVersion);
var isTestRunComplete = false;
// Cycle through the messages that the testhost sends.
// Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification.
while (!isTestRunComplete)
{
var message = await this.TryReceiveMessageAsync();
if (string.Equals(MessageType.TestRunStatsChange, message.MessageType))
{
var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(
message);
eventHandler.HandleTestRunStatsChange(testRunChangedArgs);
}
else if (string.Equals(MessageType.ExecutionComplete, message.MessageType))
{
var testRunCompletePayload =
this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message);
eventHandler.HandleTestRunComplete(
testRunCompletePayload.TestRunCompleteArgs,
testRunCompletePayload.LastRunTests,
testRunCompletePayload.RunAttachments,
testRunCompletePayload.ExecutorUris);
isTestRunComplete = true;
}
else if (string.Equals(MessageType.TestMessage, message.MessageType))
{
var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message);
eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message);
}
else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType))
{
HandleCustomHostLaunch(customHostLauncher, message);
}
}
}
catch (Exception exception)
{
EqtTrace.Error("Aborting Test Run Operation: {0}", exception);
eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun);
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
this.CleanupCommunicationIfProcessExit();
}
this.testPlatformEventSource.TranslationLayerExecutionStop();
}
private void OnTestRunAbort(ITestRunEventsHandler testRunEventsHandler, Exception exception, bool getClientError)
{
if (this.IsOperationComplete())
{
EqtTrace.Verbose("TestRequestSender: OnTestRunAbort: Operation is already complete. Skip error message.");
return;
}
EqtTrace.Verbose("TestRequestSender: OnTestRunAbort: Set operation complete.");
this.SetOperationComplete();
var reason = this.GetAbortErrorMessage(exception, getClientError);
EqtTrace.Error("TestRequestSender: Aborting test run because {0}", reason);
this.LogErrorMessage(string.Format(CommonResources.AbortedTestRun, reason));
// notify test run abort to vstest console wrapper.
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero);
var payload = new TestRunCompletePayload {
TestRunCompleteArgs = completeArgs
};
var rawMessage = this.dataSerializer.SerializePayload(MessageType.ExecutionComplete, payload);
testRunEventsHandler.HandleRawMessage(rawMessage);
// notify of a test run complete and bail out.
testRunEventsHandler.HandleTestRunComplete(completeArgs, null, null, null);
}
// mock a VsTest run. Provides test result one by one at 10 ms intervals
// note: a lot of information is still missing (vs real VsTest). You will have to add them if your test requires them
private void MoqTestRun(ITestRunEventsHandler testRunEvents, IReadOnlyList <TestResult> testResults,
TestCase timeOutTest = null)
{
Task.Run(() =>
{
var timer = new Stopwatch();
testRunEvents.HandleTestRunStatsChange(
new TestRunChangedEventArgs(new TestRunStatistics(0, null), null, timeOutTest == null ? null : new[] { timeOutTest }));
for (var i = 0; i < testResults.Count; i++)
{
Thread.Sleep(10);
testResults[i].EndTime = DateTimeOffset.Now;
testRunEvents.HandleTestRunStatsChange(new TestRunChangedEventArgs(
new TestRunStatistics(i + 1, null), new[] { testResults[i] }, null));
}
if (timeOutTest != null)
{
testRunEvents.HandleTestRunStatsChange(new TestRunChangedEventArgs(
new TestRunStatistics(testResults.Count, null), null, new[] { timeOutTest }));
}
Thread.Sleep(10);
testRunEvents.HandleTestRunComplete(
new TestRunCompleteEventArgs(new TestRunStatistics(testResults.Count, null), false, false, null,
null, timer.Elapsed),
null,
null,
null);
});
}
private void OnTestRunAbort(ITestRunEventsHandler testRunEventsHandler, Exception exception)
{
EqtTrace.Error("Server: TestExecution: Aborting test run because {0}", exception);
// log console message to vstest console
testRunEventsHandler.HandleLogMessage(TestMessageLevel.Error, CommonResources.AbortedTestRun);
// log console message to vstest console wrapper
var testMessagePayload = new TestMessagePayload {
MessageLevel = TestMessageLevel.Error, Message = CommonResources.AbortedTestRun
};
var rawMessage = this.dataSerializer.SerializePayload(MessageType.TestMessage, testMessagePayload);
testRunEventsHandler.HandleRawMessage(rawMessage);
// notify test run abort to vstest console wrapper
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero);
var payload = new TestRunCompletePayload {
TestRunCompleteArgs = completeArgs
};
rawMessage = this.dataSerializer.SerializePayload(MessageType.ExecutionComplete, payload);
testRunEventsHandler.HandleRawMessage(rawMessage);
// notify of a test run complete and bail out.
testRunEventsHandler.HandleTestRunComplete(completeArgs, null, null, null);
this.CleanupCommunicationIfProcessExit();
}
/// <inheritdoc/>
public int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler)
{
try
{
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings);
var testPackages = new List <string>(testRunCriteria.HasSpecificSources ? testRunCriteria.Sources :
// If the test execution is with a test filter, group them by sources
testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key));
// This code should be in sync with ProxyExecutionManager.StartTestRun executionContext
var executionContext = new TestExecutionContext(
testRunCriteria.FrequencyOfRunStatsChangeEvent,
testRunCriteria.RunStatsChangeEventTimeout,
inIsolation: runConfiguration.InIsolation,
keepAlive: testRunCriteria.KeepAlive,
isDataCollectionEnabled: false,
areTestCaseLevelEventsRequired: false,
hasTestRun: true,
isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug),
testCaseFilter: testRunCriteria.TestCaseFilter,
filterOptions: testRunCriteria.FilterOptions);
// Initialize extension before execution
this.InitializeExtensions(testPackages);
if (testRunCriteria.HasSpecificSources)
{
var runRequest = testRunCriteria.CreateTestRunCriteriaForSources(testHostManager, testRunCriteria.TestRunSettings, executionContext, testPackages);
Task.Run(() => executionManager.StartTestRun(runRequest.AdapterSourceMap, runRequest.Package,
runRequest.RunSettings, runRequest.TestExecutionContext, null, eventHandler));
}
else
{
var runRequest = testRunCriteria.CreateTestRunCriteriaForTests(testHostManager, testRunCriteria.TestRunSettings, executionContext, testPackages);
Task.Run(() => executionManager.StartTestRun(runRequest.Tests, runRequest.Package,
runRequest.RunSettings, runRequest.TestExecutionContext, null, eventHandler));
}
}
catch (Exception exception)
{
EqtTrace.Error("InProcessProxyexecutionManager.StartTestRun: Failed to start test run: {0}", exception);
// Send exception message.
eventHandler.HandleLogMessage(TestMessageLevel.Error, exception.ToString());
// Send a run complete to caller.
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, new Collection <AttachmentSet>(), TimeSpan.Zero);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
}
return(0);
}
private void OnExecutionMessageReceived(object sender, MessageReceivedEventArgs messageReceived, ITestRunEventsHandler testRunEventsHandler)
{
try
{
var rawMessage = messageReceived.Data;
// Send raw message first to unblock handlers waiting to send message to IDEs
testRunEventsHandler.HandleRawMessage(rawMessage);
var message = this.dataSerializer.DeserializeMessage(rawMessage);
switch (message.MessageType)
{
case MessageType.TestRunStatsChange:
var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(message);
testRunEventsHandler.HandleTestRunStatsChange(testRunChangedArgs);
break;
case MessageType.ExecutionComplete:
var testRunCompletePayload = this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message);
testRunEventsHandler.HandleTestRunComplete(
testRunCompletePayload.TestRunCompleteArgs,
testRunCompletePayload.LastRunTests,
testRunCompletePayload.RunAttachments,
testRunCompletePayload.ExecutorUris);
this.SetOperationComplete();
break;
case MessageType.TestMessage:
var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message);
testRunEventsHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message);
break;
case MessageType.LaunchAdapterProcessWithDebuggerAttached:
var testProcessStartInfo = this.dataSerializer.DeserializePayload <TestProcessStartInfo>(message);
int processId = testRunEventsHandler.LaunchProcessWithDebuggerAttached(testProcessStartInfo);
var data =
this.dataSerializer.SerializePayload(
MessageType.LaunchAdapterProcessWithDebuggerAttachedCallback,
processId,
this.protocolVersion);
this.channel.Send(data);
break;
}
}
catch (Exception exception)
{
this.OnTestRunAbort(testRunEventsHandler, exception, false);
}
}
/// <summary>
/// Aborts the test execution.
/// </summary>
public void Abort(ITestRunEventsHandler testRunEventsHandler)
{
if (this.activeTestRun == null)
{
var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, true, null, null, TimeSpan.Zero);
testRunEventsHandler.HandleTestRunComplete(testRunCompleteEventArgs, null, null, null);
}
else
{
this.activeTestRun.Abort();
}
}
private void ListenAndReportTestResults(ITestRunEventsHandler testRunEventsHandler)
{
var isTestRunComplete = false;
// Cycle through the messages that the testhost sends.
// Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification.
while (!isTestRunComplete)
{
try
{
var rawMessage = this.TryReceiveRawMessage();
// Send raw message first to unblock handlers waiting to send message to IDEs
testRunEventsHandler.HandleRawMessage(rawMessage);
var message = this.dataSerializer.DeserializeMessage(rawMessage);
if (string.Equals(MessageType.TestRunStatsChange, message.MessageType))
{
var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(
message);
testRunEventsHandler.HandleTestRunStatsChange(testRunChangedArgs);
}
else if (string.Equals(MessageType.ExecutionComplete, message.MessageType))
{
var testRunCompletePayload =
this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message);
testRunEventsHandler.HandleTestRunComplete(
testRunCompletePayload.TestRunCompleteArgs,
testRunCompletePayload.LastRunTests,
testRunCompletePayload.RunAttachments,
testRunCompletePayload.ExecutorUris);
isTestRunComplete = true;
}
else if (string.Equals(MessageType.TestMessage, message.MessageType))
{
var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message);
testRunEventsHandler.HandleLogMessage(
testMessagePayload.MessageLevel,
testMessagePayload.Message);
}
else if (string.Equals(MessageType.LaunchAdapterProcessWithDebuggerAttached, message.MessageType))
{
var testProcessStartInfo = this.dataSerializer.DeserializePayload <TestProcessStartInfo>(message);
int processId = testRunEventsHandler.LaunchProcessWithDebuggerAttached(testProcessStartInfo);
this.communicationManager.SendMessage(
MessageType.LaunchAdapterProcessWithDebuggerAttachedCallback,
processId,
version: this.protocolVersion);
}
}
catch (IOException exception)
{
// To avoid further communication with remote host
this.sendMessagesToRemoteHost = false;
this.OnTestRunAbort(testRunEventsHandler, exception);
isTestRunComplete = true;
}
catch (Exception exception)
{
this.OnTestRunAbort(testRunEventsHandler, exception);
isTestRunComplete = true;
}
}
}
/// <summary>
/// Starts the test run
/// </summary>
/// <param name="testRunCriteria"> The settings/options for the test run. </param>
/// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param>
/// <returns> The process id of the runner executing tests. </returns>
public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler)
{
try
{
var executionEngineStartTime = DateTime.UtcNow;
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("ProxyExecutionManager: Test host is always Lazy initialize.");
}
var testPackages = new List <string>(testRunCriteria.HasSpecificSources ? testRunCriteria.Sources :
// If the test execution is with a test filter, group them by sources
testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key));
this.isCommunicationEstablished = this.SetupChannel(testPackages, this.cancellationTokenSource.Token);
if (this.isCommunicationEstablished)
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("ProxyExecutionManager.StartTestRun: Canceling the current run after getting cancelation request.");
}
throw new TestPlatformException(Resources.Resources.CancelationRequested);
}
this.InitializeExtensions(testPackages);
// Collecting Time Taken to Start Discovery Engine
var executionEngineTotalTime = DateTime.UtcNow - executionEngineStartTime;
// Collecting Data Point for Time taken to start Execution Engine. In case of Parallel, it will be maximum time taken.
this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenToStartExecutionEngineExe, executionEngineTotalTime.TotalSeconds.ToString());
// This code should be in sync with InProcessProxyExecutionManager.StartTestRun executionContext
var executionContext = new TestExecutionContext(
testRunCriteria.FrequencyOfRunStatsChangeEvent,
testRunCriteria.RunStatsChangeEventTimeout,
inIsolation: false,
keepAlive: testRunCriteria.KeepAlive,
isDataCollectionEnabled: false,
areTestCaseLevelEventsRequired: false,
hasTestRun: true,
isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug),
testCaseFilter: testRunCriteria.TestCaseFilter);
// This is workaround for the bug https://github.com/Microsoft/vstest/issues/970
var runsettings = this.RemoveNodesFromRunsettingsIfRequired(testRunCriteria.TestRunSettings, (testMessageLevel, message) => { this.LogMessage(testMessageLevel, message, eventHandler); });
if (testRunCriteria.HasSpecificSources)
{
var runRequest = testRunCriteria.CreateTestRunCriteriaForSources(testHostManager, runsettings, executionContext, testPackages);
this.RequestSender.StartTestRun(runRequest, eventHandler);
}
else
{
var runRequest = testRunCriteria.CreateTestRunCriteriaForTests(testHostManager, runsettings, executionContext, testPackages);
this.RequestSender.StartTestRun(runRequest, eventHandler);
}
}
}
catch (Exception exception)
{
EqtTrace.Error("ProxyExecutionManager.StartTestRun: Failed to start test run: {0}", exception);
this.LogMessage(TestMessageLevel.Error, exception.Message, eventHandler);
// Send a run complete to caller. Similar logic is also used in ParallelProxyExecutionManager.StartTestRunOnConcurrentManager
// Aborted is `true`: in case of parallel run (or non shared host), an aborted message ensures another execution manager
// created to replace the current one. This will help if the current execution manager is aborted due to irreparable error
// and the test host is lost as well.
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, new Collection <AttachmentSet>(), TimeSpan.Zero, null);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
}
return(0);
}
/// <summary>
/// Starts the test run
/// </summary>
/// <param name="testRunCriteria"> The settings/options for the test run. </param>
/// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param>
/// <returns> The process id of the runner executing tests. </returns>
public virtual int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler)
{
try
{
if (!this.testHostManager.Shared)
{
// Non shared test host requires test source information to launch. Provide the sources
// information and create the channel.
EqtTrace.Verbose("ProxyExecutionManager: Test host is non shared. Lazy initialize.");
var testSources = testRunCriteria.Sources;
// If the test execution is with a test filter, group them by sources
if (testRunCriteria.HasSpecificTests)
{
testSources = testRunCriteria.Tests.GroupBy(tc => tc.Source).Select(g => g.Key);
}
this.InitializeExtensions(testSources);
}
this.SetupChannel(testRunCriteria.Sources);
var executionContext = new TestExecutionContext(
testRunCriteria.FrequencyOfRunStatsChangeEvent,
testRunCriteria.RunStatsChangeEventTimeout,
inIsolation: false,
keepAlive: testRunCriteria.KeepAlive,
isDataCollectionEnabled: false,
areTestCaseLevelEventsRequired: false,
hasTestRun: true,
isDebug: (testRunCriteria.TestHostLauncher != null && testRunCriteria.TestHostLauncher.IsDebug),
testCaseFilter: testRunCriteria.TestCaseFilter);
if (testRunCriteria.HasSpecificSources)
{
var runRequest = new TestRunCriteriaWithSources(
testRunCriteria.AdapterSourceMap,
testRunCriteria.TestRunSettings,
executionContext);
this.RequestSender.StartTestRun(runRequest, eventHandler);
}
else
{
var runRequest = new TestRunCriteriaWithTests(
testRunCriteria.Tests,
testRunCriteria.TestRunSettings,
executionContext);
this.RequestSender.StartTestRun(runRequest, eventHandler);
}
}
catch (Exception exception)
{
EqtTrace.Error("ProxyExecutionManager.StartTestRun: Failed to start test run: {0}", exception);
var completeArgs = new TestRunCompleteEventArgs(null, false, false, exception, new Collection <AttachmentSet>(), TimeSpan.Zero);
eventHandler.HandleLogMessage(TestMessageLevel.Error, exception.Message);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
}
return(0);
}