public void SetCustomLauncher(ITestHostLauncher customLauncher)
{
throw new NotImplementedException();
}
/// <inheritdoc/>
public void RunTestsWithCustomTestHost(IEnumerable <TestCase> testCases, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
{
this.EnsureInitialized();
this.requestSender.StartTestRunWithCustomHost(testCases, runSettings, testRunEventsHandler, customTestHostLauncher);
}
private void SendMessageAndListenAndReportTestResults(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher)
{
try
{
this.communicationManager.SendMessage(messageType, payload, this.protocolVersion);
var isTestRunComplete = false;
// Cycle through the messages that the testhost sends.
// Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification.
while (!isTestRunComplete)
{
var message = this.TryReceiveMessage();
if (string.Equals(MessageType.TestRunStatsChange, message.MessageType))
{
var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(
message);
eventHandler.HandleTestRunStatsChange(testRunChangedArgs);
}
else if (string.Equals(MessageType.ExecutionComplete, message.MessageType))
{
var testRunCompletePayload =
this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message);
eventHandler.HandleTestRunComplete(
testRunCompletePayload.TestRunCompleteArgs,
testRunCompletePayload.LastRunTests,
testRunCompletePayload.RunAttachments,
testRunCompletePayload.ExecutorUris);
isTestRunComplete = true;
}
else if (string.Equals(MessageType.TestMessage, message.MessageType))
{
var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message);
eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message);
}
else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType))
{
HandleCustomHostLaunch(customHostLauncher, message);
}
}
}
catch (Exception exception)
{
EqtTrace.Error("Aborting Test Run Operation: {0}", exception);
eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun);
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
this.CleanupCommunicationIfProcessExit();
}
this.testPlatformEventSource.TranslationLayerExecutionStop();
}
/// <inheritdoc/>
public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, TestPlatformOptions options, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
{
var sourceList = sources.ToList();
this.testPlatformEventSource.TranslationLayerExecutionStart(1, sourceList.Count, 0, runSettings ?? string.Empty);
this.EnsureInitialized();
this.requestSender.StartTestRunWithCustomHost(sourceList, runSettings, options, testRunEventsHandler, customTestHostLauncher);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (testRunRequestPayload.TestPlatformOptions != null)
{
this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
// Get sources to auto detect fx and arch for both run selected or run all scenario.
var sources = GetSources(testRunRequestPayload);
if (this.UpdateRunSettingsIfRequired(runsettings, sources, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsInCompatibleWithTestSettings(runsettings))
{
throw new SettingsException(string.Format(Resources.RunsettingsWithDCErrorMessage, runsettings));
}
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
var batchSize = runConfiguration.BatchSize;
if (requestData.IsTelemetryOptedIn)
{
// Collect Metrics
this.CollectMetrics(requestData, runConfiguration);
// Collect Commands
this.LogCommandsTelemetryPoints(requestData);
}
if (!commandLineOptions.IsDesignMode)
{
// Generate fakes settings only for command line scenarios. In case of
// Editors/IDEs, this responsibility is with the caller.
GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings);
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = testRunRequestPayload.TestPlatformOptions?.TestCaseFilter;
runCriteria.FilterOptions = testRunRequestPayload.TestPlatformOptions?.FilterOptions;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(requestData, runCriteria, testRunEventsRegistrar);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
this.testPlatformEventSource.ExecutionRequestStop();
// Post the run complete event
this.metricsPublisher.Result.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics);
return(success);
}
private async Task SendMessageAndListenAndReportTestResultsAsync(string messageType, object payload, ITestRunEventsHandler eventHandler, ITestHostLauncher customHostLauncher)
{
try
{
this.communicationManager.SendMessage(messageType, payload, this.protocolVersion);
var isTestRunComplete = false;
// Cycle through the messages that the testhost sends.
// Currently each of the operations are not separate tasks since they should not each take much time. This is just a notification.
while (!isTestRunComplete)
{
var message = await this.TryReceiveMessageAsync();
if (string.Equals(MessageType.TestRunStatsChange, message.MessageType))
{
var testRunChangedArgs = this.dataSerializer.DeserializePayload <TestRunChangedEventArgs>(
message);
eventHandler.HandleTestRunStatsChange(testRunChangedArgs);
}
else if (string.Equals(MessageType.ExecutionComplete, message.MessageType))
{
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("VsTestConsoleRequestSender.SendMessageAndListenAndReportTestResultsAsync: Execution complete.");
}
var testRunCompletePayload =
this.dataSerializer.DeserializePayload <TestRunCompletePayload>(message);
eventHandler.HandleTestRunComplete(
testRunCompletePayload.TestRunCompleteArgs,
testRunCompletePayload.LastRunTests,
testRunCompletePayload.RunAttachments,
testRunCompletePayload.ExecutorUris);
isTestRunComplete = true;
}
else if (string.Equals(MessageType.TestMessage, message.MessageType))
{
var testMessagePayload = this.dataSerializer.DeserializePayload <TestMessagePayload>(message);
eventHandler.HandleLogMessage(testMessagePayload.MessageLevel, testMessagePayload.Message);
}
else if (string.Equals(MessageType.CustomTestHostLaunch, message.MessageType))
{
HandleCustomHostLaunch(customHostLauncher, message);
}
else if (string.Equals(MessageType.EditorAttachDebugger, message.MessageType))
{
AttachDebuggerToProcess(customHostLauncher, message);
}
}
}
catch (Exception exception)
{
EqtTrace.Error("Aborting Test Run Operation: {0}", exception);
eventHandler.HandleLogMessage(TestMessageLevel.Error, TranslationLayerResources.AbortedTestsRun);
var completeArgs = new TestRunCompleteEventArgs(null, false, true, exception, null, TimeSpan.Zero);
eventHandler.HandleTestRunComplete(completeArgs, null, null, null);
// Earlier we were closing the connection with vstest.console in case of exceptions
// Removing that code because vstest.console might be in a healthy state and letting the client
// know of the error, so that the TL can wait for the next instruction from the client itself.
// Also, connection termination might not kill the process which could result in files being locked by testhost.
}
this.testPlatformEventSource.TranslationLayerExecutionStop();
}
/// <inheritdoc/>
public void StartTestSession(
StartTestSessionPayload payload,
ITestHostLauncher testHostLauncher,
ITestSessionEventsHandler eventsHandler,
ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.StartTestSession: Starting test session.");
if (payload.TestPlatformOptions != null)
{
this.telemetryOptedIn = payload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
if (this.UpdateRunSettingsIfRequired(
payload.RunSettings,
payload.Sources,
null,
out string updatedRunsettings))
{
payload.RunSettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(payload.RunSettings))
{
throw new SettingsException(
string.Format(
Resources.RunsettingsWithDCErrorMessage,
payload.RunSettings));
}
// TODO (copoiena): Collect metrics ?
lock (this.syncObject)
{
try
{
EqtTrace.Info("TestRequestManager.StartTestRunner: Synchronization context taken.");
this.TestPlatformEventSourceInstance.StartTestSessionStart();
var criteria = new StartTestSessionCriteria()
{
Sources = payload.Sources,
RunSettings = payload.RunSettings,
TestHostLauncher = testHostLauncher
};
this.testPlatform.StartTestSession(requestData, criteria, eventsHandler);
}
finally
{
EqtTrace.Info("TestRequestManager.StartTestSession: Starting test session completed.");
this.TestPlatformEventSourceInstance.StartTestSessionStop();
// Post the attachments processing complete event.
this.metricsPublisher.Result.PublishMetrics(
TelemetryDataConstants.StartTestSessionCompleteEvent,
requestData.MetricsCollection.Metrics);
}
}
}
public void SetCustomLauncher(ITestHostLauncher customLauncher)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="TestRunCriteria"/> class.
/// </summary>
/// <param name="tests">
/// Sources which contains tests that should be executed
/// </param>
/// <param name="frequencyOfRunStatsChangeEvent">
/// Frequency of run stats event
/// </param>
/// <param name="keepAlive">
/// Whether or not to keep the test executor process alive after run completion
/// </param>
/// <param name="testSettings">
/// Settings used for this run.
/// </param>
/// <param name="runStatsChangeEventTimeout">
/// Timeout that triggers sending results regardless of cache size.
/// </param>
/// <param name="testHostLauncher">
/// Test host launcher. If null then default will be used.
/// </param>
public TestRunCriteria(IEnumerable <TestCase> tests, long frequencyOfRunStatsChangeEvent, bool keepAlive, string testSettings, TimeSpan runStatsChangeEventTimeout, ITestHostLauncher testHostLauncher)
: base(frequencyOfRunStatsChangeEvent, keepAlive, testSettings, runStatsChangeEventTimeout, testHostLauncher)
{
var testCases = tests as IList <TestCase> ?? tests.ToList();
ValidateArg.NotNullOrEmpty(testCases, "tests");
this.Tests = testCases;
}
public TestableDotnetTestHostManager(ITestHostLauncher testHostLauncher, IProcessHelper processHelper, IFileHelper fileHelper)
: base(testHostLauncher, processHelper, fileHelper)
{
}
/// <inheritdoc/>
public async Task RunTestsWithCustomTestHostAsync(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
{
await RunTestsWithCustomTestHostAsync(sources, runSettings, null, testRunEventsHandler, customTestHostLauncher);
}
public void RunTestsWithCustomTestHost(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) => inner.RunTestsWithCustomTestHost(testCases, runSettings, options, testRunEventsHandler, customTestHostLauncher);
public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher) => inner.RunTestsWithCustomTestHost(sources, runSettings, testRunEventsHandler, customTestHostLauncher);
/// <inheritdoc/>
public void StartTestRunWithCustomHost(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler runEventsHandler, ITestHostLauncher customHostLauncher)
{
this.SendMessageAndListenAndReportTestResults(
MessageType.GetTestRunnerProcessStartInfoForRunSelected,
new TestRunRequestPayload
{
TestCases = testCases.ToList(),
RunSettings = runSettings,
DebuggingEnabled = customHostLauncher.IsDebug,
TestPlatformOptions = options
},
runEventsHandler,
customHostLauncher);
}
/// <inheritdoc/>
public async Task RunTestsWithCustomTestHostAsync(IEnumerable <TestCase> testCases, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
{
var testCaseList = testCases.ToList();
this.testPlatformEventSource.TranslationLayerExecutionStart(1, 0, testCaseList.Count, runSettings ?? string.Empty);
await this.EnsureInitializedAsync();
await this.requestSender.StartTestRunWithCustomHostAsync(testCaseList, runSettings, testRunEventsHandler, customTestHostLauncher);
}
/// <summary>
/// Initializes a new instance of the <see cref="BaseTestRunCriteria"/> class.
/// </summary>
/// <param name="frequencyOfRunStatsChangeEvent">
/// Frequency of <c>TestRunChangedEvent</c>.
/// </param>
/// <param name="keepAlive">
/// Specify if the test host process should be stay alive after run.
/// </param>
/// <param name="testSettings">
/// Test run settings.
/// </param>
/// <param name="runStatsChangeEventTimeout">
/// Timeout for a <c>TestRunChangedEvent</c>.
/// </param>
/// <param name="testHostLauncher">
/// Test host launcher.
/// </param>
public BaseTestRunCriteria(long frequencyOfRunStatsChangeEvent, bool keepAlive, string testSettings, TimeSpan runStatsChangeEventTimeout, ITestHostLauncher testHostLauncher)
{
if (frequencyOfRunStatsChangeEvent <= 0)
{
throw new ArgumentOutOfRangeException(nameof(frequencyOfRunStatsChangeEvent), Resources.NotificationFrequencyIsNotPositive);
}
if (runStatsChangeEventTimeout <= TimeSpan.MinValue)
{
throw new ArgumentOutOfRangeException(nameof(runStatsChangeEventTimeout), Resources.NotificationTimeoutIsZero);
}
this.FrequencyOfRunStatsChangeEvent = frequencyOfRunStatsChangeEvent;
this.KeepAlive = keepAlive;
this.TestRunSettings = testSettings;
this.RunStatsChangeEventTimeout = runStatsChangeEventTimeout;
this.TestHostLauncher = testHostLauncher;
}
/// <inheritdoc/>
public async Task StartTestRunWithCustomHostAsync(IEnumerable <TestCase> testCases, string runSettings, TestPlatformOptions options, ITestRunEventsHandler runEventsHandler, ITestHostLauncher customHostLauncher)
{
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("VsTestConsoleRequestSender.StartTestRunWithCustomHostAsync: Starting test run.");
}
await this.SendMessageAndListenAndReportTestResultsAsync(
MessageType.GetTestRunnerProcessStartInfoForRunSelected,
new TestRunRequestPayload()
{
TestCases = testCases.ToList(),
RunSettings = runSettings,
DebuggingEnabled = customHostLauncher.IsDebug,
TestPlatformOptions = options
},
runEventsHandler,
customHostLauncher);
}
/// <inheritdoc/>
public void RunTestsWithCustomTestHost(IEnumerable <string> sources, string runSettings, ITestRunEventsHandler testRunEventsHandler, ITestHostLauncher customTestHostLauncher)
{
this.RunTestsWithCustomTestHost(sources, runSettings, options: null, testRunEventsHandler: testRunEventsHandler, customTestHostLauncher: customTestHostLauncher);
}
/// <inheritdoc/>
public void SetCustomLauncher(ITestHostLauncher customLauncher)
{
this.customTestHostLauncher = customLauncher;
}
/// <inheritdoc />
public void RunTests(
TestRunRequestPayload testRunRequestPayload,
ITestHostLauncher testHostLauncher,
ITestRunEventsRegistrar testRunEventsRegistrar,
ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (testRunRequestPayload.TestPlatformOptions != null)
{
this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
// Get sources to auto detect fx and arch for both run selected or run all scenario.
var sources = GetSources(testRunRequestPayload);
if (this.UpdateRunSettingsIfRequired(
runsettings,
sources,
testRunEventsRegistrar,
out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(runsettings))
{
throw new SettingsException(
string.Format(
Resources.RunsettingsWithDCErrorMessage,
runsettings));
}
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
var batchSize = runConfiguration.BatchSize;
if (requestData.IsTelemetryOptedIn)
{
// Collect metrics.
this.CollectMetrics(requestData, runConfiguration);
// Collect commands.
this.LogCommandsTelemetryPoints(requestData);
// Collect data for legacy settings.
this.LogTelemetryForLegacySettings(requestData, runsettings);
}
// Get Fakes data collector settings.
if (!string.Equals(Environment.GetEnvironmentVariable("VSTEST_SKIP_FAKES_CONFIGURATION"), "1"))
{
// The commandline options do not have sources in design time mode,
// and so we fall back to using sources instead.
if (this.commandLineOptions.Sources.Any())
{
GenerateFakesUtilities.GenerateFakesSettings(
this.commandLineOptions,
this.commandLineOptions.Sources.ToList(),
ref runsettings);
}
else if (sources.Any())
{
GenerateFakesUtilities.GenerateFakesSettings(
this.commandLineOptions,
sources,
ref runsettings);
}
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestPlatformOptions?.TestCaseFilter,
testRunRequestPayload.TestPlatformOptions?.FilterOptions,
testRunRequestPayload.TestSessionInfo,
debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null &&
testRunRequestPayload.DebuggingEnabled);
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestSessionInfo,
debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null &&
testRunRequestPayload.DebuggingEnabled);
}
// Run tests.
try
{
this.RunTests(
requestData,
runCriteria,
testRunEventsRegistrar,
testRunRequestPayload.TestPlatformOptions);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed.");
}
finally
{
this.TestPlatformEventSourceInstance.ExecutionRequestStop();
// Post the run complete event
this.metricsPublisher.Result.PublishMetrics(
TelemetryDataConstants.TestExecutionCompleteEvent,
requestData.MetricsCollection.Metrics);
}
}