private bool RunTests(TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
// Make sure to run the run request inside a lock as the below section is not thread-safe
// TranslationLayer can process faster as it directly gets the raw unserialized messages whereas
// below logic needs to deserialize and do some cleanup
// While this section is cleaning up, TranslationLayer can trigger run causing multiple threads to run the below section at the same time
lock (syncobject)
{
bool success = true;
using (ITestRunRequest testRunRequest = this.testPlatform.CreateTestRunRequest(testRunCriteria, protocolConfig))
{
this.currentTestRunRequest = testRunRequest;
this.runRequestCreatedEventHandle.Set();
try
{
this.testLoggerManager.RegisterTestRunEvents(testRunRequest);
this.testRunResultAggregator.RegisterTestRunEvents(testRunRequest);
testRunEventsRegistrar?.RegisterTestRunEvents(testRunRequest);
this.testPlatformEventSource.ExecutionRequestStart();
testRunRequest.ExecuteAsync();
// Wait for the run completion event
testRunRequest.WaitForCompletion();
this.testPlatformEventSource.ExecutionRequestStop();
}
catch (Exception ex)
{
EqtTrace.Error("TestRequestManager.RunTests: failed to run tests: {0}", ex);
if (ex is TestPlatformException ||
ex is SettingsException ||
ex is InvalidOperationException)
{
LoggerUtilities.RaiseTestRunError(this.testLoggerManager, this.testRunResultAggregator, ex);
success = false;
}
else
{
throw;
}
}
finally
{
this.testLoggerManager.UnregisterTestRunEvents(testRunRequest);
this.testRunResultAggregator.UnregisterTestRunEvents(testRunRequest);
testRunEventsRegistrar?.UnregisterTestRunEvents(testRunRequest);
}
}
this.currentTestRunRequest = null;
return(success);
}
}
/// <summary>
/// Default constructor.
/// </summary>
public RunTestsArgumentExecutor(
CommandLineOptions commandLineOptions,
IRunSettingsProvider runSettingsProvider,
ITestRequestManager testRequestManager,
IOutput output)
{
Contract.Requires(commandLineOptions != null);
this.commandLineOptions = commandLineOptions;
this.runSettingsManager = runSettingsProvider;
this.testRequestManager = testRequestManager;
this.output = output;
this.testRunEventsRegistrar = new TestRunRequestEventsRegistrar(this.output);
}
private void RunTests(
IRequestData requestData,
TestRunCriteria testRunCriteria,
ITestRunEventsRegistrar testRunEventsRegistrar,
TestPlatformOptions options)
{
// Make sure to run the run request inside a lock as the below section is not thread-safe.
// TranslationLayer can process faster as it directly gets the raw un-serialized messages
// whereas below logic needs to deserialize and do some cleanup.
// While this section is cleaning up, TranslationLayer can trigger run causing multiple
// threads to run the below section at the same time.
lock (this.syncObject)
{
try
{
this.currentTestRunRequest = this.testPlatform.CreateTestRunRequest(
requestData,
testRunCriteria,
options);
this.testRunResultAggregator.RegisterTestRunEvents(this.currentTestRunRequest);
testRunEventsRegistrar?.RegisterTestRunEvents(this.currentTestRunRequest);
((Microsoft.VisualStudio.TestPlatform.Client.TestPlatform) this.testPlatform).VStestLoggerManager = this.VSTestLoggerManager;
this.TestPlatformEventSourceInstance.ExecutionRequestStart();
this.currentTestRunRequest.ExecuteAsync();
// Wait for the run completion event
this.currentTestRunRequest.WaitForCompletion();
}
catch (Exception ex)
{
EqtTrace.Error("TestRequestManager.RunTests: failed to run tests: {0}", ex);
testRunResultAggregator.MarkTestRunFailed();
throw;
}
finally
{
if (this.currentTestRunRequest != null)
{
this.testRunResultAggregator.UnregisterTestRunEvents(this.currentTestRunRequest);
testRunEventsRegistrar?.UnregisterTestRunEvents(this.currentTestRunRequest);
this.currentTestRunRequest.Dispose();
this.currentTestRunRequest = null;
}
}
}
}
private bool RunTests(TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar)
{
bool success = true;
using (this.currentTestRunRequest = this.testPlatform.CreateTestRunRequest(testRunCriteria))
{
this.runRequestCreatedEventHandle.Set();
try
{
this.testLoggerManager.RegisterTestRunEvents(this.currentTestRunRequest);
this.testRunResultAggregator.RegisterTestRunEvents(this.currentTestRunRequest);
testRunEventsRegistrar?.RegisterTestRunEvents(this.currentTestRunRequest);
this.testPlatformEventSource.ExecutionRequestStart();
this.currentTestRunRequest.ExecuteAsync();
// Wait for the run completion event
this.currentTestRunRequest.WaitForCompletion();
this.testPlatformEventSource.ExecutionRequestStop();
}
catch (Exception ex)
{
if (ex is TestPlatformException ||
ex is SettingsException ||
ex is InvalidOperationException)
{
LoggerUtilities.RaiseTestRunError(this.testLoggerManager, this.testRunResultAggregator, ex);
success = false;
}
else
{
throw;
}
}
finally
{
this.testLoggerManager.UnregisterTestRunEvents(this.currentTestRunRequest);
this.testRunResultAggregator.UnregisterTestRunEvents(this.currentTestRunRequest);
testRunEventsRegistrar?.UnregisterTestRunEvents(this.currentTestRunRequest);
}
}
this.currentTestRunRequest = null;
return(success);
}
/// <summary>
/// Default constructor.
/// </summary>
public RunSpecificTestsArgumentExecutor(
CommandLineOptions options,
IRunSettingsProvider runSettingsProvider,
ITestRequestManager testRequestManager,
IOutput output)
{
Contract.Requires(options != null);
Contract.Requires(testRequestManager != null);
this.commandLineOptions = options;
this.testRequestManager = testRequestManager;
this.runSettingsManager = runSettingsProvider;
this.output = output;
this.discoveryEventsRegistrar = new DiscoveryEventsRegistrar(this.discoveryRequest_OnDiscoveredTests);
this.testRunEventsRegistrar = new TestRunRequestEventsRegistrar(this.output, this.commandLineOptions);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public void RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (testRunRequestPayload.TestPlatformOptions != null)
{
this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
// Get sources to auto detect fx and arch for both run selected or run all scenario.
var sources = GetSources(testRunRequestPayload);
if (this.UpdateRunSettingsIfRequired(runsettings, sources, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsInCompatibleWithTestSettings(runsettings))
{
throw new SettingsException(string.Format(Resources.RunsettingsWithDCErrorMessage, runsettings));
}
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
var batchSize = runConfiguration.BatchSize;
if (requestData.IsTelemetryOptedIn)
{
// Collect Metrics
this.CollectMetrics(requestData, runConfiguration);
// Collect Commands
this.LogCommandsTelemetryPoints(requestData);
// Collect data for Legacy Settings
this.LogTelemetryForLegacySettings(requestData, runsettings);
}
if (!commandLineOptions.IsDesignMode)
{
// Generate fakes settings only for command line scenarios. In case of
// Editors/IDEs, this responsibility is with the caller.
GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings);
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestPlatformOptions?.TestCaseFilter,
testRunRequestPayload.TestPlatformOptions?.FilterOptions);
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
// Run tests
try
{
this.RunTests(requestData, runCriteria, testRunEventsRegistrar, testRunRequestPayload.TestPlatformOptions);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed.");
}
finally
{
this.testPlatformEventSource.ExecutionRequestStop();
// Post the run complete event
this.metricsPublisher.Result.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics);
}
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings);
var batchSize = runConfiguration.BatchSize;
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (this.UpdateRunSettingsIfRequired(runsettings, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
runsettings = UpdateExtensionsFolderInRunSettings(runsettings);
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(runCriteria, testRunEventsRegistrar, protocolConfig);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
this.testPlatformEventSource.ExecutionRequestStop();
return(success);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings);
var batchSize = runConfiguration.BatchSize;
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
var requestData = this.GetRequestData(protocolConfig);
var metricsPublisher = this.telemetryOptedIn ? (IMetricsPublisher) new MetricsPublisher() : new NoOpMetricsPublisher();
if (this.UpdateRunSettingsIfRequired(runsettings, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = testRunRequestPayload.TestCaseFilter;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(requestData, runCriteria, testRunEventsRegistrar, protocolConfig);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
this.testPlatformEventSource.ExecutionRequestStop();
metricsPublisher.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics);
metricsPublisher.Dispose();
return(success);
}
/// <inheritdoc />
public void RunTests(
TestRunRequestPayload testRunRequestPayload,
ITestHostLauncher testHostLauncher,
ITestRunEventsRegistrar testRunEventsRegistrar,
ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (testRunRequestPayload.TestPlatformOptions != null)
{
this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
// Get sources to auto detect fx and arch for both run selected or run all scenario.
var sources = GetSources(testRunRequestPayload);
if (this.UpdateRunSettingsIfRequired(
runsettings,
sources,
testRunEventsRegistrar,
out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(runsettings))
{
throw new SettingsException(
string.Format(
Resources.RunsettingsWithDCErrorMessage,
runsettings));
}
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
var batchSize = runConfiguration.BatchSize;
if (requestData.IsTelemetryOptedIn)
{
// Collect metrics.
this.CollectMetrics(requestData, runConfiguration);
// Collect commands.
this.LogCommandsTelemetryPoints(requestData);
// Collect data for legacy settings.
this.LogTelemetryForLegacySettings(requestData, runsettings);
}
// Get Fakes data collector settings.
if (!string.Equals(Environment.GetEnvironmentVariable("VSTEST_SKIP_FAKES_CONFIGURATION"), "1"))
{
// The commandline options do not have sources in design time mode,
// and so we fall back to using sources instead.
if (this.commandLineOptions.Sources.Any())
{
GenerateFakesUtilities.GenerateFakesSettings(
this.commandLineOptions,
this.commandLineOptions.Sources.ToList(),
ref runsettings);
}
else if (sources.Any())
{
GenerateFakesUtilities.GenerateFakesSettings(
this.commandLineOptions,
sources,
ref runsettings);
}
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestPlatformOptions?.TestCaseFilter,
testRunRequestPayload.TestPlatformOptions?.FilterOptions,
testRunRequestPayload.TestSessionInfo,
debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null &&
testRunRequestPayload.DebuggingEnabled);
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestSessionInfo,
debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null &&
testRunRequestPayload.DebuggingEnabled);
}
// Run tests.
try
{
this.RunTests(
requestData,
runCriteria,
testRunEventsRegistrar,
testRunRequestPayload.TestPlatformOptions);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed.");
}
finally
{
this.TestPlatformEventSourceInstance.ExecutionRequestStop();
// Post the run complete event
this.metricsPublisher.Result.PublishMetrics(
TelemetryDataConstants.TestExecutionCompleteEvent,
requestData.MetricsCollection.Metrics);
}
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings);
var batchSize = runConfiguration.BatchSize;
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (this.TryUpdateDesignMode(runsettings, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(runCriteria, testRunEventsRegistrar);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
return(success);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar)
{
TestRunCriteria runCriteria = null;
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
this.commandLineOptions.BatchSize,
testRunRequestPayload.KeepAlive,
testRunRequestPayload.RunSettings,
this.commandLineOptions.TestRunStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
this.commandLineOptions.BatchSize,
testRunRequestPayload.KeepAlive,
testRunRequestPayload.RunSettings,
this.commandLineOptions.TestRunStatsEventTimeout,
testHostLauncher);
}
return(this.RunTests(runCriteria, testRunEventsRegistrar));
}
private void RunTests(IRequestData requestData, TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar)
{
// Make sure to run the run request inside a lock as the below section is not thread-safe
// TranslationLayer can process faster as it directly gets the raw unserialized messages whereas
// below logic needs to deserialize and do some cleanup
// While this section is cleaning up, TranslationLayer can trigger run causing multiple threads to run the below section at the same time
lock (syncobject)
{
try
{
this.currentTestRunRequest = this.testPlatform.CreateTestRunRequest(requestData, testRunCriteria);
this.testRunResultAggregator.RegisterTestRunEvents(this.currentTestRunRequest);
testRunEventsRegistrar?.RegisterTestRunEvents(this.currentTestRunRequest);
this.testPlatformEventSource.ExecutionRequestStart();
this.currentTestRunRequest.ExecuteAsync();
this.runRequestStartedEventHandle.Set();
// Wait for the run completion event
this.currentTestRunRequest.WaitForCompletion();
}
catch (Exception ex)
{
EqtTrace.Error("TestRequestManager.RunTests: failed to run tests: {0}", ex);
testRunResultAggregator.MarkTestRunFailed();
throw;
}
finally
{
if (this.currentTestRunRequest != null)
{
this.testRunResultAggregator.UnregisterTestRunEvents(this.currentTestRunRequest);
testRunEventsRegistrar?.UnregisterTestRunEvents(this.currentTestRunRequest);
this.currentTestRunRequest.Dispose();
this.currentTestRunRequest = null;
}
}
}
}
