private bool RunTests(TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig) { // Make sure to run the run request inside a lock as the below section is not thread-safe // TranslationLayer can process faster as it directly gets the raw unserialized messages whereas // below logic needs to deserialize and do some cleanup // While this section is cleaning up, TranslationLayer can trigger run causing multiple threads to run the below section at the same time lock (syncobject) { bool success = true; using (ITestRunRequest testRunRequest = this.testPlatform.CreateTestRunRequest(testRunCriteria, protocolConfig)) { this.currentTestRunRequest = testRunRequest; this.runRequestCreatedEventHandle.Set(); try { this.testLoggerManager.RegisterTestRunEvents(testRunRequest); this.testRunResultAggregator.RegisterTestRunEvents(testRunRequest); testRunEventsRegistrar?.RegisterTestRunEvents(testRunRequest); this.testPlatformEventSource.ExecutionRequestStart(); testRunRequest.ExecuteAsync(); // Wait for the run completion event testRunRequest.WaitForCompletion(); this.testPlatformEventSource.ExecutionRequestStop(); } catch (Exception ex) { EqtTrace.Error("TestRequestManager.RunTests: failed to run tests: {0}", ex); if (ex is TestPlatformException || ex is SettingsException || ex is InvalidOperationException) { LoggerUtilities.RaiseTestRunError(this.testLoggerManager, this.testRunResultAggregator, ex); success = false; } else { throw; } } finally { this.testLoggerManager.UnregisterTestRunEvents(testRunRequest); this.testRunResultAggregator.UnregisterTestRunEvents(testRunRequest); testRunEventsRegistrar?.UnregisterTestRunEvents(testRunRequest); } } this.currentTestRunRequest = null; return(success); } }
/// <summary> /// Default constructor. /// </summary> public RunTestsArgumentExecutor( CommandLineOptions commandLineOptions, IRunSettingsProvider runSettingsProvider, ITestRequestManager testRequestManager, IOutput output) { Contract.Requires(commandLineOptions != null); this.commandLineOptions = commandLineOptions; this.runSettingsManager = runSettingsProvider; this.testRequestManager = testRequestManager; this.output = output; this.testRunEventsRegistrar = new TestRunRequestEventsRegistrar(this.output); }
private void RunTests( IRequestData requestData, TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar, TestPlatformOptions options) { // Make sure to run the run request inside a lock as the below section is not thread-safe. // TranslationLayer can process faster as it directly gets the raw un-serialized messages // whereas below logic needs to deserialize and do some cleanup. // While this section is cleaning up, TranslationLayer can trigger run causing multiple // threads to run the below section at the same time. lock (this.syncObject) { try { this.currentTestRunRequest = this.testPlatform.CreateTestRunRequest( requestData, testRunCriteria, options); this.testRunResultAggregator.RegisterTestRunEvents(this.currentTestRunRequest); testRunEventsRegistrar?.RegisterTestRunEvents(this.currentTestRunRequest); ((Microsoft.VisualStudio.TestPlatform.Client.TestPlatform) this.testPlatform).VStestLoggerManager = this.VSTestLoggerManager; this.TestPlatformEventSourceInstance.ExecutionRequestStart(); this.currentTestRunRequest.ExecuteAsync(); // Wait for the run completion event this.currentTestRunRequest.WaitForCompletion(); } catch (Exception ex) { EqtTrace.Error("TestRequestManager.RunTests: failed to run tests: {0}", ex); testRunResultAggregator.MarkTestRunFailed(); throw; } finally { if (this.currentTestRunRequest != null) { this.testRunResultAggregator.UnregisterTestRunEvents(this.currentTestRunRequest); testRunEventsRegistrar?.UnregisterTestRunEvents(this.currentTestRunRequest); this.currentTestRunRequest.Dispose(); this.currentTestRunRequest = null; } } } }
private bool RunTests(TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar) { bool success = true; using (this.currentTestRunRequest = this.testPlatform.CreateTestRunRequest(testRunCriteria)) { this.runRequestCreatedEventHandle.Set(); try { this.testLoggerManager.RegisterTestRunEvents(this.currentTestRunRequest); this.testRunResultAggregator.RegisterTestRunEvents(this.currentTestRunRequest); testRunEventsRegistrar?.RegisterTestRunEvents(this.currentTestRunRequest); this.testPlatformEventSource.ExecutionRequestStart(); this.currentTestRunRequest.ExecuteAsync(); // Wait for the run completion event this.currentTestRunRequest.WaitForCompletion(); this.testPlatformEventSource.ExecutionRequestStop(); } catch (Exception ex) { if (ex is TestPlatformException || ex is SettingsException || ex is InvalidOperationException) { LoggerUtilities.RaiseTestRunError(this.testLoggerManager, this.testRunResultAggregator, ex); success = false; } else { throw; } } finally { this.testLoggerManager.UnregisterTestRunEvents(this.currentTestRunRequest); this.testRunResultAggregator.UnregisterTestRunEvents(this.currentTestRunRequest); testRunEventsRegistrar?.UnregisterTestRunEvents(this.currentTestRunRequest); } } this.currentTestRunRequest = null; return(success); }
/// <summary> /// Default constructor. /// </summary> public RunSpecificTestsArgumentExecutor( CommandLineOptions options, IRunSettingsProvider runSettingsProvider, ITestRequestManager testRequestManager, IOutput output) { Contract.Requires(options != null); Contract.Requires(testRequestManager != null); this.commandLineOptions = options; this.testRequestManager = testRequestManager; this.runSettingsManager = runSettingsProvider; this.output = output; this.discoveryEventsRegistrar = new DiscoveryEventsRegistrar(this.discoveryRequest_OnDiscoveredTests); this.testRunEventsRegistrar = new TestRunRequestEventsRegistrar(this.output, this.commandLineOptions); }
/// <summary> /// Run Tests with given a set of test cases. /// </summary> /// <param name="testRunRequestPayload">TestRun request Payload</param> /// <param name="testHostLauncher">TestHost Launcher for the run</param> /// <param name="testRunEventsRegistrar">event registrar for run events</param> /// <param name="protocolConfig">Protocol related information</param> /// <returns>True, if successful</returns> public void RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; if (testRunRequestPayload.TestPlatformOptions != null) { this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics; } var requestData = this.GetRequestData(protocolConfig); // Get sources to auto detect fx and arch for both run selected or run all scenario. var sources = GetSources(testRunRequestPayload); if (this.UpdateRunSettingsIfRequired(runsettings, sources, out string updatedRunsettings)) { runsettings = updatedRunsettings; } if (InferRunSettingsHelper.AreRunSettingsCollectorsInCompatibleWithTestSettings(runsettings)) { throw new SettingsException(string.Format(Resources.RunsettingsWithDCErrorMessage, runsettings)); } var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings); var batchSize = runConfiguration.BatchSize; if (requestData.IsTelemetryOptedIn) { // Collect Metrics this.CollectMetrics(requestData, runConfiguration); // Collect Commands this.LogCommandsTelemetryPoints(requestData); // Collect data for Legacy Settings this.LogTelemetryForLegacySettings(requestData, runsettings); } if (!commandLineOptions.IsDesignMode) { // Generate fakes settings only for command line scenarios. In case of // Editors/IDEs, this responsibility is with the caller. GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings); } if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher, testRunRequestPayload.TestPlatformOptions?.TestCaseFilter, testRunRequestPayload.TestPlatformOptions?.FilterOptions); } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); } // Run tests try { this.RunTests(requestData, runCriteria, testRunEventsRegistrar, testRunRequestPayload.TestPlatformOptions); EqtTrace.Info("TestRequestManager.RunTests: run tests completed."); } finally { this.testPlatformEventSource.ExecutionRequestStop(); // Post the run complete event this.metricsPublisher.Result.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics); } }
/// <summary> /// Run Tests with given a set of test cases. /// </summary> /// <param name="testRunRequestPayload">TestRun request Payload</param> /// <param name="testHostLauncher">TestHost Launcher for the run</param> /// <param name="testRunEventsRegistrar">event registrar for run events</param> /// <param name="protocolConfig">Protocol related information</param> /// <returns>True, if successful</returns> public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings); var batchSize = runConfiguration.BatchSize; TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; if (this.UpdateRunSettingsIfRequired(runsettings, out string updatedRunsettings)) { runsettings = updatedRunsettings; } runsettings = UpdateExtensionsFolderInRunSettings(runsettings); if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue; } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); } var success = this.RunTests(runCriteria, testRunEventsRegistrar, protocolConfig); EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success); this.testPlatformEventSource.ExecutionRequestStop(); return(success); }
/// <summary> /// Run Tests with given a set of test cases. /// </summary> /// <param name="testRunRequestPayload">TestRun request Payload</param> /// <param name="testHostLauncher">TestHost Launcher for the run</param> /// <param name="testRunEventsRegistrar">event registrar for run events</param> /// <param name="protocolConfig">Protocol related information</param> /// <returns>True, if successful</returns> public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings); var batchSize = runConfiguration.BatchSize; TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; var requestData = this.GetRequestData(protocolConfig); var metricsPublisher = this.telemetryOptedIn ? (IMetricsPublisher) new MetricsPublisher() : new NoOpMetricsPublisher(); if (this.UpdateRunSettingsIfRequired(runsettings, out string updatedRunsettings)) { runsettings = updatedRunsettings; } if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); runCriteria.TestCaseFilter = testRunRequestPayload.TestCaseFilter; } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); } var success = this.RunTests(requestData, runCriteria, testRunEventsRegistrar, protocolConfig); EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success); this.testPlatformEventSource.ExecutionRequestStop(); metricsPublisher.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics); metricsPublisher.Dispose(); return(success); }
/// <inheritdoc /> public void RunTests( TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; if (testRunRequestPayload.TestPlatformOptions != null) { this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics; } var requestData = this.GetRequestData(protocolConfig); // Get sources to auto detect fx and arch for both run selected or run all scenario. var sources = GetSources(testRunRequestPayload); if (this.UpdateRunSettingsIfRequired( runsettings, sources, testRunEventsRegistrar, out string updatedRunsettings)) { runsettings = updatedRunsettings; } if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(runsettings)) { throw new SettingsException( string.Format( Resources.RunsettingsWithDCErrorMessage, runsettings)); } var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings); var batchSize = runConfiguration.BatchSize; if (requestData.IsTelemetryOptedIn) { // Collect metrics. this.CollectMetrics(requestData, runConfiguration); // Collect commands. this.LogCommandsTelemetryPoints(requestData); // Collect data for legacy settings. this.LogTelemetryForLegacySettings(requestData, runsettings); } // Get Fakes data collector settings. if (!string.Equals(Environment.GetEnvironmentVariable("VSTEST_SKIP_FAKES_CONFIGURATION"), "1")) { // The commandline options do not have sources in design time mode, // and so we fall back to using sources instead. if (this.commandLineOptions.Sources.Any()) { GenerateFakesUtilities.GenerateFakesSettings( this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings); } else if (sources.Any()) { GenerateFakesUtilities.GenerateFakesSettings( this.commandLineOptions, sources, ref runsettings); } } if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher, testRunRequestPayload.TestPlatformOptions?.TestCaseFilter, testRunRequestPayload.TestPlatformOptions?.FilterOptions, testRunRequestPayload.TestSessionInfo, debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null && testRunRequestPayload.DebuggingEnabled); } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher, testRunRequestPayload.TestSessionInfo, debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null && testRunRequestPayload.DebuggingEnabled); } // Run tests. try { this.RunTests( requestData, runCriteria, testRunEventsRegistrar, testRunRequestPayload.TestPlatformOptions); EqtTrace.Info("TestRequestManager.RunTests: run tests completed."); } finally { this.TestPlatformEventSourceInstance.ExecutionRequestStop(); // Post the run complete event this.metricsPublisher.Result.PublishMetrics( TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics); } }
/// <summary> /// Run Tests with given a set of test cases. /// </summary> /// <param name="testRunRequestPayload">TestRun request Payload</param> /// <param name="testHostLauncher">TestHost Launcher for the run</param> /// <param name="testRunEventsRegistrar">event registrar for run events</param> /// <returns>True, if successful</returns> public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings); var batchSize = runConfiguration.BatchSize; TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; if (this.TryUpdateDesignMode(runsettings, out string updatedRunsettings)) { runsettings = updatedRunsettings; } if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue; } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); } var success = this.RunTests(runCriteria, testRunEventsRegistrar); EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success); return(success); }
/// <summary> /// Run Tests with given a set of test cases. /// </summary> /// <param name="testRunRequestPayload">TestRun request Payload</param> /// <param name="testHostLauncher">TestHost Launcher for the run</param> /// <param name="testRunEventsRegistrar">event registrar for run events</param> /// <returns>True, if successful</returns> public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar) { TestRunCriteria runCriteria = null; if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, this.commandLineOptions.BatchSize, testRunRequestPayload.KeepAlive, testRunRequestPayload.RunSettings, this.commandLineOptions.TestRunStatsEventTimeout, testHostLauncher); runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue; } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, this.commandLineOptions.BatchSize, testRunRequestPayload.KeepAlive, testRunRequestPayload.RunSettings, this.commandLineOptions.TestRunStatsEventTimeout, testHostLauncher); } return(this.RunTests(runCriteria, testRunEventsRegistrar)); }
private void RunTests(IRequestData requestData, TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar) { // Make sure to run the run request inside a lock as the below section is not thread-safe // TranslationLayer can process faster as it directly gets the raw unserialized messages whereas // below logic needs to deserialize and do some cleanup // While this section is cleaning up, TranslationLayer can trigger run causing multiple threads to run the below section at the same time lock (syncobject) { try { this.currentTestRunRequest = this.testPlatform.CreateTestRunRequest(requestData, testRunCriteria); this.testRunResultAggregator.RegisterTestRunEvents(this.currentTestRunRequest); testRunEventsRegistrar?.RegisterTestRunEvents(this.currentTestRunRequest); this.testPlatformEventSource.ExecutionRequestStart(); this.currentTestRunRequest.ExecuteAsync(); this.runRequestStartedEventHandle.Set(); // Wait for the run completion event this.currentTestRunRequest.WaitForCompletion(); } catch (Exception ex) { EqtTrace.Error("TestRequestManager.RunTests: failed to run tests: {0}", ex); testRunResultAggregator.MarkTestRunFailed(); throw; } finally { if (this.currentTestRunRequest != null) { this.testRunResultAggregator.UnregisterTestRunEvents(this.currentTestRunRequest); testRunEventsRegistrar?.UnregisterTestRunEvents(this.currentTestRunRequest); this.currentTestRunRequest.Dispose(); this.currentTestRunRequest = null; } } } }