public void UpdateRunSettingsWithAnEmptyRunSettingsShouldAddValuesSpecifiedInRunConfiguration()
{
var settings = @"<RunSettings></RunSettings>";
var xmlDocument = this.GetXmlDocument(settings);
InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(xmlDocument, Architecture.X64, Framework.DefaultFramework, "temp");
var xml = xmlDocument.OuterXml;
StringAssert.Contains(xml, "<TargetPlatform>X64</TargetPlatform>");
StringAssert.Contains(xml, $"<TargetFrameworkVersion>{Framework.DefaultFramework.Name}</TargetFrameworkVersion>");
StringAssert.Contains(xml, "<ResultsDirectory>temp</ResultsDirectory>");
}
private void LogTelemetryForLegacySettings(IRequestData requestData, string runsettings)
{
requestData.MetricsCollection.Add(TelemetryDataConstants.TestSettingsUsed, InferRunSettingsHelper.IsTestSettingsEnabled(runsettings));
if (InferRunSettingsHelper.TryGetLegacySettingElements(runsettings, out Dictionary <string, string> legacySettingsTelemetry))
{
foreach (var ciData in legacySettingsTelemetry)
{
// We are collecting telemetry for the legacy nodes and attributes used in the runsettings.
requestData.MetricsCollection.Add(string.Format("{0}.{1}", TelemetryDataConstants.LegacySettingPrefix, ciData.Key), ciData.Value);
}
}
}
public void UpdateRunSettingsShouldThrowIfArchitectureSetIsIncompatibleWithCurrentSystemArchitecture()
{
var settings = @"<RunSettings></RunSettings>";
var xmlDocument = this.GetXmlDocument(settings);
Action action = () => InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(xmlDocument, Architecture.ARM, Framework.DefaultFramework, "temp");
Assert.That.Throws <SettingsException>(action)
.WithMessage(string.Format(
"Incompatible Target platform settings '{0}' with system architecture '{1}'.",
"ARM",
XmlRunSettingsUtilities.OSArchitecture.ToString()));
}
public void UpdateRunSettingsShouldNotUpdatePlatformOrFrameworkOrResultsDirectoryIfRunSettingsAlreadyHasIt()
{
var settings = @"<RunSettings><RunConfiguration><TargetPlatform>X86</TargetPlatform><TargetFrameworkVersion>Framework40</TargetFrameworkVersion><ResultsDirectory>someplace</ResultsDirectory></RunConfiguration></RunSettings>";
var xmlDocument = this.GetXmlDocument(settings);
InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(xmlDocument, Architecture.X64, Framework.DefaultFramework, "temp");
var xml = xmlDocument.OuterXml;
StringAssert.Contains(xml, "<TargetPlatform>X86</TargetPlatform>");
StringAssert.Contains(xml, "<TargetFrameworkVersion>Framework40</TargetFrameworkVersion>");
StringAssert.Contains(xml, "<ResultsDirectory>someplace</ResultsDirectory>");
}
private bool ShouldRunInNoIsolation(string runsettings, bool isParallelEnabled, bool isDataCollectorEnabled)
{
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
if (runConfiguration.InIsolation)
{
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("TestEngine.ShouldRunInNoIsolation: running test in isolation");
}
return(false);
}
// Run tests in isolation if run is authored using testsettings.
if (InferRunSettingsHelper.IsTestSettingsEnabled(runsettings))
{
return(false);
}
var currentProcessPath = this.processHelper.GetCurrentProcessFileName();
// If running with the dotnet executable, then don't run in InProcess
if (currentProcessPath.EndsWith("dotnet", StringComparison.OrdinalIgnoreCase) ||
currentProcessPath.EndsWith("dotnet.exe", StringComparison.OrdinalIgnoreCase))
{
return(false);
}
// Return true if
// 1) Not running in parallel
// 2) Data collector is not enabled
// 3) Target framework is x86 or anyCpu
// 4) DisableAppDomain is false
// 5) Not running in design mode
// 6) target framework is NETFramework (Desktop test)
if (!isParallelEnabled &&
!isDataCollectorEnabled &&
(runConfiguration.TargetPlatform == Architecture.X86 || runConfiguration.TargetPlatform == Architecture.AnyCPU) &&
!runConfiguration.DisableAppDomain &&
!runConfiguration.DesignMode &&
runConfiguration.TargetFramework.Name.IndexOf("netframework", StringComparison.OrdinalIgnoreCase) >= 0)
{
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("TestEngine.ShouldRunInNoIsolation: running test in process(inside vstest.console.exe process)");
}
return(true);
}
return(false);
}
public void GetEnvironmentVariablesWithInvalidValuesInRunSettingsShouldReturnNull()
{
string runSettingsXml = @"<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<Foo>
</EnvironmentVariables>
</RunConfiguration>
</RunSettings>" ;
var envVars = InferRunSettingsHelper.GetEnvironmentVariables(runSettingsXml);
Assert.IsNull(envVars);
}
private bool UpdateCollectSourceInformation(
XmlDocument document,
RunConfiguration runConfiguration)
{
bool updateRequired = !runConfiguration.CollectSourceInformationSet;
if (updateRequired)
{
InferRunSettingsHelper.UpdateCollectSourceInformation(
document,
this.commandLineOptions.ShouldCollectSourceInformation);
}
return(updateRequired);
}
public void UpdateRunSettingsShouldThrowIfFrameworkNodeIsInvalid()
{
var settings = @"<RunSettings><RunConfiguration><TargetFrameworkVersion>foo</TargetFrameworkVersion></RunConfiguration></RunSettings>";
var navigator = this.GetNavigator(settings);
Action action = () => InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(navigator, Architecture.X86, Framework.DefaultFramework, "temp");
Assert.That.Throws <XmlException>(action)
.WithMessage(string.Format("An error occurred while loading the settings. Error: {0}.",
string.Format("Invalid setting '{0}'. Invalid value '{1}' specified for '{2}'",
"RunConfiguration",
"foo",
"TargetFrameworkVersion")));
}
public void GetEnvironmentVariablesWithDuplicateEnvValuesInRunSettingsShouldReturnValidDictionary()
{
string runSettingsXml = @"<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<RANDOM_PATH>C:\temp</RANDOM_PATH>
<RANDOM_PATH>C:\temp2</RANDOM_PATH>
</EnvironmentVariables>
</RunConfiguration>
</RunSettings>" ;
var envVars = InferRunSettingsHelper.GetEnvironmentVariables(runSettingsXml);
Assert.AreEqual(1, envVars.Count);
Assert.AreEqual(@"C:\temp", envVars["RANDOM_PATH"]);
}
/// <summary>
/// This function will remove the unknown run settings nodes from the run settings strings.
/// This is necessary because older test hosts may throw exceptions when encountering
/// unknown nodes.
/// </summary>
///
/// <param name="runsettingsXml">Run settings string.</param>
///
/// <returns>The run settings after removing non-required nodes.</returns>
public string RemoveNodesFromRunsettingsIfRequired(string runsettingsXml, Action<TestMessageLevel, string> logMessage)
{
var updatedRunSettingsXml = runsettingsXml;
if (!this.makeRunsettingsCompatibleSet)
{
this.CompatIssueWithVersionCheckAndRunsettings();
}
if (this.makeRunsettingsCompatible)
{
logMessage.Invoke(TestMessageLevel.Warning, CrossPlatEngineResources.OldTestHostIsGettingUsed);
updatedRunSettingsXml = InferRunSettingsHelper.MakeRunsettingsCompatible(runsettingsXml);
}
return updatedRunSettingsXml;
}
private static string AddDefaultRunSettings(string runSettings)
{
var architecture = Constants.DefaultPlatform;
var framework = Framework.DefaultFramework;
var defaultResultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.ResultsDirectoryName);
using (var stream = new StringReader(runSettings))
using (var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings))
{
var document = new XmlDocument();
document.Load(reader);
InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(document, architecture, framework, defaultResultsDirectory);
return(document.OuterXml);
}
}
private bool UpdatePlatform(XmlDocument document, XPathNavigator navigator, List <string> sources, IDictionary <string, Architecture> sourcePlatforms, Architecture defaultArchitecture, out Architecture chosenPlatform)
{
// Get platform from sources
var inferedPlatform = inferHelper.AutoDetectArchitecture(sources, sourcePlatforms, defaultArchitecture);
// Get platform from runsettings
bool updatePlatform = IsAutoPlatformDetectRequired(navigator, out chosenPlatform);
// Update platform if required. For command line scenario update happens in ArgumentProcessor.
if (updatePlatform)
{
InferRunSettingsHelper.UpdateTargetPlatform(document, inferedPlatform.ToString(), overwrite: true);
chosenPlatform = inferedPlatform;
}
return(updatePlatform);
}
private bool UpdateRunSettingsIfRequired(string runsettingsXml, out string updatedRunSettingsXml)
{
bool settingsUpdated = false;
updatedRunSettingsXml = runsettingsXml;
if (!string.IsNullOrEmpty(runsettingsXml))
{
// TargetFramework is full CLR. Set DesignMode based on current context.
using (var stream = new StringReader(runsettingsXml))
using (var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings))
{
var document = new XmlDocument();
document.Load(reader);
var navigator = document.CreateNavigator();
// If user is already setting DesignMode via runsettings or CLI args; we skip.
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettingsXml);
if (!runConfiguration.DesignModeSet)
{
InferRunSettingsHelper.UpdateDesignMode(navigator, this.commandLineOptions.IsDesignMode);
settingsUpdated = true;
}
if (!runConfiguration.CollectSourceInformationSet)
{
InferRunSettingsHelper.UpdateCollectSourceInformation(navigator, this.commandLineOptions.ShouldCollectSourceInformation);
settingsUpdated = true;
}
if (InferRunSettingsHelper.TryGetDeviceXml(navigator, out string deviceXml))
{
InferRunSettingsHelper.UpdateTargetDeviceInformation(navigator, deviceXml);
settingsUpdated = true;
}
updatedRunSettingsXml = navigator.OuterXml;
}
}
return(settingsUpdated);
}
private void CheckSourcesForCompatibility(Framework chosenFramework, Architecture chosenPlatform, IDictionary <string, Architecture> sourcePlatforms, IDictionary <string, Framework> sourceFrameworks, IBaseTestEventsRegistrar registrar)
{
// Find compatible sources
var compatibleSources = InferRunSettingsHelper.FilterCompatibleSources(chosenPlatform, chosenFramework, sourcePlatforms, sourceFrameworks, out var incompatibleSettingWarning);
// Raise warnings for incompatible sources
if (!string.IsNullOrEmpty(incompatibleSettingWarning))
{
EqtTrace.Warning(incompatibleSettingWarning);
registrar?.LogWarning(incompatibleSettingWarning);
}
// Log compatible sources
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("Compatible sources list : ");
EqtTrace.Info(string.Join("\n", compatibleSources.ToArray()));
}
}
/// <inheritdoc />
public void Initialize(string argument)
{
// 1. Disable all other data collectors. Enable only those data collectors that are explicitly specified by user.
// 2. Check if Code Coverage Data Collector is specified in runsettings, if not add it and also set enable to true.
// if argument is null or doesn't contain any element, don't do anything.
if (string.IsNullOrWhiteSpace(argument))
{
throw new CommandLineException(
string.Format(
CultureInfo.CurrentUICulture,
CommandLineResources.DataCollectorFriendlyNameInvalid,
argument));
}
if (InferRunSettingsHelper.IsTestSettingsEnabled(this.runSettingsManager.ActiveRunSettings.SettingsXml))
{
throw new SettingsException(string.Format(CommandLineResources.CollectWithTestSettingErrorMessage, argument));
}
AddDataCollectorToRunSettings(argument, this.runSettingsManager, this.fileHelper);
}
private bool IsAutoPlatformDetectRequired(XPathNavigator navigator, out Architecture chosenPlatform)
{
bool required = true;
chosenPlatform = Architecture.Default;
if (commandLineOptions.IsDesignMode)
{
bool isValidPlatform = InferRunSettingsHelper.TryGetPlatformXml(navigator, out var platformXml);
required = !isValidPlatform || string.IsNullOrWhiteSpace(platformXml);
if (!required)
{
chosenPlatform = (Architecture)Enum.Parse(typeof(Architecture), platformXml, true);
}
}
else if (!commandLineOptions.IsDesignMode && commandLineOptions.ArchitectureSpecified)
{
required = false;
chosenPlatform = commandLineOptions.TargetArchitecture;
}
return(required);
}
public void FilterCompatibleSourcesShouldIdentifyCompatibleSources()
{
sourceArchitectures["x64net45.exe"] = Architecture.X64;
sourceArchitectures["x86net45.dll"] = Architecture.X86;
sourceFrameworks["x64net45.exe"] = frameworkNet45;
sourceFrameworks["x86net45.dll"] = frameworkNet45;
StringBuilder sb = new StringBuilder();
sb.AppendLine();
sb.AppendLine(GetSourceIncompatibleMessage("x64net45.exe"));
var expected = string.Format(CultureInfo.CurrentCulture, OMResources.DisplayChosenSettings, frameworkNet45, Constants.DefaultPlatform, sb.ToString(), @"http://go.microsoft.com/fwlink/?LinkID=236877&clcid=0x409");
string warningMessage = string.Empty;
var compatibleSources = InferRunSettingsHelper.FilterCompatibleSources(Constants.DefaultPlatform, frameworkNet45, sourceArchitectures, sourceFrameworks, out warningMessage);
// only "x86net45.dll" is the compatible source
Assert.AreEqual(1, compatibleSources.Count());
Assert.AreEqual(expected, warningMessage);
}
public void UpdateTargetDeviceValueFromOldMsTestSettings()
{
var settings = @"<RunSettings>
<RunConfiguration>
<MaxCpuCount>2</MaxCpuCount>
<DisableParallelization>False</DisableParallelization>
<DisableAppDomain>False</DisableAppDomain>
</RunConfiguration>
<MSPhoneTest>
<TargetDevice>169.254.193.190</TargetDevice>
</MSPhoneTest>
</RunSettings>";
var navigator = this.GetNavigator(settings);
var result = InferRunSettingsHelper.TryGetDeviceXml(navigator, out string deviceXml);
Assert.IsTrue(result);
InferRunSettingsHelper.UpdateTargetDevice(navigator, deviceXml);
Assert.AreEqual(deviceXml.ToString(), this.GetValueOf(navigator, "/RunSettings/RunConfiguration/TargetDevice"));
}
public void UpdateRunSettingsShouldNotModifyXmlIfNavigatorIsNotAtRootNode(string settingName)
{
var settings = @"<RunSettings><RunConfiguration></RunConfiguration></RunSettings>";
var navigator = this.GetNavigator(settings);
navigator.MoveToFirstChild();
switch (settingName.ToUpperInvariant())
{
case "DESIGNMODE":
InferRunSettingsHelper.UpdateDesignMode(navigator, true);
break;
case "COLLECTSOURCEINFORMATION":
InferRunSettingsHelper.UpdateCollectSourceInformation(navigator, true);
break;
}
;
navigator.MoveToRoot();
Assert.IsTrue(navigator.InnerXml.IndexOf(settingName, StringComparison.OrdinalIgnoreCase) < 0);
}
private bool IsAutoFrameworkDetectRequired(XPathNavigator navigator, out Framework chosenFramework)
{
bool required = true;
chosenFramework = null;
if (commandLineOptions.IsDesignMode)
{
bool isValidFx =
InferRunSettingsHelper.TryGetFrameworkXml(navigator, out var frameworkFromrunsettingsXml);
required = !isValidFx || string.IsNullOrWhiteSpace(frameworkFromrunsettingsXml);
if (!required)
{
chosenFramework = Framework.FromString(frameworkFromrunsettingsXml);
}
}
else if (!commandLineOptions.IsDesignMode && commandLineOptions.FrameworkVersionSpecified)
{
required = false;
chosenFramework = commandLineOptions.TargetFrameworkVersion;
}
return(required);
}
private bool UpdateFramework(XmlDocument document, XPathNavigator navigator, List <string> sources, IDictionary <string, Framework> sourceFrameworks, IBaseTestEventsRegistrar registrar, out Framework chosenFramework)
{
// Get framework from sources
var inferedFramework = inferHelper.AutoDetectFramework(sources, sourceFrameworks);
// Get framework from runsettings.
bool updateFramework = IsAutoFrameworkDetectRequired(navigator, out chosenFramework);
// Update framework if required. For command line scenario update happens in ArgumentProcessor.
if (updateFramework)
{
InferRunSettingsHelper.UpdateTargetFramework(document, inferedFramework?.ToString(), overwrite: true);
chosenFramework = inferedFramework;
}
// Raise warnings for unsupported frameworks.
if (Constants.DotNetFramework35.Equals(chosenFramework.Name))
{
EqtTrace.Warning("TestRequestManager.UpdateRunSettingsIfRequired: throw warning on /Framework:Framework35 option.");
registrar.LogWarning(Resources.Framework35NotSupported);
}
return(updateFramework);
}
private bool UpdateRunSettingsIfRequired(string runsettingsXml, List <string> sources, out string updatedRunSettingsXml)
{
bool settingsUpdated = false;
updatedRunSettingsXml = runsettingsXml;
IDictionary <string, Architecture> sourcePlatforms = new Dictionary <string, Architecture>();
IDictionary <string, Framework> sourceFrameworks = new Dictionary <string, Framework>();
if (!string.IsNullOrEmpty(runsettingsXml))
{
// TargetFramework is full CLR. Set DesignMode based on current context.
using (var stream = new StringReader(runsettingsXml))
using (var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings))
{
var document = new XmlDocument();
document.Load(reader);
var navigator = document.CreateNavigator();
var inferedFramework = inferHelper.AutoDetectFramework(sources, sourceFrameworks);
Framework chosenFramework;
var inferedPlatform = inferHelper.AutoDetectArchitecture(sources, sourcePlatforms);
Architecture chosenPlatform;
// Update frmaework and platform if required. For commandline scenario update happens in ArgumentProcessor.
bool updateFramework = IsAutoFrameworkDetectRequired(navigator, out chosenFramework);
bool updatePlatform = IsAutoPlatformDetectRequired(navigator, out chosenPlatform);
if (updateFramework)
{
InferRunSettingsHelper.UpdateTargetFramework(document, inferedFramework?.ToString(), overwrite: true);
chosenFramework = inferedFramework;
settingsUpdated = true;
}
if (updatePlatform)
{
InferRunSettingsHelper.UpdateTargetPlatform(document, inferedPlatform.ToString(), overwrite: true);
chosenPlatform = inferedPlatform;
settingsUpdated = true;
}
var compatibleSources = InferRunSettingsHelper.FilterCompatibleSources(chosenPlatform, chosenFramework, sourcePlatforms, sourceFrameworks, out var incompatibleSettingWarning);
if (!string.IsNullOrEmpty(incompatibleSettingWarning))
{
EqtTrace.Info(incompatibleSettingWarning);
ConsoleLogger.RaiseTestRunWarning(incompatibleSettingWarning);
}
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("Compatible sources list : ");
EqtTrace.Info(string.Join("\n", compatibleSources.ToArray()));
}
// If user is already setting DesignMode via runsettings or CLI args; we skip.
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettingsXml);
if (!runConfiguration.DesignModeSet)
{
InferRunSettingsHelper.UpdateDesignMode(document, this.commandLineOptions.IsDesignMode);
settingsUpdated = true;
}
if (!runConfiguration.CollectSourceInformationSet)
{
InferRunSettingsHelper.UpdateCollectSourceInformation(document, this.commandLineOptions.ShouldCollectSourceInformation);
settingsUpdated = true;
}
if (InferRunSettingsHelper.TryGetDeviceXml(navigator, out string deviceXml))
{
InferRunSettingsHelper.UpdateTargetDevice(document, deviceXml);
settingsUpdated = true;
}
var designMode = runConfiguration.DesignModeSet ?
runConfiguration.DesignMode :
this.commandLineOptions.IsDesignMode;
// Add or update console logger.
if (!designMode)
{
AddOrUpdateConsoleLogger(document, runsettingsXml);
settingsUpdated = true;
}
else
{
settingsUpdated = UpdateConsoleLoggerIfExists(document, runsettingsXml) || settingsUpdated;
}
updatedRunSettingsXml = navigator.OuterXml;
}
}
return(settingsUpdated);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public void RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (testRunRequestPayload.TestPlatformOptions != null)
{
this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
// Get sources to auto detect fx and arch for both run selected or run all scenario.
var sources = GetSources(testRunRequestPayload);
if (this.UpdateRunSettingsIfRequired(runsettings, sources, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsInCompatibleWithTestSettings(runsettings))
{
throw new SettingsException(string.Format(Resources.RunsettingsWithDCErrorMessage, runsettings));
}
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
var batchSize = runConfiguration.BatchSize;
if (requestData.IsTelemetryOptedIn)
{
// Collect Metrics
this.CollectMetrics(requestData, runConfiguration);
// Collect Commands
this.LogCommandsTelemetryPoints(requestData);
// Collect data for Legacy Settings
this.LogTelemetryForLegacySettings(requestData, runsettings);
}
if (!commandLineOptions.IsDesignMode)
{
// Generate fakes settings only for command line scenarios. In case of
// Editors/IDEs, this responsibility is with the caller.
GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings);
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestPlatformOptions?.TestCaseFilter,
testRunRequestPayload.TestPlatformOptions?.FilterOptions);
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
// Run tests
try
{
this.RunTests(requestData, runCriteria, testRunEventsRegistrar, testRunRequestPayload.TestPlatformOptions);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed.");
}
finally
{
this.testPlatformEventSource.ExecutionRequestStop();
// Post the run complete event
this.metricsPublisher.Result.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics);
}
}
/// <inheritdoc />
public void RunTests(
TestRunRequestPayload testRunRequestPayload,
ITestHostLauncher testHostLauncher,
ITestRunEventsRegistrar testRunEventsRegistrar,
ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (testRunRequestPayload.TestPlatformOptions != null)
{
this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
// Get sources to auto detect fx and arch for both run selected or run all scenario.
var sources = GetSources(testRunRequestPayload);
if (this.UpdateRunSettingsIfRequired(
runsettings,
sources,
testRunEventsRegistrar,
out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(runsettings))
{
throw new SettingsException(
string.Format(
Resources.RunsettingsWithDCErrorMessage,
runsettings));
}
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
var batchSize = runConfiguration.BatchSize;
if (requestData.IsTelemetryOptedIn)
{
// Collect metrics.
this.CollectMetrics(requestData, runConfiguration);
// Collect commands.
this.LogCommandsTelemetryPoints(requestData);
// Collect data for legacy settings.
this.LogTelemetryForLegacySettings(requestData, runsettings);
}
// Get Fakes data collector settings.
if (!string.Equals(Environment.GetEnvironmentVariable("VSTEST_SKIP_FAKES_CONFIGURATION"), "1"))
{
// The commandline options do not have sources in design time mode,
// and so we fall back to using sources instead.
if (this.commandLineOptions.Sources.Any())
{
GenerateFakesUtilities.GenerateFakesSettings(
this.commandLineOptions,
this.commandLineOptions.Sources.ToList(),
ref runsettings);
}
else if (sources.Any())
{
GenerateFakesUtilities.GenerateFakesSettings(
this.commandLineOptions,
sources,
ref runsettings);
}
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestPlatformOptions?.TestCaseFilter,
testRunRequestPayload.TestPlatformOptions?.FilterOptions,
testRunRequestPayload.TestSessionInfo,
debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null &&
testRunRequestPayload.DebuggingEnabled);
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestSessionInfo,
debugEnabledForTestSession: testRunRequestPayload.TestSessionInfo != null &&
testRunRequestPayload.DebuggingEnabled);
}
// Run tests.
try
{
this.RunTests(
requestData,
runCriteria,
testRunEventsRegistrar,
testRunRequestPayload.TestPlatformOptions);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed.");
}
finally
{
this.TestPlatformEventSourceInstance.ExecutionRequestStop();
// Post the run complete event
this.metricsPublisher.Result.PublishMetrics(
TelemetryDataConstants.TestExecutionCompleteEvent,
requestData.MetricsCollection.Metrics);
}
}
/// <inheritdoc/>
public void StartTestSession(
StartTestSessionPayload payload,
ITestHostLauncher testHostLauncher,
ITestSessionEventsHandler eventsHandler,
ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.StartTestSession: Starting test session.");
if (payload.TestPlatformOptions != null)
{
this.telemetryOptedIn = payload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
if (this.UpdateRunSettingsIfRequired(
payload.RunSettings,
payload.Sources,
null,
out string updatedRunsettings))
{
payload.RunSettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsIncompatibleWithTestSettings(payload.RunSettings))
{
throw new SettingsException(
string.Format(
Resources.RunsettingsWithDCErrorMessage,
payload.RunSettings));
}
// TODO (copoiena): Collect metrics ?
lock (this.syncObject)
{
try
{
EqtTrace.Info("TestRequestManager.StartTestRunner: Synchronization context taken.");
this.TestPlatformEventSourceInstance.StartTestSessionStart();
var criteria = new StartTestSessionCriteria()
{
Sources = payload.Sources,
RunSettings = payload.RunSettings,
TestHostLauncher = testHostLauncher
};
this.testPlatform.StartTestSession(requestData, criteria, eventsHandler);
}
finally
{
EqtTrace.Info("TestRequestManager.StartTestSession: Starting test session completed.");
this.TestPlatformEventSourceInstance.StartTestSessionStop();
// Post the attachments processing complete event.
this.metricsPublisher.Result.PublishMetrics(
TelemetryDataConstants.StartTestSessionCompleteEvent,
requestData.MetricsCollection.Metrics);
}
}
}
/// <summary>
/// Ensure that the engine is ready for test operations.
/// Usually includes starting up the test host process.
/// </summary>
/// <param name="sources">
/// List of test sources.
/// </param>
/// <param name="cancellationToken">
/// </param>
/// <returns>
/// Returns true if Communation is established b/w runner and host
/// </returns>
public virtual bool SetupChannel(IEnumerable <string> sources, string runSettings)
{
this.CancellationTokenSource.Token.ThrowTestPlatformExceptionIfCancellationRequested();
var connTimeout = EnvironmentHelper.GetConnectionTimeout();
if (!this.initialized)
{
this.testHostProcessStdError = string.Empty;
TestHostConnectionInfo testHostConnectionInfo = this.testHostManager.GetTestHostConnectionInfo();
var portNumber = 0;
if (testHostConnectionInfo.Role == ConnectionRole.Client)
{
portNumber = this.RequestSender.InitializeCommunication();
testHostConnectionInfo.Endpoint += portNumber;
}
var processId = this.processHelper.GetCurrentProcessId();
var connectionInfo = new TestRunnerConnectionInfo {
Port = portNumber, ConnectionInfo = testHostConnectionInfo, RunnerProcessId = processId, LogFile = this.GetTimestampedLogFile(EqtTrace.LogFile), TraceLevel = (int)EqtTrace.TraceLevel
};
// Subscribe to TestHost Event
this.testHostManager.HostLaunched += this.TestHostManagerHostLaunched;
this.testHostManager.HostExited += this.TestHostManagerHostExited;
// Get envVars from run settings
var envVars = InferRunSettingsHelper.GetEnvironmentVariables(runSettings);
// Get the test process start info
var testHostStartInfo = this.UpdateTestProcessStartInfo(this.testHostManager.GetTestHostProcessStartInfo(sources, envVars, connectionInfo));
try
{
// Launch the test host.
var hostLaunchedTask = this.testHostManager.LaunchTestHostAsync(testHostStartInfo, this.CancellationTokenSource.Token);
this.testHostLaunched = hostLaunchedTask.Result;
if (this.testHostLaunched && testHostConnectionInfo.Role == ConnectionRole.Host)
{
// If test runtime is service host, try to poll for connection as client
this.RequestSender.InitializeCommunication();
}
}
catch (Exception ex)
{
EqtTrace.Error("ProxyOperationManager: Failed to launch testhost :{0}", ex);
this.CancellationTokenSource.Token.ThrowTestPlatformExceptionIfCancellationRequested();
throw new TestPlatformException(string.Format(CultureInfo.CurrentUICulture, CrossPlatEngineResources.FailedToLaunchTestHost, ex.ToString()));
}
// Warn the user that execution will wait for debugger attach.
var hostDebugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_DEBUG");
var nativeHostDebugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_NATIVE_DEBUG");
if (!string.IsNullOrEmpty(hostDebugEnabled) && hostDebugEnabled.Equals("1", StringComparison.Ordinal) ||
new PlatformEnvironment().OperatingSystem.Equals(PlatformOperatingSystem.Windows) &&
!string.IsNullOrEmpty(nativeHostDebugEnabled) && nativeHostDebugEnabled.Equals("1", StringComparison.Ordinal))
{
ConsoleOutput.Instance.WriteLine(CrossPlatEngineResources.HostDebuggerWarning, OutputLevel.Warning);
ConsoleOutput.Instance.WriteLine(
string.Format("Process Id: {0}, Name: {1}", this.testHostProcessId, this.processHelper.GetProcessName(this.testHostProcessId)),
OutputLevel.Information);
// Increase connection timeout when debugging is enabled.
connTimeout *= 5;
}
// If TestHost does not launch then throw exception
// If Testhost launches, wait for connection.
if (!this.testHostLaunched ||
!this.RequestSender.WaitForRequestHandlerConnection(connTimeout * 1000, this.CancellationTokenSource.Token))
{
// Throw a test platform exception with the appropriate message if user requested cancellation
this.CancellationTokenSource.Token.ThrowTestPlatformExceptionIfCancellationRequested();
// Throw a test platform exception along with the error messages from the test if the test host exited unexpectedly
// before communication was established
this.ThrowOnTestHostExited(this.testHostExited.IsSet);
// Throw a test platform exception stating the connection to test could not be established even after waiting
// for the configure timeout period
this.ThrowExceptionOnConnectionFailure(connTimeout);
}
// Handling special case for dotnet core projects with older test hosts
// Older test hosts are not aware of protocol version check
// Hence we should not be sending VersionCheck message to these test hosts
this.CompatIssueWithVersionCheckAndRunsettings();
if (this.versionCheckRequired)
{
this.RequestSender.CheckVersionWithTestHost();
}
this.initialized = true;
}
return(true);
}
/// <summary>
/// Ensures that the engine is ready for test operations. Usually includes starting up the
/// test host process.
/// </summary>
///
/// <param name="sources">List of test sources.</param>
/// <param name="runSettings">Run settings to be used.</param>
///
/// <returns>
/// Returns true if the communication is established b/w runner and host, false otherwise.
/// </returns>
public virtual bool SetupChannel(IEnumerable <string> sources, string runSettings)
{
this.CancellationTokenSource.Token.ThrowTestPlatformExceptionIfCancellationRequested();
//System.Diagnostics.Debugger.Launch();
//System.Diagnostics.Debugger.Break();
if (this.initialized)
{
return(true);
}
var connTimeout = EnvironmentHelper.GetConnectionTimeout();
this.testHostProcessStdError = string.Empty;
TestHostConnectionInfo testHostConnectionInfo = this.TestHostManager.GetTestHostConnectionInfo();
var portNumber = 0;
if (testHostConnectionInfo.Role == ConnectionRole.Client)
{
portNumber = this.RequestSender.InitializeCommunication();
testHostConnectionInfo.Endpoint += portNumber;
}
var processId = this.processHelper.GetCurrentProcessId();
var connectionInfo = new TestRunnerConnectionInfo()
{
Port = portNumber,
ConnectionInfo = testHostConnectionInfo,
RunnerProcessId = processId,
LogFile = this.GetTimestampedLogFile(EqtTrace.LogFile),
TraceLevel = (int)EqtTrace.TraceLevel
};
// Subscribe to test host events.
this.TestHostManager.HostLaunched += this.TestHostManagerHostLaunched;
this.TestHostManager.HostExited += this.TestHostManagerHostExited;
// Get environment variables from run settings.
var envVars = InferRunSettingsHelper.GetEnvironmentVariables(runSettings);
// Get the test process start info.
var hostStartInfo = this.TestHostManager.GetTestHostProcessStartInfo(
sources,
envVars,
connectionInfo);
EqtTrace.Verbose("DefaultTestHostmanager::Received path is: " + hostStartInfo.FileName);
if (!File.Exists(hostStartInfo.FileName))
{
// Somehow can not set path to file in Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Hosting.DefaultTestHostManager::GetTestHostProcessStartInfo()
// :( :( :( Why ?? :(
var fileName = Path.GetFileName(hostStartInfo.FileName);
var directory = Path.GetDirectoryName(hostStartInfo.FileName);
var frameworkMoniker = fileName.Split('.');
EqtTrace.Verbose("DefaultTestHostmanager::[1] directory is: " + directory);
while (directory.Contains("\\TestHost"))
{
directory = directory.Replace("\\TestHost", "");
}
EqtTrace.Verbose("DefaultTestHostmanager::[2] directory is: " + directory);
if (directory.Contains("\\TestsHosts") == false)
{
fileName = Path.Combine(directory, "TestsHosts", frameworkMoniker[1], "win7-x86", fileName);
}
else
{
fileName = Path.Combine(directory, frameworkMoniker[1], "win7-x86", fileName);
}
hostStartInfo.FileName = fileName;
EqtTrace.Verbose("DefaultTestHostmanager::Fixed path to: " + hostStartInfo.FileName);
}
var testHostStartInfo = this.UpdateTestProcessStartInfo(hostStartInfo);
try
{
// Launch the test host.
var hostLaunchedTask = this.TestHostManager.LaunchTestHostAsync(
testHostStartInfo,
this.CancellationTokenSource.Token);
this.testHostLaunched = hostLaunchedTask.Result;
if (this.testHostLaunched && testHostConnectionInfo.Role == ConnectionRole.Host)
{
// If test runtime is service host, try to poll for connection as client.
this.RequestSender.InitializeCommunication();
}
}
catch (Exception ex)
{
EqtTrace.Error("ProxyOperationManager: Failed to launch testhost :{0}", ex);
this.CancellationTokenSource.Token.ThrowTestPlatformExceptionIfCancellationRequested();
throw new TestPlatformException(string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.FailedToLaunchTestHost,
ex.ToString()));
}
// Warn the user that execution will wait for debugger attach.
var hostDebugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_DEBUG");
var nativeHostDebugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_NATIVE_DEBUG");
if (!string.IsNullOrEmpty(hostDebugEnabled) && hostDebugEnabled.Equals("1", StringComparison.Ordinal) ||
new PlatformEnvironment().OperatingSystem.Equals(PlatformOperatingSystem.Windows) &&
!string.IsNullOrEmpty(nativeHostDebugEnabled) && nativeHostDebugEnabled.Equals("1", StringComparison.Ordinal))
{
ConsoleOutput.Instance.WriteLine(
CrossPlatEngineResources.HostDebuggerWarning,
OutputLevel.Warning);
ConsoleOutput.Instance.WriteLine(
string.Format(
"Process Id: {0}, Name: {1}",
this.testHostProcessId,
this.processHelper.GetProcessName(this.testHostProcessId)),
OutputLevel.Information);
// Increase connection timeout when debugging is enabled.
connTimeout *= 5;
}
// If test host does not launch then throw exception, otherwise wait for connection.
if (!this.testHostLaunched ||
!this.RequestSender.WaitForRequestHandlerConnection(
connTimeout * 1000,
this.CancellationTokenSource.Token))
{
EqtTrace.Verbose($"Test host failed to start Test host launched:{testHostLaunched} test host exited: {testHostExited.IsSet}");
// Throw a test platform exception with the appropriate message if user requested cancellation.
this.CancellationTokenSource.Token.ThrowTestPlatformExceptionIfCancellationRequested();
// Throw a test platform exception along with the error messages from the test if the test host exited unexpectedly
// before communication was established.
this.ThrowOnTestHostExited(this.testHostExited.IsSet);
// Throw a test platform exception stating the connection to test could not be established even after waiting
// for the configure timeout period.
this.ThrowExceptionOnConnectionFailure(connTimeout);
}
// Handling special case for dotnet core projects with older test hosts.
// Older test hosts are not aware of protocol version check, hence we should not be
// sending VersionCheck message to these test hosts.
this.CompatIssueWithVersionCheckAndRunsettings();
if (this.versionCheckRequired)
{
this.RequestSender.CheckVersionWithTestHost();
}
this.initialized = true;
return(true);
}
