public void RunTestsIfThrowsInvalidOperationExceptionShouldNotThrowOut()
{
var payload = new TestRunRequestPayload()
{
Sources = new List <string>()
{
"a", "b"
}
};
var createRunRequestCalled = 0;
TestRunCriteria observedCriteria = null;
var mockRunRequest = new Mock <ITestRunRequest>();
this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny <IRequestData>(), It.IsAny <TestRunCriteria>())).Callback(
(IRequestData requestData, TestRunCriteria runCriteria) =>
{
createRunRequestCalled++;
observedCriteria = runCriteria;
}).Returns(mockRunRequest.Object);
mockRunRequest.Setup(mr => mr.ExecuteAsync()).Throws(new InvalidOperationException("HelloWorld"));
var mockRunEventsRegistrar = new Mock <ITestRunEventsRegistrar>();
var mockCustomlauncher = new Mock <ITestHostLauncher>();
var success = this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object, this.protocolConfig);
Assert.IsFalse(success, "RunTests call must fail due to exception");
}
public void RunTestsShouldPassSameProtocolConfigInRequestData()
{
var payload = new TestRunRequestPayload()
{
Sources = new List <string>()
{
"a"
},
};
var mockProtocolConfig = new ProtocolConfig {
Version = 4
};
IRequestData actualRequestData = null;
var mockDiscoveryRequest = new Mock <ITestRunRequest>();
this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny <IRequestData>(), It.IsAny <TestRunCriteria>())).Callback(
(IRequestData requestData, TestRunCriteria runCriteria) =>
{
actualRequestData = requestData;
}).Returns(mockDiscoveryRequest.Object);
// Act.
this.testRequestManager.RunTests(payload, new Mock <ITestHostLauncher>().Object, new Mock <ITestRunEventsRegistrar>().Object, mockProtocolConfig);
// Verify.
Assert.AreEqual(4, actualRequestData.ProtocolConfig.Version);
}
private void StartTestRun(TestRunRequestPayload testRunPayload, ITestRequestManager testRequestManager, bool skipTestHostLaunch)
{
Task.Run(
delegate
{
try
{
testRequestManager.ResetOptions();
var customLauncher = skipTestHostLaunch ?
DesignModeTestHostLauncherFactory.GetCustomHostLauncherForTestRun(this, testRunPayload) : null;
testRequestManager.RunTests(testRunPayload, customLauncher, new DesignModeTestEventsRegistrar(this), this.protocolConfig);
}
catch (Exception ex)
{
EqtTrace.Error("DesignModeClient: Exception in StartTestRun: " + ex);
// If there is an exception during test run request creation or some time during the process
// In such cases, TestPlatform will never send a TestRunComplete event and IDE need to be sent a run complete message
// We need recoverability in translationlayer-designmode scenarios
var testMessagePayload = new TestMessagePayload {
MessageLevel = TestMessageLevel.Error, Message = ex.ToString()
};
this.communicationManager.SendMessage(MessageType.TestMessage, testMessagePayload);
var runCompletePayload = new TestRunCompletePayload()
{
TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, true, ex, null, TimeSpan.MinValue),
LastRunTests = null
};
// Send run complete to translation layer
this.communicationManager.SendMessage(MessageType.ExecutionComplete, runCompletePayload);
}
});
}
private void RunTests(string runSettings)
{
// create/start test run
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("RunTestsArgumentProcessor:Execute: Test run is starting.");
}
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("RunTestsArgumentProcessor:Execute: Queuing Test run.");
}
// for command line keep alive is always false.
// for Windows Store apps it should be false, as Windows Store apps executor should terminate after finishing the test execution.
var keepAlive = false;
var runRequestPayload = new TestRunRequestPayload()
{
Sources = this.commandLineOptions.Sources.ToList(), RunSettings = runSettings, KeepAlive = keepAlive, TestPlatformOptions = new TestPlatformOptions()
{
TestCaseFilter = this.commandLineOptions.TestCaseFilterValue
}
};
this.testRequestManager.RunTests(runRequestPayload, null, this.testRunEventsRegistrar, Constants.DefaultProtocolConfig);
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("RunTestsArgumentProcessor:Execute: Test run is completed.");
}
}
private bool RunTests(IEnumerable <string> sources)
{
// create/start test run
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("RunTestsArgumentProcessor:Execute: Test run is starting.");
}
var runSettings = this.runSettingsManager.ActiveRunSettings.SettingsXml;
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("RunTestsArgumentProcessor:Execute: Queuing Test run.");
}
// for command line keep alive is always false.
// for Windows Store apps it should be false, as Windows Store apps executor should terminate after finishing the test execution.
var keepAlive = false;
GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runSettings);
var runRequestPayload = new TestRunRequestPayload()
{
Sources = this.commandLineOptions.Sources.ToList(), RunSettings = runSettings, KeepAlive = keepAlive, TestCaseFilter = this.commandLineOptions.TestCaseFilterValue
};
var result = this.testRequestManager.RunTests(runRequestPayload, null, this.testRunEventsRegistrar, Constants.DefaultProtocolConfig);
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("RunTestsArgumentProcessor:Execute: Test run is completed.");
}
return(result);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar)
{
TestRunCriteria runCriteria = null;
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
this.commandLineOptions.BatchSize,
testRunRequestPayload.KeepAlive,
testRunRequestPayload.RunSettings,
this.commandLineOptions.TestRunStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
this.commandLineOptions.BatchSize,
testRunRequestPayload.KeepAlive,
testRunRequestPayload.RunSettings,
this.commandLineOptions.TestRunStatsEventTimeout,
testHostLauncher);
}
return(this.RunTests(runCriteria, testRunEventsRegistrar));
}
public void RunTestsIfThrowsExceptionShouldThrowOut()
{
var payload = new TestRunRequestPayload()
{
Sources = new List <string>()
{
"a", "b"
}
};
var createRunRequestCalled = 0;
TestRunCriteria observedCriteria = null;
var mockRunRequest = new Mock <ITestRunRequest>();
this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny <TestRunCriteria>())).Callback <TestRunCriteria>(
(runCriteria) =>
{
createRunRequestCalled++;
observedCriteria = runCriteria;
}).Returns(mockRunRequest.Object);
mockRunRequest.Setup(mr => mr.ExecuteAsync()).Throws(new NotImplementedException("HelloWorld"));
var mockRunEventsRegistrar = new Mock <ITestRunEventsRegistrar>();
var mockCustomlauncher = new Mock <ITestHostLauncher>();
this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object);
}
public void RunTestsShouldReadTheBatchSizeFromSettingsAndSetItForTestRunCriteria()
{
var payload = new TestRunRequestPayload()
{
Sources = new List <string>()
{
"a"
},
RunSettings =
@"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<BatchSize>15</BatchSize>
</RunConfiguration>
</RunSettings>"
};
TestRunCriteria actualTestRunCriteria = null;
var mockDiscoveryRequest = new Mock <ITestRunRequest>();
this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny <TestRunCriteria>())).Callback <TestRunCriteria>(
(criteria) =>
{
actualTestRunCriteria = criteria;
}).Returns(mockDiscoveryRequest.Object);
var success = this.testRequestManager.RunTests(payload, new Mock <ITestHostLauncher>().Object, new Mock <ITestRunEventsRegistrar>().Object);
Assert.AreEqual(15, actualTestRunCriteria.FrequencyOfRunStatsChangeEvent);
}
public void DesignModeClientConnectShouldSendTestMessageAndExecutionCompleteOnTestPlatformExceptionInTestRun()
{
var payload = new TestRunRequestPayload();
var testRunAll = new Message {
MessageType = MessageType.TestRunAllSourcesWithDefaultHost, Payload = JToken.FromObject(payload)
};
this.mockCommunicationManager.Setup(cm => cm.WaitForServerConnection(It.IsAny <int>())).Returns(true);
this.mockCommunicationManager.SetupSequence(cm => cm.ReceiveMessage()).Returns(testRunAll);
this.mockCommunicationManager
.Setup(cm => cm.SendMessage(MessageType.ExecutionComplete, It.IsAny <TestRunCompletePayload>()))
.Callback(() => this.complateEvent.Set());
this.mockTestRequestManager.Setup(
rm => rm.RunTests(
It.IsAny <TestRunRequestPayload>(),
null,
It.IsAny <DesignModeTestEventsRegistrar>(),
It.IsAny <ProtocolConfig>())).Throws(new TestPlatformException("Hello world"));
this.designModeClient.ConnectToClientAndProcessRequests(PortNumber, this.mockTestRequestManager.Object);
Assert.IsTrue(this.complateEvent.WaitOne(Timeout), "Execution not completed.");
this.mockCommunicationManager.Verify(cm => cm.SendMessage(MessageType.TestMessage, It.IsAny <TestMessagePayload>()), Times.Once());
this.mockCommunicationManager.Verify(cm => cm.SendMessage(MessageType.ExecutionComplete, It.IsAny <TestRunCompletePayload>()), Times.Once());
}
private void StartTestRun(TestRunRequestPayload testRunPayload, ITestRequestManager testRequestManager, bool skipTestHostLaunch)
{
Task.Run(
delegate
{
try
{
testRequestManager.ResetOptions();
var customLauncher = skipTestHostLaunch ?
DesignModeTestHostLauncherFactory.GetCustomHostLauncherForTestRun(this, testRunPayload) : null;
testRequestManager.RunTests(testRunPayload, customLauncher, new DesignModeTestEventsRegistrar(this), this.protocolConfig);
}
catch (Exception ex)
{
EqtTrace.Error("DesignModeClient: Exception in StartTestRun: " + ex);
var testMessagePayload = new TestMessagePayload {
MessageLevel = TestMessageLevel.Error, Message = ex.ToString()
};
this.communicationManager.SendMessage(MessageType.TestMessage, testMessagePayload);
var runCompletePayload = new TestRunCompletePayload()
{
TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, true, ex, null, TimeSpan.MinValue),
LastRunTests = null
};
// Send run complete to translation layer
this.communicationManager.SendMessage(MessageType.ExecutionComplete, runCompletePayload);
}
});
}
void GetProcessStartInfo(IEnumerable <string> testAssemblies)
{
var message = new TestRunRequestPayload {
Sources = testAssemblies.ToList(),
RunSettings = null
};
communicationManager.SendMessage(MessageType.GetTestRunnerProcessStartInfoForRunAll, message);
}
void RunTests(IEnumerable <string> testAssemblies)
{
var message = new TestRunRequestPayload {
Sources = testAssemblies.ToList(),
RunSettings = null
};
communicationManager.SendMessage(MessageType.TestRunAllSourcesWithDefaultHost, message);
}
void GetProcessStartInfo(Project project, IEnumerable <TestCase> testCases)
{
var message = new TestRunRequestPayload {
TestCases = testCases.ToList(),
RunSettings = GetRunSettings(project)
};
communicationManager.SendMessage(MessageType.GetTestRunnerProcessStartInfoForRunSelected, message);
}
void GetProcessStartInfo(Project project)
{
var message = new TestRunRequestPayload {
Sources = new List <string> (new [] { GetAssemblyFileName(project) }),
RunSettings = GetRunSettings(project)
};
communicationManager.SendMessage(MessageType.GetTestRunnerProcessStartInfoForRunAll, message);
}
void RunTests(Project project)
{
var message = new TestRunRequestPayload {
Sources = new List <string> (new [] { GetAssemblyFileName(project) }),
RunSettings = GetRunSettings(project)
};
communicationManager.SendMessage(MessageType.TestRunAllSourcesWithDefaultHost, message);
}
void RunTests(Project project, IEnumerable <TestCase> testCases)
{
var message = new TestRunRequestPayload {
TestCases = testCases.ToList(),
RunSettings = GetRunSettings(project)
};
communicationManager.SendMessage(MessageType.TestRunSelectedTestCasesDefaultHost, message);
}
public void DesignModeTestHostFactoryShouldReturnDebugLauncherIfDebuggingEnabled()
{
var mockDesignModeClient = new Mock <IDesignModeClient>();
var testRunRequestPayload = new TestRunRequestPayload {
DebuggingEnabled = true
};
var launcher = DesignModeTestHostLauncherFactory.GetCustomHostLauncherForTestRun(mockDesignModeClient.Object, testRunRequestPayload);
Assert.IsTrue(launcher.IsDebug, "Factory must return non-debug launcher if debugging is enabled.");
}
public void RunTestsWithSourcesShouldCallTestPlatformAndSucceed()
{
var payload = new TestRunRequestPayload()
{
Sources = new List <string>()
{
"a", "b"
}
};
var createRunRequestCalled = 0;
TestRunCriteria observedCriteria = null;
var mockRunRequest = new Mock <ITestRunRequest>();
this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny <TestRunCriteria>())).Callback <TestRunCriteria>(
(runCriteria) =>
{
createRunRequestCalled++;
observedCriteria = runCriteria;
}).Returns(mockRunRequest.Object);
var mockRunEventsRegistrar = new Mock <ITestRunEventsRegistrar>();
var mockCustomlauncher = new Mock <ITestHostLauncher>();
string testCaseFilterValue = "TestFilter";
CommandLineOptions.Instance.TestCaseFilterValue = testCaseFilterValue;
this.testRequestManager = new TestRequestManager(CommandLineOptions.Instance,
this.mockTestPlatform.Object,
TestLoggerManager.Instance,
TestRunResultAggregator.Instance,
this.mockTestPlatformEventSource.Object);
var success = this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object);
Assert.IsTrue(success, "RunTests call must succeed");
Assert.AreEqual(testCaseFilterValue, observedCriteria.TestCaseFilter, "TestCaseFilter must be set");
Assert.AreEqual(createRunRequestCalled, 1, "CreateRunRequest must be invoked only once.");
Assert.AreEqual(2, observedCriteria.Sources.Count(), "All Sources must be used for discovery request");
Assert.AreEqual("a", observedCriteria.Sources.First(), "First Source in list is incorrect");
Assert.AreEqual("b", observedCriteria.Sources.ElementAt(1), "Second Source in list is incorrect");
// Check for the default value for the frequency
Assert.AreEqual(10, observedCriteria.FrequencyOfRunStatsChangeEvent);
mockRunEventsRegistrar.Verify(md => md.RegisterTestRunEvents(It.IsAny <ITestRunRequest>()), Times.Once);
mockRunEventsRegistrar.Verify(md => md.UnregisterTestRunEvents(It.IsAny <ITestRunRequest>()), Times.Once);
mockRunRequest.Verify(md => md.ExecuteAsync(), Times.Once);
mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStart(), Times.Once);
mockTestPlatformEventSource.Verify(mt => mt.ExecutionRequestStop(), Times.Once);
}
/// <summary>
/// Executes the selected tests
/// </summary>
private bool ExecuteSelectedTests()
{
bool result = true;
if (this.selectedTestCases.Count > 0)
{
if (this.undiscoveredFilters.Count() != 0)
{
string missingFilters = string.Join(", ", this.undiscoveredFilters);
string warningMessage = string.Format(CultureInfo.CurrentCulture, CommandLineResources.SomeTestsUnavailableAfterFiltering, this.discoveredTestCount, missingFilters);
this.output.Warning(false, warningMessage);
}
// for command line keep alive is always false.
bool keepAlive = false;
EqtTrace.Verbose("RunSpecificTestsArgumentProcessor:Execute: Test run is queued.");
var runRequestPayload = new TestRunRequestPayload()
{
TestCases = this.selectedTestCases.ToList(), RunSettings = this.effectiveRunSettings, KeepAlive = keepAlive, TestPlatformOptions = new TestPlatformOptions()
{
TestCaseFilter = this.commandLineOptions.TestCaseFilterValue
}
};
result &= this.testRequestManager.RunTests(runRequestPayload, null, null, Constants.DefaultProtocolConfig);
}
else
{
string warningMessage;
if (this.discoveredTestCount > 0)
{
// No tests that matched any of the given strings.
warningMessage = string.Format(CultureInfo.CurrentCulture, CommandLineResources.NoTestsAvailableAfterFiltering, this.discoveredTestCount, string.Join(", ", this.selectedTestNames));
}
else
{
// No tests were discovered from the given sources.
warningMessage = string.Format(CultureInfo.CurrentUICulture, CommandLineResources.NoTestsAvailableInSources, string.Join(", ", this.commandLineOptions.Sources));
if (string.IsNullOrEmpty(this.commandLineOptions.TestAdapterPath))
{
warningMessage = string.Format(CultureInfo.CurrentCulture, CommandLineResources.StringFormatToJoinTwoStrings, warningMessage, CommandLineResources.SuggestTestAdapterPathIfNoTestsIsFound);
}
}
this.output.Warning(false, warningMessage);
}
return(result);
}
/// <summary>
/// Executes the selected tests
/// </summary>
private bool ExecuteSelectedTests()
{
bool result = true;
if (this.selectedTestCases.Count > 0)
{
if (this.undiscoveredFilters.Count() != 0)
{
string missingFilters = string.Join(", ", this.undiscoveredFilters);
string warningMessage = string.Format(CultureInfo.CurrentCulture, CommandLineResources.SomeTestsUnavailableAfterFiltering, this.discoveredTestCount, missingFilters);
this.output.Warning(warningMessage);
}
// for command line keep alive is always false.
bool keepAlive = false;
GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref this.effectiveRunSettings);
EqtTrace.Verbose("RunSpecificTestsArgumentProcessor:Execute: Test run is queued.");
var runRequestPayload = new TestRunRequestPayload()
{
TestCases = this.selectedTestCases.ToList(), RunSettings = this.effectiveRunSettings, KeepAlive = keepAlive
};
result &= this.testRequestManager.RunTests(runRequestPayload, null, null, Constants.DefaultProtocolConfig);
}
else
{
string warningMessage;
if (this.discoveredTestCount > 0)
{
// No tests that matched any of the given strings.
warningMessage = string.Format(CultureInfo.CurrentCulture, CommandLineResources.NoTestsAvailableAfterFiltering, this.discoveredTestCount, string.Join(", ", this.selectedTestNames));
}
else
{
// No tests were discovered from the given sources.
warningMessage = string.Format(CultureInfo.CurrentUICulture, CommandLineResources.NoTestsAvailableInSources, string.Join(", ", this.commandLineOptions.Sources));
if (!this.commandLineOptions.UseVsixExtensions)
{
warningMessage = string.Format(CultureInfo.CurrentCulture, CommandLineResources.NoTestsFoundWarningMessageWithSuggestionToUseVsix, warningMessage, CommandLineResources.SuggestUseVsixExtensionsIfNoTestsIsFound);
}
}
this.output.Warning(warningMessage);
}
return(result);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings);
var batchSize = runConfiguration.BatchSize;
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
var requestData = this.GetRequestData(protocolConfig);
var metricsPublisher = this.telemetryOptedIn ? (IMetricsPublisher) new MetricsPublisher() : new NoOpMetricsPublisher();
if (this.UpdateRunSettingsIfRequired(runsettings, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = testRunRequestPayload.TestCaseFilter;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(requestData, runCriteria, testRunEventsRegistrar, protocolConfig);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
this.testPlatformEventSource.ExecutionRequestStop();
metricsPublisher.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics);
metricsPublisher.Dispose();
return(success);
}
public void AbortTestRunShouldWaitForCreateTestRunRequest()
{
var payload = new TestRunRequestPayload()
{
Sources = new List <string>()
{
"a", "b"
}
};
TestRunCriteria observedCriteria = null;
var sw = new Stopwatch();
sw.Start();
long createRunRequestTime = 0;
long cancelRequestTime = 0;
var mockRunRequest = new Mock <ITestRunRequest>();
this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny <TestRunCriteria>())).Callback <TestRunCriteria>(
(runCriteria) =>
{
Thread.Sleep(1);
createRunRequestTime = sw.ElapsedMilliseconds;
observedCriteria = runCriteria;
}).Returns(mockRunRequest.Object);
mockRunRequest.Setup(mr => mr.Abort()).Callback(() =>
{
Thread.Sleep(1);
cancelRequestTime = sw.ElapsedMilliseconds;
});
var mockRunEventsRegistrar = new Mock <ITestRunEventsRegistrar>();
var mockCustomlauncher = new Mock <ITestHostLauncher>();
var cancelTask = Task.Run(() => this.testRequestManager.AbortTestRun());
var runTask = Task.Run(() => this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object));
Task.WaitAll(cancelTask, runTask);
Assert.IsTrue(cancelRequestTime > createRunRequestTime, "CancelRequest must execute after create run request");
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings);
var batchSize = runConfiguration.BatchSize;
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (this.UpdateRunSettingsIfRequired(runsettings, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
runsettings = UpdateExtensionsFolderInRunSettings(runsettings);
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(runCriteria, testRunEventsRegistrar, protocolConfig);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
this.testPlatformEventSource.ExecutionRequestStop();
return(success);
}
private List <String> GetSources(TestRunRequestPayload testRunRequestPayload)
{
List <string> sources = new List <string>();
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Count > 0)
{
sources = testRunRequestPayload.Sources;
}
else if (testRunRequestPayload.TestCases != null && testRunRequestPayload.TestCases.Count > 0)
{
ISet <string> sourcesSet = new HashSet <string>();
foreach (var testCase in testRunRequestPayload.TestCases)
{
sourcesSet.Add(testCase.Source);
}
sources = sourcesSet.ToList();
}
return(sources);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings);
var batchSize = runConfiguration.BatchSize;
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (this.TryUpdateDesignMode(runsettings, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(runCriteria, testRunEventsRegistrar);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
return(success);
}
public void RunTestsShouldUpdateDesignModeIfRunnerIsInDesignMode(bool designModeValue)
{
var runsettings =
"<RunSettings><RunConfiguration><TargetFrameworkVersion>.NETFramework,Version=v4.5</TargetFrameworkVersion></RunConfiguration></RunSettings>";
var payload = new TestRunRequestPayload
{
RunSettings = runsettings,
Sources = new List <string> {
"c:\\testproject.dll"
}
};
this.commandLineOptions.IsDesignMode = designModeValue;
this.testRequestManager.RunTests(payload, new Mock <ITestHostLauncher>().Object, new Mock <ITestRunEventsRegistrar>().Object);
var designmode = $"<DesignMode>{designModeValue}</DesignMode>";
this.mockTestPlatform.Verify(tp => tp.CreateTestRunRequest(It.Is <TestRunCriteria>(rc => rc.TestRunSettings.Contains(designmode))));
}
private void StartTestRun(TestRunRequestPayload testRunPayload, ITestRequestManager testRequestManager, bool shouldLaunchTesthost)
{
Task.Run(
() =>
{
try
{
testRequestManager.ResetOptions();
// We must avoid re-launching the test host if the test run payload already
// contains test session info. Test session info being present is an indicative
// of an already running test host spawned by a start test session call.
var customLauncher =
shouldLaunchTesthost && testRunPayload.TestSessionInfo == null
? DesignModeTestHostLauncherFactory.GetCustomHostLauncherForTestRun(
this,
testRunPayload.DebuggingEnabled)
: null;
testRequestManager.RunTests(testRunPayload, customLauncher, new DesignModeTestEventsRegistrar(this), this.protocolConfig);
}
catch (Exception ex)
{
EqtTrace.Error("DesignModeClient: Exception in StartTestRun: " + ex);
var testMessagePayload = new TestMessagePayload {
MessageLevel = TestMessageLevel.Error, Message = ex.ToString()
};
this.communicationManager.SendMessage(MessageType.TestMessage, testMessagePayload);
var runCompletePayload = new TestRunCompletePayload()
{
TestRunCompleteArgs = new TestRunCompleteEventArgs(null, false, true, ex, null, TimeSpan.MinValue),
LastRunTests = null
};
// Send run complete to translation layer
this.communicationManager.SendMessage(MessageType.ExecutionComplete, runCompletePayload);
}
});
}
public void DesignModeClientWithGetTestRunnerProcessStartInfoShouldDeserializeTestsWithTraitsCorrectly()
{
// Arrange.
var testCase = new TestCase("A.C.M", new Uri("d:\\executor"), "A.dll");
testCase.Traits.Add(new Trait("foo", "bar"));
var testList = new System.Collections.Generic.List <TestCase> {
testCase
};
var testRunPayload = new TestRunRequestPayload {
RunSettings = null, TestCases = testList
};
var getProcessStartInfoMessage = new Message
{
MessageType = MessageType.GetTestRunnerProcessStartInfoForRunSelected,
Payload = JToken.FromObject("random")
};
var sessionEnd = new Message {
MessageType = MessageType.SessionEnd
};
TestRunRequestPayload receivedTestRunPayload = null;
var allTasksComplete = new ManualResetEvent(false);
// Setup mocks.
this.mockCommunicationManager.Setup(cm => cm.WaitForServerConnection(It.IsAny <int>())).Returns(true);
this.mockCommunicationManager.Setup(cm => cm.DeserializePayload <TestRunRequestPayload>(getProcessStartInfoMessage))
.Returns(testRunPayload);
this.mockTestRequestManager.Setup(
trm =>
trm.RunTests(
It.IsAny <TestRunRequestPayload>(),
It.IsAny <ITestHostLauncher>(),
It.IsAny <ITestRunEventsRegistrar>()))
.Callback(
(TestRunRequestPayload trp,
ITestHostLauncher testHostManager,
ITestRunEventsRegistrar testRunEventsRegistrar) =>
{
allTasksComplete.Set();
receivedTestRunPayload = trp;
});
this.mockCommunicationManager.SetupSequence(cm => cm.ReceiveMessage())
.Returns(getProcessStartInfoMessage)
.Returns(sessionEnd);
// Act.
this.designModeClient.ConnectToClientAndProcessRequests(0, this.mockTestRequestManager.Object);
// wait for the internal spawned of tasks to complete.
allTasksComplete.WaitOne(1000);
// Assert.
Assert.IsNotNull(receivedTestRunPayload);
Assert.IsNotNull(receivedTestRunPayload.TestCases);
Assert.AreEqual(1, receivedTestRunPayload.TestCases.Count);
// Validate traits
var traits = receivedTestRunPayload.TestCases.ToArray()[0].Traits;
Assert.AreEqual("foo", traits.ToArray()[0].Name);
Assert.AreEqual("bar", traits.ToArray()[0].Value);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public void RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (testRunRequestPayload.TestPlatformOptions != null)
{
this.telemetryOptedIn = testRunRequestPayload.TestPlatformOptions.CollectMetrics;
}
var requestData = this.GetRequestData(protocolConfig);
// Get sources to auto detect fx and arch for both run selected or run all scenario.
var sources = GetSources(testRunRequestPayload);
if (this.UpdateRunSettingsIfRequired(runsettings, sources, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (InferRunSettingsHelper.AreRunSettingsCollectorsInCompatibleWithTestSettings(runsettings))
{
throw new SettingsException(string.Format(Resources.RunsettingsWithDCErrorMessage, runsettings));
}
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettings);
var batchSize = runConfiguration.BatchSize;
if (requestData.IsTelemetryOptedIn)
{
// Collect Metrics
this.CollectMetrics(requestData, runConfiguration);
// Collect Commands
this.LogCommandsTelemetryPoints(requestData);
// Collect data for Legacy Settings
this.LogTelemetryForLegacySettings(requestData, runsettings);
}
if (!commandLineOptions.IsDesignMode)
{
// Generate fakes settings only for command line scenarios. In case of
// Editors/IDEs, this responsibility is with the caller.
GenerateFakesUtilities.GenerateFakesSettings(this.commandLineOptions, this.commandLineOptions.Sources.ToList(), ref runsettings);
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher,
testRunRequestPayload.TestPlatformOptions?.TestCaseFilter,
testRunRequestPayload.TestPlatformOptions?.FilterOptions);
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
// Run tests
try
{
this.RunTests(requestData, runCriteria, testRunEventsRegistrar, testRunRequestPayload.TestPlatformOptions);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed.");
}
finally
{
this.testPlatformEventSource.ExecutionRequestStop();
// Post the run complete event
this.metricsPublisher.Result.PublishMetrics(TelemetryDataConstants.TestExecutionCompleteEvent, requestData.MetricsCollection.Metrics);
}
}
public static ITestHostLauncher GetCustomHostLauncherForTestRun(IDesignModeClient designModeClient, TestRunRequestPayload testRunRequestPayload)
{
ITestHostLauncher testHostLauncher = null;
if (!testRunRequestPayload.DebuggingEnabled)
{
testHostLauncher = defaultLauncher = defaultLauncher ?? new DesignModeTestHostLauncher(designModeClient);
}
else
{
testHostLauncher = debugLauncher = debugLauncher ?? new DesignModeDebugTestHostLauncher(designModeClient);
}
return(testHostLauncher);
}
