private static int GetDistinctNumberOfSources(TestRunCriteria testRunCriteria)
{
// No point in creating more processes if number of sources is less than what user configured for
int numSources = 1;
if (testRunCriteria.HasSpecificTests)
{
numSources = new System.Collections.Generic.HashSet <string>(
testRunCriteria.Tests.Select((testCase) => testCase.Source)).Count;
}
else
{
numSources = testRunCriteria.Sources.Count();
}
return(numSources);
}
public void StartTestRunShouldUpdateTestPlauginCacheWithExtensionsReturnByTestHost()
{
this.mockTestHostManager.Setup(o => o.GetTestPlatformExtensions(It.IsAny <IEnumerable <string> >(), It.IsAny <IEnumerable <string> >())).Returns(new List <string> {
"C:\\dummy.dll"
});
var expectedResult = TestPluginCache.Instance.GetExtensionPaths(string.Empty);
expectedResult.Add("C:\\dummy.dll");
var testRunCriteria = new TestRunCriteria(new List <string> {
"source.dll"
}, 10);
this.inProcessProxyExecutionManager.StartTestRun(testRunCriteria, null);
CollectionAssert.AreEquivalent(expectedResult, TestPluginCache.Instance.GetExtensionPaths(string.Empty));
}
public void GetExecutionManagerShouldReturnParallelExecutionManagerIfHostIsNotShared()
{
string settingXml =
@"<RunSettings>
<RunConfiguration>
<InIsolation>true</InIsolation>
</RunConfiguration >
</RunSettings>";
this.testableTestRuntimeProvider = new TestableRuntimeProvider(false);
var testRunCriteria = new TestRunCriteria(new List <string> {
"1.dll", "2.dll"
}, 100, false, settingXml);
Assert.IsNotNull(this.testEngine.GetExecutionManager(this.mockRequestData.Object, this.testableTestRuntimeProvider, testRunCriteria));
Assert.IsInstanceOfType(this.testEngine.GetExecutionManager(this.mockRequestData.Object, this.testableTestRuntimeProvider, testRunCriteria), typeof(ParallelProxyExecutionManager));
}
private void InvokeCreateTestRunRequest(TestPlatformOptions options = null)
{
this.executionManager.Setup(dm => dm.Initialize(false)).Verifiable();
this.testEngine.Setup(te => te.GetExecutionManager(this.mockRequestData.Object, this.hostManager.Object, It.IsAny <TestRunCriteria>())).Returns(this.executionManager.Object);
this.testEngine.Setup(te => te.GetExtensionManager()).Returns(this.extensionManager.Object);
this.testEngine.Setup(te => te.GetLoggerManager(this.mockRequestData.Object)).Returns(this.loggerManager.Object);
var tp = new TestableTestPlatform(this.testEngine.Object, this.hostManager.Object);
var testRunCriteria = new TestRunCriteria(new List <string> {
"foo"
}, 10);
this.hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources))
.Returns(testRunCriteria.Sources);
tp.CreateTestRunRequest(this.mockRequestData.Object, testRunCriteria, options);
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <param name="protocolConfig">Protocol related information</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings);
var batchSize = runConfiguration.BatchSize;
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (this.UpdateRunSettingsIfRequired(runsettings, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
runsettings = UpdateExtensionsFolderInRunSettings(runsettings);
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(runCriteria, testRunEventsRegistrar, protocolConfig);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
this.testPlatformEventSource.ExecutionRequestStop();
return(success);
}
/// <inheritdoc/>
public ITestRunRequest CreateTestRunRequest(
IRequestData requestData,
TestRunCriteria testRunCriteria,
TestPlatformOptions options)
{
if (testRunCriteria == null)
{
throw new ArgumentNullException(nameof(testRunCriteria));
}
this.AddExtensionAssemblies(testRunCriteria.TestRunSettings);
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings);
// Update extension assemblies from source when design mode is false.
if (!runConfiguration.DesignMode)
{
this.AddExtensionAssembliesFromSource(testRunCriteria);
}
// Initialize loggers.
var loggerManager = this.TestEngine.GetLoggerManager(requestData);
loggerManager.Initialize(testRunCriteria.TestRunSettings);
SubscribeLogger(loggerManager, this.VStestLoggerManager);
var testHostManager = this.testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings);
ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings);
testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings);
// NOTE: The custom launcher should not be set when we have test session info available.
if (testRunCriteria.TestHostLauncher != null)
{
testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher);
}
var executionManager = this.TestEngine.GetExecutionManager(requestData, testHostManager, testRunCriteria);
executionManager.Initialize(options?.SkipDefaultAdapters ?? false);
return(new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager));
}
public void AbortTestRunShouldWaitForCreateTestRunRequest()
{
var payload = new TestRunRequestPayload()
{
Sources = new List <string>()
{
"a", "b"
}
};
TestRunCriteria observedCriteria = null;
var sw = new Stopwatch();
sw.Start();
long createRunRequestTime = 0;
long cancelRequestTime = 0;
var mockRunRequest = new Mock <ITestRunRequest>();
this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny <TestRunCriteria>())).Callback <TestRunCriteria>(
(runCriteria) =>
{
Thread.Sleep(1);
createRunRequestTime = sw.ElapsedMilliseconds;
observedCriteria = runCriteria;
}).Returns(mockRunRequest.Object);
mockRunRequest.Setup(mr => mr.Abort()).Callback(() =>
{
Thread.Sleep(1);
cancelRequestTime = sw.ElapsedMilliseconds;
});
var mockRunEventsRegistrar = new Mock <ITestRunEventsRegistrar>();
var mockCustomlauncher = new Mock <ITestHostLauncher>();
var cancelTask = Task.Run(() => this.testRequestManager.AbortTestRun());
var runTask = Task.Run(() => this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object));
Task.WaitAll(cancelTask, runTask);
Assert.IsTrue(cancelRequestTime > createRunRequestTime, "CancelRequest must execute after create run request");
}
public void LaunchProcessWithDebuggerAttachedShouldNotCallCustomLauncherIfLauncherIsNotDebug()
{
var mockCustomLauncher = new Mock <ITestHostLauncher>();
testRunCriteria = new TestRunCriteria(new List <string> {
"foo"
}, 1, false, null, TimeSpan.Zero, mockCustomLauncher.Object);
executionManager = new Mock <IProxyExecutionManager>();
testRunRequest = new TestRunRequest(testRunCriteria, executionManager.Object);
testRunRequest.ExecuteAsync();
var testProcessStartInfo = new TestProcessStartInfo();
testRunRequest.LaunchProcessWithDebuggerAttached(testProcessStartInfo);
mockCustomLauncher.Verify(ml => ml.LaunchTestHost(It.IsAny <TestProcessStartInfo>()), Times.Never);
}
public void StartTestRunShouldCatchExceptionAndCallHandleRunComplete()
{
var testRunCriteria = new TestRunCriteria(new List <string> {
"source.dll"
}, 10);
var mockTestRunEventsHandler = new Mock <ITestRunEventsHandler>();
var manualResetEvent = new ManualResetEvent(true);
this.mockExecutionManager.Setup(o => o.StartTestRun(testRunCriteria.AdapterSourceMap, null, testRunCriteria.TestRunSettings, It.IsAny <TestExecutionContext>(), null, mockTestRunEventsHandler.Object)).Callback(
() => throw new Exception());
mockTestRunEventsHandler.Setup(o => o.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(), null, null, null)).Callback(
() => manualResetEvent.Set());
this.inProcessProxyExecutionManager.StartTestRun(testRunCriteria, mockTestRunEventsHandler.Object);
Assert.IsTrue(manualResetEvent.WaitOne(5000), "ITestRunEventsHandler.HandleTestRunComplete should get called");
}
public void CreateTestRunRequestShouldInitializeManagersAndCreateTestRunRequestWithSpecifiedCriteria()
{
this.executionManager.Setup(dm => dm.Initialize()).Verifiable();
this.testEngine.Setup(te => te.GetExecutionManager(this.hostManager.Object, It.IsAny <TestRunCriteria>(), It.IsAny <ProtocolConfig>())).Returns(this.executionManager.Object);
this.testEngine.Setup(te => te.GetExtensionManager()).Returns(this.extensionManager.Object);
var tp = new TestableTestPlatform(this.testEngine.Object, this.hostManager.Object);
var testRunCriteria = new TestRunCriteria(new List <string> {
"foo"
}, 10);
var testRunRequest = tp.CreateTestRunRequest(testRunCriteria, It.IsAny <ProtocolConfig>());
var actualTestRunRequest = testRunRequest as TestRunRequest;
this.hostManager.Verify(hm => hm.Initialize(It.IsAny <TestSessionMessageLogger>(), It.IsAny <string>()), Times.Once);
this.executionManager.Verify(em => em.Initialize(), Times.Once);
Assert.AreEqual(testRunCriteria, actualTestRunRequest.TestRunCriteria);
}
public void StartTestRunShouldProcessAllTestCases()
{
this.proxyParallelExecutionManager = new ParallelProxyExecutionManager(this.proxyManagerFunc, 3);
var tests = CreateTestCases();
var testRunCriteria = new TestRunCriteria(tests, 100);
var processedTestCases = new List <TestCase>();
SetupMockManagersForTestCase(processedTestCases, testRunCriteria);
AutoResetEvent completeEvent = new AutoResetEvent(false);
this.SetupHandleTestRunComplete(completeEvent);
this.proxyParallelExecutionManager.StartTestRun(testRunCriteria, this.mockHandler.Object);
Assert.IsTrue(completeEvent.WaitOne(taskTimeout), "Test run not completed.");
Assert.AreEqual(tests.Count, processedTestCases.Count, "All Tests must be processed.");
AssertMissingAndDuplicateTestCases(tests, processedTestCases);
}
public void TestRunRequestRaiseShouldInvokeTestRunStartHandlerOfLoggersOnlyIfRegistered()
{
counter = 0;
waitHandle.Reset();
TestRunCriteria testRunCriteria = new TestRunCriteria(new List <string> {
@"x:dummy\foo.dll"
}, 10)
{
TestCaseFilter = "Name=Test1"
};
TestRunStartEventArgs testRunStartEventArgs = new TestRunStartEventArgs(testRunCriteria);
// mock for ITestRunRequest
var testRunRequest = new Mock <ITestRunRequest>();
// setup TestLogger
TestLoggerManager.Instance.AddLogger(new Uri(loggerUri), new Dictionary <string, string>());
TestLoggerManager.Instance.EnableLogging();
// Register TestRunRequest object
TestLoggerManager.Instance.RegisterTestRunEvents(testRunRequest.Object);
//Raise an event on mock object
testRunRequest.Raise(
m => m.OnRunStart += null,
testRunStartEventArgs);
// Assertions when test run events registered
waitHandle.WaitOne();
Assert.AreEqual(counter, 1);
// Unregister TestRunRequest object
TestLoggerManager.Instance.UnregisterTestRunEvents(testRunRequest.Object);
//Raise an event on mock object
testRunRequest.Raise(
m => m.OnRunStart += null,
testRunStartEventArgs);
// Assertions when test run events unregistered
Assert.AreEqual(counter, 1);
}
public void LaunchProcessWithDebuggerAttachedShouldCallCustomLauncherIfLauncherIsDebugAndRunInProgress()
{
var mockCustomLauncher = new Mock <ITestHostLauncher>();
testRunCriteria = new TestRunCriteria(new List <string> {
"foo"
}, 1, false, null, TimeSpan.Zero, mockCustomLauncher.Object);
executionManager = new Mock <IProxyExecutionManager>();
testRunRequest = new TestRunRequest(testRunCriteria, executionManager.Object);
testRunRequest.ExecuteAsync();
var testProcessStartInfo = new TestProcessStartInfo();
mockCustomLauncher.Setup(ml => ml.IsDebug).Returns(true);
testRunRequest.LaunchProcessWithDebuggerAttached(testProcessStartInfo);
mockCustomLauncher.Verify(ml => ml.LaunchTestHost(testProcessStartInfo), Times.Once);
}
internal TestRunRequest(IRequestData requestData, TestRunCriteria testRunCriteria, IProxyExecutionManager executionManager, ITestLoggerManager loggerManager, IDataSerializer dataSerializer)
{
Debug.Assert(testRunCriteria != null, "Test run criteria cannot be null");
Debug.Assert(executionManager != null, "ExecutionManager cannot be null");
Debug.Assert(requestData != null, "request Data is null");
Debug.Assert(loggerManager != null, "LoggerManager cannot be null");
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("TestRunRequest.ExecuteAsync: Creating test run request.");
}
this.testRunCriteria = testRunCriteria;
this.ExecutionManager = executionManager;
this.LoggerManager = loggerManager;
this.State = TestRunState.Pending;
this.dataSerializer = dataSerializer;
this.requestData = requestData;
}
public void CreateTestRunRequestShouldSetCustomHostLauncherOnEngineDefaultLauncherIfSpecified()
{
var mockCustomLauncher = new Mock <ITestHostLauncher>();
this.executionManager.Setup(dm => dm.Initialize()).Verifiable();
this.testEngine.Setup(te => te.GetExecutionManager(this.hostManager.Object, It.IsAny <TestRunCriteria>(), It.IsAny <ProtocolConfig>())).Returns(this.executionManager.Object);
this.testEngine.Setup(te => te.GetExtensionManager()).Returns(this.extensionManager.Object);
var tp = new TestableTestPlatform(this.testEngine.Object, this.hostManager.Object);
var testRunCriteria = new TestRunCriteria(new List <string> {
"foo"
}, 10, false, null, TimeSpan.Zero, mockCustomLauncher.Object);
var testRunRequest = tp.CreateTestRunRequest(testRunCriteria, It.IsAny <ProtocolConfig>());
var actualTestRunRequest = testRunRequest as TestRunRequest;
Assert.AreEqual(testRunCriteria, actualTestRunRequest.TestRunCriteria);
this.hostManager.Verify(hl => hl.SetCustomLauncher(mockCustomLauncher.Object), Times.Once);
}
private void RunTests(IRequestData requestData, TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar)
{
// Make sure to run the run request inside a lock as the below section is not thread-safe
// TranslationLayer can process faster as it directly gets the raw unserialized messages whereas
// below logic needs to deserialize and do some cleanup
// While this section is cleaning up, TranslationLayer can trigger run causing multiple threads to run the below section at the same time
lock (syncobject)
{
try
{
this.currentTestRunRequest = this.testPlatform.CreateTestRunRequest(requestData, testRunCriteria);
this.testRunResultAggregator.RegisterTestRunEvents(this.currentTestRunRequest);
testRunEventsRegistrar?.RegisterTestRunEvents(this.currentTestRunRequest);
this.testPlatformEventSource.ExecutionRequestStart();
this.currentTestRunRequest.ExecuteAsync();
this.runRequestStartedEventHandle.Set();
// Wait for the run completion event
this.currentTestRunRequest.WaitForCompletion();
}
catch (Exception ex)
{
EqtTrace.Error("TestRequestManager.RunTests: failed to run tests: {0}", ex);
testRunResultAggregator.MarkTestRunFailed();
throw;
}
finally
{
if (this.currentTestRunRequest != null)
{
this.testRunResultAggregator.UnregisterTestRunEvents(this.currentTestRunRequest);
testRunEventsRegistrar?.UnregisterTestRunEvents(this.currentTestRunRequest);
this.currentTestRunRequest.Dispose();
this.currentTestRunRequest = null;
}
}
}
}
/// <summary>
/// The create test run request.
/// </summary>
/// <param name="testRunCriteria"> The test run criteria. </param>
/// <param name="protocolConfig"> Protocol related information. </param>
/// <returns> The <see cref="ITestRunRequest"/>. </returns>
/// <exception cref="ArgumentNullException"> Throws if parameter is null. </exception>
public ITestRunRequest CreateTestRunRequest(TestRunCriteria testRunCriteria, ProtocolConfig protocolConfig)
{
if (testRunCriteria == null)
{
throw new ArgumentNullException(nameof(testRunCriteria));
}
this.AddExtensionAssemblies(testRunCriteria.TestRunSettings);
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings);
// Update and initialize loggers only when DesignMode is false
if (runConfiguration.DesignMode == false)
{
this.AddExtensionAssembliesFromSource(testRunCriteria);
// Initialize loggers
TestLoggerManager.Instance.InitializeLoggers();
}
var testHostManager = this.testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings);
testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings);
// Allow TestRuntimeProvider to update source map, this is required for remote scenarios.
// If we run for specific tests, then we expect the test case object to contain correct source path for remote scenario as well
if (!testRunCriteria.HasSpecificTests)
{
UpdateTestSources(testRunCriteria.Sources, testRunCriteria.AdapterSourceMap, testHostManager);
}
if (testRunCriteria.TestHostLauncher != null)
{
testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher);
}
var executionManager = this.TestEngine.GetExecutionManager(testHostManager, testRunCriteria, protocolConfig);
executionManager.Initialize();
return(new TestRunRequest(testRunCriteria, executionManager));
}
/// <summary>
/// Run Tests with given a set of test cases.
/// </summary>
/// <param name="testRunRequestPayload">TestRun request Payload</param>
/// <param name="testHostLauncher">TestHost Launcher for the run</param>
/// <param name="testRunEventsRegistrar">event registrar for run events</param>
/// <returns>True, if successful</returns>
public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar)
{
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings);
var batchSize = runConfiguration.BatchSize;
TestRunCriteria runCriteria = null;
var runsettings = testRunRequestPayload.RunSettings;
if (this.TryUpdateDesignMode(runsettings, out string updatedRunsettings))
{
runsettings = updatedRunsettings;
}
if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any())
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.Sources,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue;
}
else
{
runCriteria = new TestRunCriteria(
testRunRequestPayload.TestCases,
batchSize,
testRunRequestPayload.KeepAlive,
runsettings,
this.commandLineOptions.TestStatsEventTimeout,
testHostLauncher);
}
var success = this.RunTests(runCriteria, testRunEventsRegistrar);
EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success);
return(success);
}
/// <summary>
/// Starts the test run
/// </summary>
/// <param name="testRunCriteria"> The settings/options for the test run. </param>
/// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param>
/// <returns> The process id of the runner executing tests. </returns>
public override int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler)
{
var currentEventHandler = eventHandler;
if (this.ProxyDataCollectionManager != null)
{
currentEventHandler = new DataCollectionTestRunEventsHandler(eventHandler, this.ProxyDataCollectionManager);
}
// Log all the exceptions that has occured while initializing DataCollectionClient
if (this.DataCollectionRunEventsHandler?.ExceptionMessages?.Count > 0)
{
foreach (var message in this.DataCollectionRunEventsHandler.ExceptionMessages)
{
currentEventHandler.HandleLogMessage(TestMessageLevel.Error, message);
}
}
return(base.StartTestRun(testRunCriteria, currentEventHandler));
}
public static TestRunCriteriaWithTests CreateTestRunCriteriaForTests(this TestRunCriteria testRunCriteria, ITestRuntimeProvider testRuntimeProvider,
string runSettings, TestExecutionContext executionContext, IEnumerable <string> inputPackages)
{
if (TryCheckTestSourceDifferFromPackage(testRuntimeProvider, inputPackages, out IEnumerable <string> actualTestSources))
{
// In UWP scenario TestCase object contains the package as source, which is not actual test source for adapters,
// so update test case before sending them.
// We are limiting that a testhost will always run for a single package, A package can contain multiple sources
foreach (var tc in testRunCriteria.Tests)
{
tc.Source = actualTestSources.FirstOrDefault();
}
}
else
{
inputPackages = null;
}
return(new TestRunCriteriaWithTests(testRunCriteria.Tests, inputPackages?.FirstOrDefault(), runSettings, executionContext));
}
public void GetExtensionManagerShouldCollectMetrics()
{
string settingXml =
@"<RunSettings>
<RunConfiguration>
<DisableAppDomain>false</DisableAppDomain>
<DesignMode>false</DesignMode>
<TargetFrameworkVersion>.NETFramework, Version=v4.5</TargetFrameworkVersion>
<MaxCpuCount>1</MaxCpuCount>
</RunConfiguration>
</RunSettings>";
var testRunCriteria = new TestRunCriteria(new List <string> {
"1.dll", "2.dll"
}, 100, false, settingXml);
var executionManager = this.testEngine.GetExecutionManager(this.mockRequestData.Object, this.testableTestRuntimeProvider, testRunCriteria);
this.mockMetricsCollection.Verify(mc => mc.Add(TelemetryDataConstants.ParallelEnabledDuringExecution, It.IsAny <object>()), Times.Once);
}
public void SourcesShouldEnumerateThroughAllSourcesInTheAdapterSourceMap()
{
var adapterSourceMap = new Dictionary <string, IEnumerable <string> >();
var sourceSet1 = new List <string> {
"s1.dll", "s2.dll"
};
var sourceSet2 = new List <string> {
"s1.json", "s2.json"
};
adapterSourceMap.Add("dummyadapter1", sourceSet1);
adapterSourceMap.Add("dummyadapter2", sourceSet2);
var testRunCriteria = new TestRunCriteria(adapterSourceMap, 10, false, null, TimeSpan.MaxValue, null);
var expectedSourceSet = new List <string>(sourceSet1);
expectedSourceSet.AddRange(sourceSet2);
CollectionAssert.AreEqual(expectedSourceSet, testRunCriteria.Sources.ToList());
}
public void CreateTestRunRequestShouldAllowRuntimeProviderToUpdateAdapterSource()
{
var updatedSources = new List <string> {
@"x:\dummy2\foo.dll"
};
var originalSource = new List <string> {
@"x:dummy\foo.dll"
};
var additionalExtensions = new List <string> {
"foo.TestLogger.dll", "Joo.TestLogger.dll"
};
this.mockFileHelper.Setup(fh => fh.DirectoryExists(It.IsAny <string>())).Returns(true);
this.mockFileHelper.Setup(fh => fh.EnumerateFiles(It.IsAny <string>(), System.IO.SearchOption.TopDirectoryOnly, It.IsAny <string[]>())).Returns(additionalExtensions);
this.executionManager.Setup(dm => dm.Initialize()).Verifiable();
string settingsXml =
@"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<DesignMode>false</DesignMode>
</RunConfiguration>
</RunSettings>";
var testRunCriteria = new TestRunCriteria(originalSource, 10, false, settingsXml, TimeSpan.Zero);
this.hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources))
.Returns(updatedSources);
this.testEngine.Setup(te => te.GetExecutionManager(this.hostManager.Object, It.IsAny <TestRunCriteria>(), It.IsAny <ProtocolConfig>())).Returns(this.executionManager.Object);
this.testEngine.Setup(te => te.GetExtensionManager()).Returns(this.extensionManager.Object);
var tp = new TestableTestPlatform(this.testEngine.Object, this.mockFileHelper.Object, this.hostManager.Object);
var testRunRequest = tp.CreateTestRunRequest(testRunCriteria, It.IsAny <ProtocolConfig>());
this.hostManager.Verify(hm => hm.GetTestSources(originalSource), Times.Once);
Assert.IsTrue(!testRunCriteria.Sources.Except(updatedSources).Any());
this.extensionManager.Verify(em => em.UseAdditionalExtensions(additionalExtensions, true));
}
public void StartTestRunShouldUpdateTestPlauginCacheWithExtensionsReturnByTestHost()
{
this.mockTestHostManager.Setup(o => o.GetTestPlatformExtensions(It.IsAny <IEnumerable <string> >(), It.IsAny <IEnumerable <string> >())).Returns(new List <string> {
"C:\\dummy.dll"
});
List <string> expectedResult = new List <string>();
if (TestPluginCache.Instance.PathToExtensions != null)
{
expectedResult.AddRange(TestPluginCache.Instance.PathToExtensions);
}
expectedResult.Add("C:\\dummy.dll");
var testRunCriteria = new TestRunCriteria(new List <string> {
"source.dll"
}, 10);
this.inProcessProxyExecutionManager.StartTestRun(testRunCriteria, null);
Assert.IsFalse(expectedResult.Except(TestPluginCache.Instance.PathToExtensions).Any());
}
public void GetExecutionManagerShouldReturnInProcessProxyexecutionManager()
{
string settingXml =
@"<RunSettings>
<RunConfiguration>
<DisableAppDomain>false</DisableAppDomain>
<DesignMode>false</DesignMode>
<TargetFrameworkVersion>.NETFramework, Version=v4.5</TargetFrameworkVersion>
<MaxCpuCount>1</MaxCpuCount>
</RunConfiguration>
</RunSettings>";
var testRunCriteria = new TestRunCriteria(new List <string> {
"1.dll", "2.dll"
}, 100, false, settingXml);
var executionManager = this.testEngine.GetExecutionManager(this.mockRequestData.Object, this.testableTestRuntimeProvider, testRunCriteria, this.protocolConfig);
Assert.IsNotNull(executionManager);
Assert.IsInstanceOfType(executionManager, typeof(InProcessProxyExecutionManager));
}
/// <summary>
/// The create test run request.
/// </summary>
/// <param name="testRunCriteria"> The test run criteria. </param>
/// <returns> The <see cref="ITestRunRequest"/>. </returns>
/// <exception cref="ArgumentNullException"> Throws if parameter is null. </exception>
public ITestRunRequest CreateTestRunRequest(TestRunCriteria testRunCriteria)
{
if (testRunCriteria == null)
{
throw new ArgumentNullException("testRunCriteria");
}
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings);
var testHostManager = this.TestEngine.GetDefaultTestHostManager(runConfiguration);
if (testRunCriteria.TestHostLauncher != null)
{
testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher);
}
var executionManager = this.TestEngine.GetExecutionManager(testHostManager, testRunCriteria);
executionManager.Initialize();
return(new TestRunRequest(testRunCriteria, executionManager));
}
public int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler)
{
this.hasSpecificTestsRun = testRunCriteria.HasSpecificTests;
this.actualTestRunCriteria = testRunCriteria;
if (this.hasSpecificTestsRun)
{
var testCasesBySource = new Dictionary <string, List <TestCase> >();
foreach (var test in testRunCriteria.Tests)
{
if (!testCasesBySource.ContainsKey(test.Source))
{
testCasesBySource.Add(test.Source, new List <TestCase>());
}
testCasesBySource[test.Source].Add(test);
}
// Do not use "Dictionary.ValueCollection.Enumerator" - it becomes undetermenstic once we go out of scope of this method
// Use "ToArray" to copy ValueColleciton to a simple array and use it's enumerator
// Set the enumerator for parallel yielding of testCases
// Whenever a concurrent executor becomes free, it picks up the next set of testCases using this enumerator
var testCaseLists = testCasesBySource.Values.ToArray();
this.testCaseListEnumerator = testCaseLists.GetEnumerator();
this.availableTestSources = testCaseLists.Length;
}
else
{
// Set the enumerator for parallel yielding of sources
// Whenever a concurrent executor becomes free, it picks up the next source using this enumerator
this.sourceEnumerator = testRunCriteria.Sources.GetEnumerator();
this.availableTestSources = testRunCriteria.Sources.Count();
}
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("ParallelProxyExecutionManager: Start execution. Total sources: " + this.availableTestSources);
}
return(this.StartTestRunPrivate(eventHandler));
}
/// <summary>
/// The create test run request.
/// </summary>
/// <param name="testRunCriteria"> The test run criteria. </param>
/// <param name="protocolConfig"> Protocol related information. </param>
/// <returns> The <see cref="ITestRunRequest"/>. </returns>
/// <exception cref="ArgumentNullException"> Throws if parameter is null. </exception>
public ITestRunRequest CreateTestRunRequest(IRequestData requestData, TestRunCriteria testRunCriteria)
{
if (testRunCriteria == null)
{
throw new ArgumentNullException(nameof(testRunCriteria));
}
this.AddExtensionAssemblies(testRunCriteria.TestRunSettings);
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings);
// Update extension assemblies from source when design mode is false.
if (runConfiguration.DesignMode == false)
{
this.AddExtensionAssembliesFromSource(testRunCriteria);
}
// Initialize loggers
var loggerManager = this.TestEngine.GetLoggerManager(requestData);
loggerManager.Initialize(testRunCriteria.TestRunSettings);
var testHostManager = this.testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings);
ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings);
testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings);
if (testRunCriteria.TestHostLauncher != null)
{
testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher);
}
var executionManager = this.TestEngine.GetExecutionManager(requestData, testHostManager, testRunCriteria);
executionManager.Initialize();
return(new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager));
}
/// <summary>
/// Starts the test run
/// </summary>
/// <param name="testRunCriteria"> The settings/options for the test run. </param>
/// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param>
/// <returns> The process id of the runner executing tests. </returns>
public override int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler)
{
var currentEventHandler = eventHandler;
if (this.ProxyDataCollectionManager != null)
{
currentEventHandler = new DataCollectionTestRunEventsHandler(eventHandler, this.ProxyDataCollectionManager);
}
// Log all the messages that are reported while initializing DataCollectionClient
if (this.DataCollectionRunEventsHandler.Messages.Count > 0)
{
foreach (var message in this.DataCollectionRunEventsHandler.Messages)
{
currentEventHandler.HandleLogMessage(message.Item1, message.Item2);
}
this.DataCollectionRunEventsHandler.Messages.Clear();
}
return(base.StartTestRun(testRunCriteria, currentEventHandler));
}
public void OnTestSessionTimeoutShouldNotGetCalledWhenTestSessionTimeoutIsZero()
{
string settingsXml =
@"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<TestSessionTimeout>0</TestSessionTimeout>
</RunConfiguration>
</RunSettings>";
var testRunCriteria = new TestRunCriteria(new List <string> {
"foo"
}, 1, true, settingsXml);
var executionManager = new Mock <IProxyExecutionManager>();
var testRunRequest = new TestRunRequest(this.mockRequestData.Object, testRunCriteria, executionManager.Object);
executionManager.Setup(o => o.StartTestRun(It.IsAny <TestRunCriteria>(), It.IsAny <ITestRunEventsHandler>())).Callback(() => System.Threading.Thread.Sleep(5 * 1000));
testRunRequest.ExecuteAsync();
executionManager.Verify(o => o.Abort(), Times.Never);
}
