private static int GetDistinctNumberOfSources(TestRunCriteria testRunCriteria) { // No point in creating more processes if number of sources is less than what user configured for int numSources = 1; if (testRunCriteria.HasSpecificTests) { numSources = new System.Collections.Generic.HashSet <string>( testRunCriteria.Tests.Select((testCase) => testCase.Source)).Count; } else { numSources = testRunCriteria.Sources.Count(); } return(numSources); }
public void StartTestRunShouldUpdateTestPlauginCacheWithExtensionsReturnByTestHost() { this.mockTestHostManager.Setup(o => o.GetTestPlatformExtensions(It.IsAny <IEnumerable <string> >(), It.IsAny <IEnumerable <string> >())).Returns(new List <string> { "C:\\dummy.dll" }); var expectedResult = TestPluginCache.Instance.GetExtensionPaths(string.Empty); expectedResult.Add("C:\\dummy.dll"); var testRunCriteria = new TestRunCriteria(new List <string> { "source.dll" }, 10); this.inProcessProxyExecutionManager.StartTestRun(testRunCriteria, null); CollectionAssert.AreEquivalent(expectedResult, TestPluginCache.Instance.GetExtensionPaths(string.Empty)); }
public void GetExecutionManagerShouldReturnParallelExecutionManagerIfHostIsNotShared() { string settingXml = @"<RunSettings> <RunConfiguration> <InIsolation>true</InIsolation> </RunConfiguration > </RunSettings>"; this.testableTestRuntimeProvider = new TestableRuntimeProvider(false); var testRunCriteria = new TestRunCriteria(new List <string> { "1.dll", "2.dll" }, 100, false, settingXml); Assert.IsNotNull(this.testEngine.GetExecutionManager(this.mockRequestData.Object, this.testableTestRuntimeProvider, testRunCriteria)); Assert.IsInstanceOfType(this.testEngine.GetExecutionManager(this.mockRequestData.Object, this.testableTestRuntimeProvider, testRunCriteria), typeof(ParallelProxyExecutionManager)); }
private void InvokeCreateTestRunRequest(TestPlatformOptions options = null) { this.executionManager.Setup(dm => dm.Initialize(false)).Verifiable(); this.testEngine.Setup(te => te.GetExecutionManager(this.mockRequestData.Object, this.hostManager.Object, It.IsAny <TestRunCriteria>())).Returns(this.executionManager.Object); this.testEngine.Setup(te => te.GetExtensionManager()).Returns(this.extensionManager.Object); this.testEngine.Setup(te => te.GetLoggerManager(this.mockRequestData.Object)).Returns(this.loggerManager.Object); var tp = new TestableTestPlatform(this.testEngine.Object, this.hostManager.Object); var testRunCriteria = new TestRunCriteria(new List <string> { "foo" }, 10); this.hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources)) .Returns(testRunCriteria.Sources); tp.CreateTestRunRequest(this.mockRequestData.Object, testRunCriteria, options); }
/// <summary> /// Run Tests with given a set of test cases. /// </summary> /// <param name="testRunRequestPayload">TestRun request Payload</param> /// <param name="testHostLauncher">TestHost Launcher for the run</param> /// <param name="testRunEventsRegistrar">event registrar for run events</param> /// <param name="protocolConfig">Protocol related information</param> /// <returns>True, if successful</returns> public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar, ProtocolConfig protocolConfig) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings); var batchSize = runConfiguration.BatchSize; TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; if (this.UpdateRunSettingsIfRequired(runsettings, out string updatedRunsettings)) { runsettings = updatedRunsettings; } runsettings = UpdateExtensionsFolderInRunSettings(runsettings); if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue; } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); } var success = this.RunTests(runCriteria, testRunEventsRegistrar, protocolConfig); EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success); this.testPlatformEventSource.ExecutionRequestStop(); return(success); }
/// <inheritdoc/> public ITestRunRequest CreateTestRunRequest( IRequestData requestData, TestRunCriteria testRunCriteria, TestPlatformOptions options) { if (testRunCriteria == null) { throw new ArgumentNullException(nameof(testRunCriteria)); } this.AddExtensionAssemblies(testRunCriteria.TestRunSettings); var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings); // Update extension assemblies from source when design mode is false. if (!runConfiguration.DesignMode) { this.AddExtensionAssembliesFromSource(testRunCriteria); } // Initialize loggers. var loggerManager = this.TestEngine.GetLoggerManager(requestData); loggerManager.Initialize(testRunCriteria.TestRunSettings); SubscribeLogger(loggerManager, this.VStestLoggerManager); var testHostManager = this.testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings); testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings); // NOTE: The custom launcher should not be set when we have test session info available. if (testRunCriteria.TestHostLauncher != null) { testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher); } var executionManager = this.TestEngine.GetExecutionManager(requestData, testHostManager, testRunCriteria); executionManager.Initialize(options?.SkipDefaultAdapters ?? false); return(new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager)); }
public void AbortTestRunShouldWaitForCreateTestRunRequest() { var payload = new TestRunRequestPayload() { Sources = new List <string>() { "a", "b" } }; TestRunCriteria observedCriteria = null; var sw = new Stopwatch(); sw.Start(); long createRunRequestTime = 0; long cancelRequestTime = 0; var mockRunRequest = new Mock <ITestRunRequest>(); this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny <TestRunCriteria>())).Callback <TestRunCriteria>( (runCriteria) => { Thread.Sleep(1); createRunRequestTime = sw.ElapsedMilliseconds; observedCriteria = runCriteria; }).Returns(mockRunRequest.Object); mockRunRequest.Setup(mr => mr.Abort()).Callback(() => { Thread.Sleep(1); cancelRequestTime = sw.ElapsedMilliseconds; }); var mockRunEventsRegistrar = new Mock <ITestRunEventsRegistrar>(); var mockCustomlauncher = new Mock <ITestHostLauncher>(); var cancelTask = Task.Run(() => this.testRequestManager.AbortTestRun()); var runTask = Task.Run(() => this.testRequestManager.RunTests(payload, mockCustomlauncher.Object, mockRunEventsRegistrar.Object)); Task.WaitAll(cancelTask, runTask); Assert.IsTrue(cancelRequestTime > createRunRequestTime, "CancelRequest must execute after create run request"); }
public void LaunchProcessWithDebuggerAttachedShouldNotCallCustomLauncherIfLauncherIsNotDebug() { var mockCustomLauncher = new Mock <ITestHostLauncher>(); testRunCriteria = new TestRunCriteria(new List <string> { "foo" }, 1, false, null, TimeSpan.Zero, mockCustomLauncher.Object); executionManager = new Mock <IProxyExecutionManager>(); testRunRequest = new TestRunRequest(testRunCriteria, executionManager.Object); testRunRequest.ExecuteAsync(); var testProcessStartInfo = new TestProcessStartInfo(); testRunRequest.LaunchProcessWithDebuggerAttached(testProcessStartInfo); mockCustomLauncher.Verify(ml => ml.LaunchTestHost(It.IsAny <TestProcessStartInfo>()), Times.Never); }
public void StartTestRunShouldCatchExceptionAndCallHandleRunComplete() { var testRunCriteria = new TestRunCriteria(new List <string> { "source.dll" }, 10); var mockTestRunEventsHandler = new Mock <ITestRunEventsHandler>(); var manualResetEvent = new ManualResetEvent(true); this.mockExecutionManager.Setup(o => o.StartTestRun(testRunCriteria.AdapterSourceMap, null, testRunCriteria.TestRunSettings, It.IsAny <TestExecutionContext>(), null, mockTestRunEventsHandler.Object)).Callback( () => throw new Exception()); mockTestRunEventsHandler.Setup(o => o.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(), null, null, null)).Callback( () => manualResetEvent.Set()); this.inProcessProxyExecutionManager.StartTestRun(testRunCriteria, mockTestRunEventsHandler.Object); Assert.IsTrue(manualResetEvent.WaitOne(5000), "ITestRunEventsHandler.HandleTestRunComplete should get called"); }
public void CreateTestRunRequestShouldInitializeManagersAndCreateTestRunRequestWithSpecifiedCriteria() { this.executionManager.Setup(dm => dm.Initialize()).Verifiable(); this.testEngine.Setup(te => te.GetExecutionManager(this.hostManager.Object, It.IsAny <TestRunCriteria>(), It.IsAny <ProtocolConfig>())).Returns(this.executionManager.Object); this.testEngine.Setup(te => te.GetExtensionManager()).Returns(this.extensionManager.Object); var tp = new TestableTestPlatform(this.testEngine.Object, this.hostManager.Object); var testRunCriteria = new TestRunCriteria(new List <string> { "foo" }, 10); var testRunRequest = tp.CreateTestRunRequest(testRunCriteria, It.IsAny <ProtocolConfig>()); var actualTestRunRequest = testRunRequest as TestRunRequest; this.hostManager.Verify(hm => hm.Initialize(It.IsAny <TestSessionMessageLogger>(), It.IsAny <string>()), Times.Once); this.executionManager.Verify(em => em.Initialize(), Times.Once); Assert.AreEqual(testRunCriteria, actualTestRunRequest.TestRunCriteria); }
public void StartTestRunShouldProcessAllTestCases() { this.proxyParallelExecutionManager = new ParallelProxyExecutionManager(this.proxyManagerFunc, 3); var tests = CreateTestCases(); var testRunCriteria = new TestRunCriteria(tests, 100); var processedTestCases = new List <TestCase>(); SetupMockManagersForTestCase(processedTestCases, testRunCriteria); AutoResetEvent completeEvent = new AutoResetEvent(false); this.SetupHandleTestRunComplete(completeEvent); this.proxyParallelExecutionManager.StartTestRun(testRunCriteria, this.mockHandler.Object); Assert.IsTrue(completeEvent.WaitOne(taskTimeout), "Test run not completed."); Assert.AreEqual(tests.Count, processedTestCases.Count, "All Tests must be processed."); AssertMissingAndDuplicateTestCases(tests, processedTestCases); }
public void TestRunRequestRaiseShouldInvokeTestRunStartHandlerOfLoggersOnlyIfRegistered() { counter = 0; waitHandle.Reset(); TestRunCriteria testRunCriteria = new TestRunCriteria(new List <string> { @"x:dummy\foo.dll" }, 10) { TestCaseFilter = "Name=Test1" }; TestRunStartEventArgs testRunStartEventArgs = new TestRunStartEventArgs(testRunCriteria); // mock for ITestRunRequest var testRunRequest = new Mock <ITestRunRequest>(); // setup TestLogger TestLoggerManager.Instance.AddLogger(new Uri(loggerUri), new Dictionary <string, string>()); TestLoggerManager.Instance.EnableLogging(); // Register TestRunRequest object TestLoggerManager.Instance.RegisterTestRunEvents(testRunRequest.Object); //Raise an event on mock object testRunRequest.Raise( m => m.OnRunStart += null, testRunStartEventArgs); // Assertions when test run events registered waitHandle.WaitOne(); Assert.AreEqual(counter, 1); // Unregister TestRunRequest object TestLoggerManager.Instance.UnregisterTestRunEvents(testRunRequest.Object); //Raise an event on mock object testRunRequest.Raise( m => m.OnRunStart += null, testRunStartEventArgs); // Assertions when test run events unregistered Assert.AreEqual(counter, 1); }
public void LaunchProcessWithDebuggerAttachedShouldCallCustomLauncherIfLauncherIsDebugAndRunInProgress() { var mockCustomLauncher = new Mock <ITestHostLauncher>(); testRunCriteria = new TestRunCriteria(new List <string> { "foo" }, 1, false, null, TimeSpan.Zero, mockCustomLauncher.Object); executionManager = new Mock <IProxyExecutionManager>(); testRunRequest = new TestRunRequest(testRunCriteria, executionManager.Object); testRunRequest.ExecuteAsync(); var testProcessStartInfo = new TestProcessStartInfo(); mockCustomLauncher.Setup(ml => ml.IsDebug).Returns(true); testRunRequest.LaunchProcessWithDebuggerAttached(testProcessStartInfo); mockCustomLauncher.Verify(ml => ml.LaunchTestHost(testProcessStartInfo), Times.Once); }
internal TestRunRequest(IRequestData requestData, TestRunCriteria testRunCriteria, IProxyExecutionManager executionManager, ITestLoggerManager loggerManager, IDataSerializer dataSerializer) { Debug.Assert(testRunCriteria != null, "Test run criteria cannot be null"); Debug.Assert(executionManager != null, "ExecutionManager cannot be null"); Debug.Assert(requestData != null, "request Data is null"); Debug.Assert(loggerManager != null, "LoggerManager cannot be null"); if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("TestRunRequest.ExecuteAsync: Creating test run request."); } this.testRunCriteria = testRunCriteria; this.ExecutionManager = executionManager; this.LoggerManager = loggerManager; this.State = TestRunState.Pending; this.dataSerializer = dataSerializer; this.requestData = requestData; }
public void CreateTestRunRequestShouldSetCustomHostLauncherOnEngineDefaultLauncherIfSpecified() { var mockCustomLauncher = new Mock <ITestHostLauncher>(); this.executionManager.Setup(dm => dm.Initialize()).Verifiable(); this.testEngine.Setup(te => te.GetExecutionManager(this.hostManager.Object, It.IsAny <TestRunCriteria>(), It.IsAny <ProtocolConfig>())).Returns(this.executionManager.Object); this.testEngine.Setup(te => te.GetExtensionManager()).Returns(this.extensionManager.Object); var tp = new TestableTestPlatform(this.testEngine.Object, this.hostManager.Object); var testRunCriteria = new TestRunCriteria(new List <string> { "foo" }, 10, false, null, TimeSpan.Zero, mockCustomLauncher.Object); var testRunRequest = tp.CreateTestRunRequest(testRunCriteria, It.IsAny <ProtocolConfig>()); var actualTestRunRequest = testRunRequest as TestRunRequest; Assert.AreEqual(testRunCriteria, actualTestRunRequest.TestRunCriteria); this.hostManager.Verify(hl => hl.SetCustomLauncher(mockCustomLauncher.Object), Times.Once); }
private void RunTests(IRequestData requestData, TestRunCriteria testRunCriteria, ITestRunEventsRegistrar testRunEventsRegistrar) { // Make sure to run the run request inside a lock as the below section is not thread-safe // TranslationLayer can process faster as it directly gets the raw unserialized messages whereas // below logic needs to deserialize and do some cleanup // While this section is cleaning up, TranslationLayer can trigger run causing multiple threads to run the below section at the same time lock (syncobject) { try { this.currentTestRunRequest = this.testPlatform.CreateTestRunRequest(requestData, testRunCriteria); this.testRunResultAggregator.RegisterTestRunEvents(this.currentTestRunRequest); testRunEventsRegistrar?.RegisterTestRunEvents(this.currentTestRunRequest); this.testPlatformEventSource.ExecutionRequestStart(); this.currentTestRunRequest.ExecuteAsync(); this.runRequestStartedEventHandle.Set(); // Wait for the run completion event this.currentTestRunRequest.WaitForCompletion(); } catch (Exception ex) { EqtTrace.Error("TestRequestManager.RunTests: failed to run tests: {0}", ex); testRunResultAggregator.MarkTestRunFailed(); throw; } finally { if (this.currentTestRunRequest != null) { this.testRunResultAggregator.UnregisterTestRunEvents(this.currentTestRunRequest); testRunEventsRegistrar?.UnregisterTestRunEvents(this.currentTestRunRequest); this.currentTestRunRequest.Dispose(); this.currentTestRunRequest = null; } } } }
/// <summary> /// The create test run request. /// </summary> /// <param name="testRunCriteria"> The test run criteria. </param> /// <param name="protocolConfig"> Protocol related information. </param> /// <returns> The <see cref="ITestRunRequest"/>. </returns> /// <exception cref="ArgumentNullException"> Throws if parameter is null. </exception> public ITestRunRequest CreateTestRunRequest(TestRunCriteria testRunCriteria, ProtocolConfig protocolConfig) { if (testRunCriteria == null) { throw new ArgumentNullException(nameof(testRunCriteria)); } this.AddExtensionAssemblies(testRunCriteria.TestRunSettings); var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings); // Update and initialize loggers only when DesignMode is false if (runConfiguration.DesignMode == false) { this.AddExtensionAssembliesFromSource(testRunCriteria); // Initialize loggers TestLoggerManager.Instance.InitializeLoggers(); } var testHostManager = this.testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings); // Allow TestRuntimeProvider to update source map, this is required for remote scenarios. // If we run for specific tests, then we expect the test case object to contain correct source path for remote scenario as well if (!testRunCriteria.HasSpecificTests) { UpdateTestSources(testRunCriteria.Sources, testRunCriteria.AdapterSourceMap, testHostManager); } if (testRunCriteria.TestHostLauncher != null) { testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher); } var executionManager = this.TestEngine.GetExecutionManager(testHostManager, testRunCriteria, protocolConfig); executionManager.Initialize(); return(new TestRunRequest(testRunCriteria, executionManager)); }
/// <summary> /// Run Tests with given a set of test cases. /// </summary> /// <param name="testRunRequestPayload">TestRun request Payload</param> /// <param name="testHostLauncher">TestHost Launcher for the run</param> /// <param name="testRunEventsRegistrar">event registrar for run events</param> /// <returns>True, if successful</returns> public bool RunTests(TestRunRequestPayload testRunRequestPayload, ITestHostLauncher testHostLauncher, ITestRunEventsRegistrar testRunEventsRegistrar) { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunRequestPayload.RunSettings); var batchSize = runConfiguration.BatchSize; TestRunCriteria runCriteria = null; var runsettings = testRunRequestPayload.RunSettings; if (this.TryUpdateDesignMode(runsettings, out string updatedRunsettings)) { runsettings = updatedRunsettings; } if (testRunRequestPayload.Sources != null && testRunRequestPayload.Sources.Any()) { runCriteria = new TestRunCriteria( testRunRequestPayload.Sources, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); runCriteria.TestCaseFilter = this.commandLineOptions.TestCaseFilterValue; } else { runCriteria = new TestRunCriteria( testRunRequestPayload.TestCases, batchSize, testRunRequestPayload.KeepAlive, runsettings, this.commandLineOptions.TestStatsEventTimeout, testHostLauncher); } var success = this.RunTests(runCriteria, testRunEventsRegistrar); EqtTrace.Info("TestRequestManager.RunTests: run tests completed, sucessful: {0}.", success); return(success); }
/// <summary> /// Starts the test run /// </summary> /// <param name="testRunCriteria"> The settings/options for the test run. </param> /// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param> /// <returns> The process id of the runner executing tests. </returns> public override int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { var currentEventHandler = eventHandler; if (this.ProxyDataCollectionManager != null) { currentEventHandler = new DataCollectionTestRunEventsHandler(eventHandler, this.ProxyDataCollectionManager); } // Log all the exceptions that has occured while initializing DataCollectionClient if (this.DataCollectionRunEventsHandler?.ExceptionMessages?.Count > 0) { foreach (var message in this.DataCollectionRunEventsHandler.ExceptionMessages) { currentEventHandler.HandleLogMessage(TestMessageLevel.Error, message); } } return(base.StartTestRun(testRunCriteria, currentEventHandler)); }
public static TestRunCriteriaWithTests CreateTestRunCriteriaForTests(this TestRunCriteria testRunCriteria, ITestRuntimeProvider testRuntimeProvider, string runSettings, TestExecutionContext executionContext, IEnumerable <string> inputPackages) { if (TryCheckTestSourceDifferFromPackage(testRuntimeProvider, inputPackages, out IEnumerable <string> actualTestSources)) { // In UWP scenario TestCase object contains the package as source, which is not actual test source for adapters, // so update test case before sending them. // We are limiting that a testhost will always run for a single package, A package can contain multiple sources foreach (var tc in testRunCriteria.Tests) { tc.Source = actualTestSources.FirstOrDefault(); } } else { inputPackages = null; } return(new TestRunCriteriaWithTests(testRunCriteria.Tests, inputPackages?.FirstOrDefault(), runSettings, executionContext)); }
public void GetExtensionManagerShouldCollectMetrics() { string settingXml = @"<RunSettings> <RunConfiguration> <DisableAppDomain>false</DisableAppDomain> <DesignMode>false</DesignMode> <TargetFrameworkVersion>.NETFramework, Version=v4.5</TargetFrameworkVersion> <MaxCpuCount>1</MaxCpuCount> </RunConfiguration> </RunSettings>"; var testRunCriteria = new TestRunCriteria(new List <string> { "1.dll", "2.dll" }, 100, false, settingXml); var executionManager = this.testEngine.GetExecutionManager(this.mockRequestData.Object, this.testableTestRuntimeProvider, testRunCriteria); this.mockMetricsCollection.Verify(mc => mc.Add(TelemetryDataConstants.ParallelEnabledDuringExecution, It.IsAny <object>()), Times.Once); }
public void SourcesShouldEnumerateThroughAllSourcesInTheAdapterSourceMap() { var adapterSourceMap = new Dictionary <string, IEnumerable <string> >(); var sourceSet1 = new List <string> { "s1.dll", "s2.dll" }; var sourceSet2 = new List <string> { "s1.json", "s2.json" }; adapterSourceMap.Add("dummyadapter1", sourceSet1); adapterSourceMap.Add("dummyadapter2", sourceSet2); var testRunCriteria = new TestRunCriteria(adapterSourceMap, 10, false, null, TimeSpan.MaxValue, null); var expectedSourceSet = new List <string>(sourceSet1); expectedSourceSet.AddRange(sourceSet2); CollectionAssert.AreEqual(expectedSourceSet, testRunCriteria.Sources.ToList()); }
public void CreateTestRunRequestShouldAllowRuntimeProviderToUpdateAdapterSource() { var updatedSources = new List <string> { @"x:\dummy2\foo.dll" }; var originalSource = new List <string> { @"x:dummy\foo.dll" }; var additionalExtensions = new List <string> { "foo.TestLogger.dll", "Joo.TestLogger.dll" }; this.mockFileHelper.Setup(fh => fh.DirectoryExists(It.IsAny <string>())).Returns(true); this.mockFileHelper.Setup(fh => fh.EnumerateFiles(It.IsAny <string>(), System.IO.SearchOption.TopDirectoryOnly, It.IsAny <string[]>())).Returns(additionalExtensions); this.executionManager.Setup(dm => dm.Initialize()).Verifiable(); string settingsXml = @"<?xml version=""1.0"" encoding=""utf-8""?> <RunSettings> <RunConfiguration> <DesignMode>false</DesignMode> </RunConfiguration> </RunSettings>"; var testRunCriteria = new TestRunCriteria(originalSource, 10, false, settingsXml, TimeSpan.Zero); this.hostManager.Setup(hm => hm.GetTestSources(testRunCriteria.Sources)) .Returns(updatedSources); this.testEngine.Setup(te => te.GetExecutionManager(this.hostManager.Object, It.IsAny <TestRunCriteria>(), It.IsAny <ProtocolConfig>())).Returns(this.executionManager.Object); this.testEngine.Setup(te => te.GetExtensionManager()).Returns(this.extensionManager.Object); var tp = new TestableTestPlatform(this.testEngine.Object, this.mockFileHelper.Object, this.hostManager.Object); var testRunRequest = tp.CreateTestRunRequest(testRunCriteria, It.IsAny <ProtocolConfig>()); this.hostManager.Verify(hm => hm.GetTestSources(originalSource), Times.Once); Assert.IsTrue(!testRunCriteria.Sources.Except(updatedSources).Any()); this.extensionManager.Verify(em => em.UseAdditionalExtensions(additionalExtensions, true)); }
public void StartTestRunShouldUpdateTestPlauginCacheWithExtensionsReturnByTestHost() { this.mockTestHostManager.Setup(o => o.GetTestPlatformExtensions(It.IsAny <IEnumerable <string> >(), It.IsAny <IEnumerable <string> >())).Returns(new List <string> { "C:\\dummy.dll" }); List <string> expectedResult = new List <string>(); if (TestPluginCache.Instance.PathToExtensions != null) { expectedResult.AddRange(TestPluginCache.Instance.PathToExtensions); } expectedResult.Add("C:\\dummy.dll"); var testRunCriteria = new TestRunCriteria(new List <string> { "source.dll" }, 10); this.inProcessProxyExecutionManager.StartTestRun(testRunCriteria, null); Assert.IsFalse(expectedResult.Except(TestPluginCache.Instance.PathToExtensions).Any()); }
public void GetExecutionManagerShouldReturnInProcessProxyexecutionManager() { string settingXml = @"<RunSettings> <RunConfiguration> <DisableAppDomain>false</DisableAppDomain> <DesignMode>false</DesignMode> <TargetFrameworkVersion>.NETFramework, Version=v4.5</TargetFrameworkVersion> <MaxCpuCount>1</MaxCpuCount> </RunConfiguration> </RunSettings>"; var testRunCriteria = new TestRunCriteria(new List <string> { "1.dll", "2.dll" }, 100, false, settingXml); var executionManager = this.testEngine.GetExecutionManager(this.mockRequestData.Object, this.testableTestRuntimeProvider, testRunCriteria, this.protocolConfig); Assert.IsNotNull(executionManager); Assert.IsInstanceOfType(executionManager, typeof(InProcessProxyExecutionManager)); }
/// <summary> /// The create test run request. /// </summary> /// <param name="testRunCriteria"> The test run criteria. </param> /// <returns> The <see cref="ITestRunRequest"/>. </returns> /// <exception cref="ArgumentNullException"> Throws if parameter is null. </exception> public ITestRunRequest CreateTestRunRequest(TestRunCriteria testRunCriteria) { if (testRunCriteria == null) { throw new ArgumentNullException("testRunCriteria"); } var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings); var testHostManager = this.TestEngine.GetDefaultTestHostManager(runConfiguration); if (testRunCriteria.TestHostLauncher != null) { testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher); } var executionManager = this.TestEngine.GetExecutionManager(testHostManager, testRunCriteria); executionManager.Initialize(); return(new TestRunRequest(testRunCriteria, executionManager)); }
public int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { this.hasSpecificTestsRun = testRunCriteria.HasSpecificTests; this.actualTestRunCriteria = testRunCriteria; if (this.hasSpecificTestsRun) { var testCasesBySource = new Dictionary <string, List <TestCase> >(); foreach (var test in testRunCriteria.Tests) { if (!testCasesBySource.ContainsKey(test.Source)) { testCasesBySource.Add(test.Source, new List <TestCase>()); } testCasesBySource[test.Source].Add(test); } // Do not use "Dictionary.ValueCollection.Enumerator" - it becomes undetermenstic once we go out of scope of this method // Use "ToArray" to copy ValueColleciton to a simple array and use it's enumerator // Set the enumerator for parallel yielding of testCases // Whenever a concurrent executor becomes free, it picks up the next set of testCases using this enumerator var testCaseLists = testCasesBySource.Values.ToArray(); this.testCaseListEnumerator = testCaseLists.GetEnumerator(); this.availableTestSources = testCaseLists.Length; } else { // Set the enumerator for parallel yielding of sources // Whenever a concurrent executor becomes free, it picks up the next source using this enumerator this.sourceEnumerator = testRunCriteria.Sources.GetEnumerator(); this.availableTestSources = testRunCriteria.Sources.Count(); } if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("ParallelProxyExecutionManager: Start execution. Total sources: " + this.availableTestSources); } return(this.StartTestRunPrivate(eventHandler)); }
/// <summary> /// The create test run request. /// </summary> /// <param name="testRunCriteria"> The test run criteria. </param> /// <param name="protocolConfig"> Protocol related information. </param> /// <returns> The <see cref="ITestRunRequest"/>. </returns> /// <exception cref="ArgumentNullException"> Throws if parameter is null. </exception> public ITestRunRequest CreateTestRunRequest(IRequestData requestData, TestRunCriteria testRunCriteria) { if (testRunCriteria == null) { throw new ArgumentNullException(nameof(testRunCriteria)); } this.AddExtensionAssemblies(testRunCriteria.TestRunSettings); var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings); // Update extension assemblies from source when design mode is false. if (runConfiguration.DesignMode == false) { this.AddExtensionAssembliesFromSource(testRunCriteria); } // Initialize loggers var loggerManager = this.TestEngine.GetLoggerManager(requestData); loggerManager.Initialize(testRunCriteria.TestRunSettings); var testHostManager = this.testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings); testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings); if (testRunCriteria.TestHostLauncher != null) { testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher); } var executionManager = this.TestEngine.GetExecutionManager(requestData, testHostManager, testRunCriteria); executionManager.Initialize(); return(new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager)); }
/// <summary> /// Starts the test run /// </summary> /// <param name="testRunCriteria"> The settings/options for the test run. </param> /// <param name="eventHandler"> EventHandler for handling execution events from Engine. </param> /// <returns> The process id of the runner executing tests. </returns> public override int StartTestRun(TestRunCriteria testRunCriteria, ITestRunEventsHandler eventHandler) { var currentEventHandler = eventHandler; if (this.ProxyDataCollectionManager != null) { currentEventHandler = new DataCollectionTestRunEventsHandler(eventHandler, this.ProxyDataCollectionManager); } // Log all the messages that are reported while initializing DataCollectionClient if (this.DataCollectionRunEventsHandler.Messages.Count > 0) { foreach (var message in this.DataCollectionRunEventsHandler.Messages) { currentEventHandler.HandleLogMessage(message.Item1, message.Item2); } this.DataCollectionRunEventsHandler.Messages.Clear(); } return(base.StartTestRun(testRunCriteria, currentEventHandler)); }
public void OnTestSessionTimeoutShouldNotGetCalledWhenTestSessionTimeoutIsZero() { string settingsXml = @"<?xml version=""1.0"" encoding=""utf-8""?> <RunSettings> <RunConfiguration> <TestSessionTimeout>0</TestSessionTimeout> </RunConfiguration> </RunSettings>"; var testRunCriteria = new TestRunCriteria(new List <string> { "foo" }, 1, true, settingsXml); var executionManager = new Mock <IProxyExecutionManager>(); var testRunRequest = new TestRunRequest(this.mockRequestData.Object, testRunCriteria, executionManager.Object); executionManager.Setup(o => o.StartTestRun(It.IsAny <TestRunCriteria>(), It.IsAny <ITestRunEventsHandler>())).Callback(() => System.Threading.Thread.Sleep(5 * 1000)); testRunRequest.ExecuteAsync(); executionManager.Verify(o => o.Abort(), Times.Never); }