/// <summary> /// Gets unfiltered and filtered extensions from the provided test extension collection. /// </summary> /// <typeparam name="TPluginInfo"> /// </typeparam> /// <typeparam name="IExtension"> /// Type of the required extensions /// </typeparam> /// <typeparam name="IMetadata"> /// Type of metadata of required extensions /// </typeparam> /// <typeparam name="TMetadata"> /// Concrete type of metadata /// </typeparam> /// <param name="testPluginInfo"> /// The test extension dictionary. /// </param> /// <param name="unfiltered"> /// Receives unfiltered list of test extensions /// </param> /// <param name="filtered"> /// Receives test extensions filtered by Identifier data /// </param> private void GetExtensions <TPluginInfo, IExtension, IMetadata, TMetadata>( Dictionary <string, TPluginInfo> testPluginInfo, out IEnumerable <LazyExtension <IExtension, Dictionary <string, object> > > unfiltered, out IEnumerable <LazyExtension <IExtension, IMetadata> > filtered) where TMetadata : IMetadata where TPluginInfo : TestPluginInformation { var unfilteredExtensions = new List <LazyExtension <IExtension, Dictionary <string, object> > >(); var filteredExtensions = new List <LazyExtension <IExtension, IMetadata> >(); var testPlugins = this.GetValuesFromDictionary(testPluginInfo); foreach (var plugin in testPlugins) { var testExtension = new LazyExtension <IExtension, IMetadata>(plugin, typeof(TMetadata)); if (!string.IsNullOrEmpty(plugin.IdentifierData)) { filteredExtensions.Add(testExtension); } unfilteredExtensions.Add( new LazyExtension <IExtension, Dictionary <string, object> >(plugin, new Dictionary <string, object>())); } unfiltered = unfilteredExtensions; filtered = filteredExtensions; }
/// <summary> /// Gets unfiltered and filtered extensions from the provided test extension collection. /// </summary> /// <typeparam name="IExtension">Type of the required extensions</typeparam> /// <typeparam name="IMetadata">Type of metadata of required extensions</typeparam> /// <typeparam name="TMetadata">Concrete type of metadata</typeparam> /// <param name="testExtensions">The test extension collection list.</param> /// <param name="unfiltered">Receives unfiltered list of test extensions</param> /// <param name="filtered">Receives test extensions filtered by Identifier data</param> private void GetExtensions <IExtension, IMetadata, TMetadata>( TestExtensions testExtensions, out IEnumerable <LazyExtension <IExtension, Dictionary <string, object> > > unfiltered, out IEnumerable <LazyExtension <IExtension, IMetadata> > filtered) where TMetadata : IMetadata { var unfilteredExtensions = new List <LazyExtension <IExtension, Dictionary <string, object> > >(); var filteredExtensions = new List <LazyExtension <IExtension, IMetadata> >(); var testPlugins = this.GetExtensionsCollection(testExtensions, typeof(IExtension)); foreach (var plugin in testPlugins) { var testExtension = new LazyExtension <IExtension, IMetadata>(plugin, typeof(TMetadata)); if (!string.IsNullOrEmpty(plugin.IdentifierData)) { filteredExtensions.Add(testExtension); } unfilteredExtensions.Add( new LazyExtension <IExtension, Dictionary <string, object> >(plugin, new Dictionary <string, object>())); } unfiltered = unfilteredExtensions; filtered = filteredExtensions; }
public void RunTestsShouldReportAWarningIfExecutorUriIsNotDefinedInExtensionAssembly() { var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; var executorUriExtensionMap = new List <Tuple <Uri, string> > { new Tuple <Uri, string>(new Uri("executor://nonexistent/"), assemblyLocation) }; LazyExtension <ITestExecutor, ITestExecutorCapabilities> receivedExecutor = null; // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); }; this.runTestsInstance.InvokeExecutorCallback = (executor, executorUriExtensionTuple, runContext, frameworkHandle) => { receivedExecutor = executor; }; this.runTestsInstance.RunTests(); var expectedWarningMessageFormat = "Could not find test executor with URI '{0}'. Make sure that the test executor is installed and supports .net runtime version {1}."; //var runtimeVersion = string.Concat(PlatformServices.Default.Runtime.RuntimeType, " ", // PlatformServices.Default.Runtime.RuntimeVersion); var runtimeVersion = " "; var expectedWarningMessage = string.Format(expectedWarningMessageFormat, "executor://nonexistent/", runtimeVersion); this.mockTestRunEventsHandler.Verify( treh => treh.HandleLogMessage(TestMessageLevel.Warning, expectedWarningMessage), Times.Once); // Should not have been called. Assert.IsNull(receivedExecutor); }
public void InvokeExecutorShouldInvokeTestExecutorWithTheTests() { var tests = new List <TestCase> { new TestCase("A.C.M1", new Uri("e://d"), "s.dll") }; var executorUriVsTestList = new Dictionary <Tuple <Uri, string>, List <TestCase> >(); var executorUriExtensionTuple = new Tuple <Uri, string>(new Uri("e://d/"), "A.dll"); executorUriVsTestList.Add(executorUriExtensionTuple, tests); this.runTestsInstance = new TestableRunTestsWithTests( tests, null, testExecutionContext, null, this.mockTestRunEventsHandler.Object, executorUriVsTestList, this.mockRequestData.Object); var testExecutor = new RunTestsWithSourcesTests.RunTestWithSourcesExecutor(); var extension = new LazyExtension <ITestExecutor, ITestExecutorCapabilities>(testExecutor, new TestExecutorMetadata("e://d/")); IEnumerable <TestCase> receivedTests = null; RunTestsWithSourcesTests.RunTestWithSourcesExecutor.RunTestsWithTestsCallback = (t, rc, fh) => { receivedTests = t; }; this.runTestsInstance.CallInvokeExecutor(extension, executorUriExtensionTuple, null, null); Assert.IsNotNull(receivedTests); CollectionAssert.AreEqual(tests, receivedTests.ToList()); }
protected override bool ShouldAttachDebuggerToTestHost( LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUri, RunContext runContext) { return(false); }
protected override void InvokeExecutor( LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle) { executor?.Value.RunTests(this.executorUriVsSourceList[executorUriExtensionTuple], runContext, frameworkHandle); }
public void ValueShouldCreateExtensionViaTheCallback() { var mockExtension = new Mock <ITestDiscoverer>(); LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> extension = new LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>( () => { return(mockExtension.Object); }, new Mock <ITestDiscovererCapabilities>().Object); Assert.AreEqual(mockExtension.Object, extension.Value); }
public void MetadataShouldReturnMetadataSpecified() { var testDiscovererPluginInfo = new TestDiscovererPluginInformation(typeof(DummyExtension)); var mockMetadata = new Mock <ITestDiscovererCapabilities>(); LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> extension = new LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>( testDiscovererPluginInfo, mockMetadata.Object); Assert.AreEqual(mockMetadata.Object, extension.Metadata); }
public void ValueShouldCreateExtensionViaTestPluginManager() { var testDiscovererPluginInfo = new TestDiscovererPluginInformation(typeof(DummyExtension)); LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> extension = new LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>( testDiscovererPluginInfo, new Mock <ITestDiscovererCapabilities>().Object); Assert.IsNotNull(extension.Value); Assert.AreEqual(typeof(DummyExtension), extension.Value.GetType()); }
private static bool IsDiscovererFromDeprecatedLocations( LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> discoverer) { if (CrossPlatEngine.Constants.DefaultAdapters.Contains(discoverer.Metadata.DefaultExecutorUri.ToString(), StringComparer.OrdinalIgnoreCase)) { return(false); } var discovererLocation = discoverer.Value.GetType().GetTypeInfo().Assembly.GetAssemblyLocation(); return(Path.GetDirectoryName(discovererLocation) .Equals(CrossPlatEngine.Constants.DefaultAdapterLocation, StringComparison.OrdinalIgnoreCase)); }
/// <inheritdoc /> protected override bool ShouldAttachDebuggerToTestHost( LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUri, RunContext runContext) { // If the adapter doesn't implement the new test executor interface we should attach to // the default test host by default to preserve old behavior. if (!(executor?.Value is ITestExecutor2 convertedExecutor)) { return(true); } return(convertedExecutor.ShouldAttachToTestHost(this.executorUriVsTestList[executorUri], runContext)); }
public LazyExtension <TExtension, TMetadata> TryGetTestExtension(string extensionUri) { ValidateArg.NotNull <string>(extensionUri, "extensionUri"); LazyExtension <TExtension, TMetadata> testExtension = null; foreach (var availableExtensionUri in this.TestExtensionByUri.Keys) { if (string.Compare(extensionUri, availableExtensionUri.AbsoluteUri, StringComparison.OrdinalIgnoreCase) == 0) { this.TestExtensionByUri.TryGetValue(availableExtensionUri, out testExtension); break; } } return(testExtension); }
private void SetupExecutorUriMock() { var executorUriExtensionMap = new List <Tuple <Uri, string> > { new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), Microsoft.VisualStudio.TestPlatform.ObjectModel.Constants.UnspecifiedAdapterPath) }; LazyExtension <ITestExecutor, ITestExecutorCapabilities> receivedExecutor = null; // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); }; this.runTestsInstance.InvokeExecutorCallback = (executor, executorUriExtensionTuple, runContext, frameworkHandle) => { receivedExecutor = executor; }; }
public void MetadataShouldCreateMetadataFromMetadataType() { var testDiscovererPluginInfo = new TestDiscovererPluginInformation(typeof(DummyExtension)); LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> extension = new LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>( testDiscovererPluginInfo, typeof(DummyDiscovererCapability)); var metadata = extension.Metadata; Assert.IsNotNull(metadata); Assert.AreEqual(typeof(DummyDiscovererCapability), metadata.GetType()); CollectionAssert.AreEqual(new List <string> { "csv" }, (metadata as ITestDiscovererCapabilities).FileExtension.ToArray()); Assert.AreEqual("executor://unittestexecutor/", (metadata as ITestDiscovererCapabilities).DefaultExecutorUri.AbsoluteUri); }
public void ValueShouldNotCreateExtensionIfAlreadyCreated() { var numberOfTimesExtensionCreated = 0; var mockExtension = new Mock <ITestDiscoverer>(); LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> extension = new LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>( () => { numberOfTimesExtensionCreated++; return(mockExtension.Object); }, new Mock <ITestDiscovererCapabilities>().Object); var temp = extension.Value; temp = extension.Value; Assert.AreEqual(1, numberOfTimesExtensionCreated); }
private IEnumerable <LazyExtension <ISettingsProvider, ISettingsProviderCapabilities> > GetMockExtensions(params string[] settingNames) { var settingsList = new List <LazyExtension <ISettingsProvider, ISettingsProviderCapabilities> >(); foreach (var settingName in settingNames) { var mockSettingsProvider = new Mock <ISettingsProvider>(); var metadata = new TestSettingsProviderMetadata(settingName); var extension = new LazyExtension <ISettingsProvider, ISettingsProviderCapabilities>( mockSettingsProvider.Object, metadata); settingsList.Add(extension); } return(settingsList); }
public void RunTestsShouldInvokeTheTestExecutorIfAdapterAssemblyIsUnknown() { var executorUriExtensionMap = new List <Tuple <Uri, string> > { new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), Microsoft.VisualStudio.TestPlatform.ObjectModel.Constants.UnspecifiedAdapterPath) }; LazyExtension <ITestExecutor, ITestExecutorCapabilities> receivedExecutor = null; // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); }; this.runTestsInstance.InvokeExecutorCallback = (executor, executorUriExtensionTuple, runContext, frameworkHandle) => { receivedExecutor = executor; }; this.runTestsInstance.RunTests(); Assert.IsNotNull(receivedExecutor); Assert.AreEqual(BaseRunTestsExecutorUri, receivedExecutor.Metadata.ExtensionUri); }
public void InvokeExecutorShouldInvokeTestExecutorWithTheSources() { var adapterSourceMap = new Dictionary <string, IEnumerable <string> >(); adapterSourceMap.Add("a", new List <string> { "a", "aa" }); adapterSourceMap.Add("b", new List <string> { "b", "ab" }); var executorUriVsSourceList = new Dictionary <Tuple <Uri, string>, IEnumerable <string> >(); var executorUriExtensionTuple = new Tuple <Uri, string>(new Uri("e://d/"), "A.dll"); executorUriVsSourceList.Add(executorUriExtensionTuple, new List <string> { "s1.dll " }); this.runTestsInstance = new TestableRunTestsWithSources( adapterSourceMap, null, testExecutionContext, null, this.mockTestRunEventsHandler.Object, executorUriVsSourceList, this.mockRequestData.Object); var testExecutor = new RunTestWithSourcesExecutor(); var extension = new LazyExtension <ITestExecutor, ITestExecutorCapabilities>(testExecutor, new TestExecutorMetadata("e://d/")); IEnumerable <string> receivedSources = null; RunTestWithSourcesExecutor.RunTestsWithSourcesCallback = (sources, rc, fh) => { receivedSources = sources; }; this.runTestsInstance.CallInvokeExecutor(extension, executorUriExtensionTuple, null, null); Assert.IsNotNull(receivedSources); CollectionAssert.AreEqual(new List <string> { "s1.dll " }, receivedSources.ToList()); }
public void RunTestsShouldInvokeTheTestExecutorIfAdapterAssemblyIsKnown() { var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; var executorUriExtensionMap = new List <Tuple <Uri, string> > { new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), assemblyLocation) }; LazyExtension <ITestExecutor, ITestExecutorCapabilities> receivedExecutor = null; // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); }; this.runTestsInstance.InvokeExecutorCallback = (executor, executorUriExtensionTuple, runContext, frameworkHandle) => { receivedExecutor = executor; }; this.runTestsInstance.RunTests(); Assert.IsNotNull(receivedExecutor); Assert.AreEqual(BaseRunTestsExecutorUri, receivedExecutor.Metadata.ExtensionUri); }
private static bool TryToLoadDiscoverer(LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> discoverer, IMessageLogger logger, out Type discovererType) { discovererType = null; // See if discoverer can be instantiated successfully else move next. try { discovererType = discoverer.Value.GetType(); } catch (Exception e) { var mesage = string.Format( CultureInfo.CurrentUICulture, CrossPlatEngineResources.DiscovererInstantiationException, e.Message); logger.SendMessage(TestMessageLevel.Warning, mesage); EqtTrace.Error("DiscovererEnumerator.LoadTestsFromAnExtension: {0} ", e); return(false); } return(true); }
protected override void InvokeExecutor(LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle) { this.InvokeExecutorCallback?.Invoke(executor, executorUriExtensionTuple, runContext, frameworkHandle); }
public void CallInvokeExecutor(LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle) { this.InvokeExecutor(executor, executorUriExtensionTuple, runContext, frameworkHandle); }
private void DiscoverTestsFromSingleDiscoverer( LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> discoverer, Dictionary <LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>, IEnumerable <string> > discovererToSourcesMap, DiscoveryContext context, TestCaseDiscoverySink discoverySink, IMessageLogger logger, ref double totalAdaptersUsed, ref double totalTimeTakenByAdapters) { if (DiscovererEnumerator.TryToLoadDiscoverer(discoverer, logger, out var discovererType) == false) { // Fail to instantiate the discoverer type. return; } // on instantiated successfully, get tests try { EqtTrace.Verbose( "DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Loading tests for {0}", discoverer.Value.GetType().FullName); if (discoverer.Metadata.DefaultExecutorUri == null) { throw new Exception($@"DefaultExecutorUri is null, did you decorate the discoverer class with [DefaultExecutorUri()] attribute? For example [DefaultExecutorUri(""executor://example.testadapter"")]."); } var currentTotalTests = this.discoveryResultCache.TotalDiscoveredTests; var newTimeStart = DateTime.UtcNow; this.testPlatformEventSource.AdapterDiscoveryStart(discoverer.Metadata.DefaultExecutorUri.AbsoluteUri); discoverer.Value.DiscoverTests(discovererToSourcesMap[discoverer], context, logger, discoverySink); var totalAdapterRunTime = DateTime.UtcNow - newTimeStart; this.testPlatformEventSource.AdapterDiscoveryStop(this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests); // Record Total Tests Discovered By each Discoverer. var totalTestsDiscoveredByCurrentDiscoverer = this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests; this.requestData.MetricsCollection.Add( string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsByAdapter, discoverer.Metadata.DefaultExecutorUri), totalTestsDiscoveredByCurrentDiscoverer); totalAdaptersUsed++; EqtTrace.Verbose("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Done loading tests for {0}", discoverer.Value.GetType().FullName); var discovererFromDeprecatedLocations = DiscovererEnumerator.IsDiscovererFromDeprecatedLocations(discoverer); if (discovererFromDeprecatedLocations) { logger.SendMessage(TestMessageLevel.Warning, string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DeprecatedAdapterPath)); } // Collecting Data Point for Time Taken to Discover Tests by each Adapter this.requestData.MetricsCollection.Add( string.Format("{0}.{1}", TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter, discoverer.Metadata.DefaultExecutorUri), totalAdapterRunTime.TotalSeconds); totalTimeTakenByAdapters += totalAdapterRunTime.TotalSeconds; } catch (Exception e) { var message = string.Format( CultureInfo.CurrentUICulture, CrossPlatEngineResources.ExceptionFromLoadTests, discovererType.Name, e.Message); logger.SendMessage(TestMessageLevel.Error, message); EqtTrace.Error("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: {0} ", e); } }
private DiscoveryResult DiscoverTestsFromSingleDiscoverer( LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> discoverer, IEnumerable <string> sources, DiscoveryContext context, TestCaseDiscoverySink discoverySink, IMessageLogger logger, CancellationToken cancellationToken) { var result = new DiscoveryResult(); if (DiscovererEnumerator.TryToLoadDiscoverer(discoverer, logger, out var discovererType) == false) { // Fail to instantiate the discoverer type. return(result); } // on instantiated successfully, get tests try { EqtTrace.Verbose( "DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Loading tests for {0}", discoverer.Value.GetType().FullName); var currentTotalTests = this.discoveryResultCache.TotalDiscoveredTests; var newTimeStart = DateTime.UtcNow; this.testPlatformEventSource.AdapterDiscoveryStart(discoverer.Metadata.DefaultExecutorUri.AbsoluteUri); foreach (var testSource in sources) { if (cancellationToken.IsCancellationRequested) { EqtTrace.Info("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Cancellation Requested. Aborting the discovery"); break; } discoverer.Value.DiscoverTests(new[] { testSource }, context, logger, discoverySink); } var totalAdapterRunTime = DateTime.UtcNow - newTimeStart; this.testPlatformEventSource.AdapterDiscoveryStop(this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests); // Record Total Tests Discovered By each Discoverer. var totalTestsDiscoveredByCurrentDiscoverer = this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests; this.requestData.MetricsCollection.Add( string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsByAdapter, discoverer.Metadata.DefaultExecutorUri), totalTestsDiscoveredByCurrentDiscoverer); result.TotalAdaptersUsed++; EqtTrace.Verbose("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Done loading tests for {0}", discoverer.Value.GetType().FullName); var discovererFromDeprecatedLocations = DiscovererEnumerator.IsDiscovererFromDeprecatedLocations(discoverer); if (discovererFromDeprecatedLocations) { logger.SendMessage(TestMessageLevel.Warning, string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DeprecatedAdapterPath)); } // Collecting Data Point for Time Taken to Discover Tests by each Adapter this.requestData.MetricsCollection.Add($"{TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter}.{discoverer.Metadata.DefaultExecutorUri}", totalAdapterRunTime.TotalSeconds); result.TotalTimeSpentInAdapaters += totalAdapterRunTime.TotalSeconds; } catch (Exception e) { var message = string.Format(CultureInfo.CurrentUICulture, CrossPlatEngineResources.ExceptionFromLoadTests, discovererType.Name, e.Message); logger.SendMessage(TestMessageLevel.Error, message); EqtTrace.Error("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: {0} ", e); } return(result); }
/// <summary> /// Asks the adapter about attaching the debugger to the default test host. /// </summary> /// <param name="executor">The executor used to run the tests.</param> /// <param name="executorUriExtensionTuple">The executor URI.</param> /// <param name="runContext">The run context.</param> /// <returns> /// <see cref="true"/> if must attach the debugger to the default test host, /// <see cref="false"/> otherwise. /// </returns> protected abstract bool ShouldAttachDebuggerToTestHost( LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext);
protected abstract void InvokeExecutor( LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle);