private void SetupForExecutionThreadApartmentStateTests(PlatformApartmentState apartmentState)
{
this.mockThread = new Mock <IThread>();
this.runTestsInstance = new TestableBaseRunTests(
this.mockRequestData.Object,
null,
$@"<RunSettings>
<RunConfiguration>
<ExecutionThreadApartmentState>{apartmentState}</ExecutionThreadApartmentState>
</RunConfiguration>
</RunSettings>",
this.testExecutionContext,
null,
this.mockTestRunEventsHandler.Object,
this.mockTestPlatformEventSource.Object,
null,
this.mockThread.Object,
this.mockDataSerializer.Object);
TestPluginCacheHelper.SetupMockExtensions(new string[] { typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location }, () => { });
var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location;
var executorUriExtensionMap = new List <Tuple <Uri, string> >
{
new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), assemblyLocation)
};
this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); };
}
public void InitializeShouldLoadAndInitializeAllExtensions()
{
var commonAssemblyLocation = typeof(ExecutionManagerTests).GetTypeInfo().Assembly.Location;
var mockTestMessageEventHandler = new Mock <ITestMessageEventHandler>();
TestPluginCacheHelper.SetupMockExtensions(
new string[] { commonAssemblyLocation },
() => { });
this.executionManager.Initialize(new List <string> {
commonAssemblyLocation
}, mockTestMessageEventHandler.Object);
Assert.IsNotNull(TestPluginCache.Instance.TestExtensions);
// Executors
Assert.IsTrue(TestPluginCache.Instance.TestExtensions.TestExecutors.Count > 0);
var allExecutors = TestExecutorExtensionManager.Create().TestExtensions;
foreach (var executor in allExecutors)
{
Assert.IsTrue(executor.IsExtensionCreated);
}
// Settings Providers
Assert.IsTrue(TestPluginCache.Instance.TestExtensions.TestSettingsProviders.Count > 0);
var settingsProviders = SettingsProviderExtensionManager.Create().SettingsProvidersMap.Values;
foreach (var provider in settingsProviders)
{
Assert.IsTrue(provider.IsExtensionCreated);
}
}
public void DiscoverTestsShouldInitializeExtensionsIfPresent()
{
// Make sure TestPlugincache is refreshed.
TestPluginCache.Instance = null;
try
{
var extensions = new[] { "c:\\e1.dll", "c:\\e2.dll" };
// Setup Mocks.
TestPluginCacheHelper.SetupMockAdditionalPathExtensions(extensions);
this.mockFileHelper.Setup(fh => fh.Exists(It.IsAny <string>())).Returns(true);
this.mockRequestSender.Setup(s => s.WaitForRequestHandlerConnection(It.IsAny <int>(), It.IsAny <CancellationToken>())).Returns(true);
this.mockTestHostManager.Setup(th => th.GetTestPlatformExtensions(It.IsAny <IEnumerable <string> >(), It.IsAny <IEnumerable <string> >())).Returns(new[] { "c:\\e1.dll", "c:\\e2.dll" });
this.testDiscoveryManager.DiscoverTests(this.discoveryCriteria, null);
// Also verify that we have waited for client connection.
this.mockRequestSender.Verify(s => s.InitializeDiscovery(extensions), Times.Once);
}
finally
{
TestPluginCache.Instance = null;
}
}
public void LoadTestsShouldCallIntoOtherDiscoverersWhenCreatingOneFails()
{
TestPluginCacheHelper.SetupMockExtensions(
new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string>
{
"test1.csv",
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location
};
var extensionSourceMap = new Dictionary <string, IEnumerable <string> >();
extensionSourceMap.Add("_none_", sources);
var runSettings = this.runSettingsMock.Object;
string testCaseFilter = "TestFilter";
this.discovererEnumerator.LoadTests(extensionSourceMap, runSettings, testCaseFilter, this.messageLoggerMock.Object);
Assert.IsTrue(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled);
Assert.IsFalse(SingletonTestDiscoverer.IsDiscoverTestCalled);
// Also validate that the right set of arguments were passed on to the discoverer.
CollectionAssert.AreEqual(new List <string> {
sources[1]
}, ManagedDllTestDiscoverer.Sources.ToList());
Assert.AreEqual(runSettings, ManagedDllTestDiscoverer.DiscoveryContext.RunSettings);
Assert.AreEqual(testCaseFilter, (ManagedDllTestDiscoverer.DiscoveryContext as DiscoveryContext).FilterExpressionWrapper.FilterString);
Assert.AreEqual(this.messageLoggerMock.Object, ManagedDllTestDiscoverer.MessageLogger);
Assert.IsNotNull(ManagedDllTestDiscoverer.DiscoverySink);
}
public void DiscoverTestsShouldLogIfTheSameSourceIsSpecifiedTwice()
{
TestPluginCacheHelper.SetupMockExtensions(
new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string>
{
typeof(DiscoveryManagerTests).GetTypeInfo().Assembly.Location,
typeof(DiscoveryManagerTests).GetTypeInfo().Assembly.Location
};
var criteria = new DiscoveryCriteria(sources, 100, null);
var mockLogger = new Mock <ITestDiscoveryEventsHandler2>();
this.discoveryManager.DiscoverTests(criteria, mockLogger.Object);
var errorMessage = string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DuplicateSource, sources[0]);
mockLogger.Verify(
l =>
l.HandleLogMessage(
Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.TestMessageLevel.Warning,
errorMessage),
Times.Once);
}
public void TestInit()
{
this.testableTestRunCache = new TestableTestRunCache();
this.testExecutionContext = new TestExecutionContext(
frequencyOfRunStatsChangeEvent: 100,
runStatsChangeEventTimeout: TimeSpan.MaxValue,
inIsolation: false,
keepAlive: false,
isDataCollectionEnabled: false,
areTestCaseLevelEventsRequired: false,
hasTestRun: false,
isDebug: false,
testCaseFilter: null,
filterOptions: null);
this.mockTestRunEventsHandler = new Mock <ITestRunEventsHandler>();
this.mockMetricsCollection = new Mock <IMetricsCollection>();
this.mockRequestData = new Mock <IRequestData>();
this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(this.mockMetricsCollection.Object);
TestPluginCacheHelper.SetupMockExtensions(
new string[] { typeof(RunTestsWithSourcesTests).GetTypeInfo().Assembly.Location },
() => { });
TestPluginCache.Instance.DiscoverTestExtensions <TestExecutorPluginInformation, ITestExecutor>(TestPlatformConstants.TestAdapterEndsWithPattern);
TestPluginCache.Instance.DiscoverTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer>(TestPlatformConstants.TestAdapterEndsWithPattern);
}
public void LoadTestsShouldCallBothNativeAndManagedDiscoverersWithCorrectSources()
{
TestPluginCacheHelper.SetupMockExtensions(
new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
this.mockAssemblyProperties.Setup(pe => pe.GetAssemblyType("native.dll")).Returns(AssemblyType.Native);
this.mockAssemblyProperties.Setup(pe => pe.GetAssemblyType("managed.dll")).Returns(AssemblyType.Managed);
var nativeSources = new List <string>
{
"native.dll"
};
var managedSources = new List <string>
{
"managed.dll"
};
var extensionSourceMap = new Dictionary <string, IEnumerable <string> >();
extensionSourceMap.Add("_none_", nativeSources.Concat(managedSources));
string testCaseFilter = "TestFilter";
this.discovererEnumerator.LoadTests(extensionSourceMap, this.runSettingsMock.Object, testCaseFilter, this.messageLoggerMock.Object);
Assert.IsTrue(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled);
CollectionAssert.AreEqual(managedSources, ManagedDllTestDiscoverer.Sources.ToList());
Assert.IsTrue(NativeDllTestDiscoverer.IsNativeDiscoverTestCalled);
CollectionAssert.AreEqual(nativeSources, NativeDllTestDiscoverer.Sources.ToList());
Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled);
}
public void GetTestExtensionsShouldDiscoverExtensionsOnlyOnce()
{
var discoveryCount = 0;
TestPluginCacheHelper.SetupMockExtensions(typeof(TestPluginManagerTests), () => { discoveryCount++; });
IEnumerable <LazyExtension <ITestDiscoverer, Dictionary <string, object> > > unfilteredTestExtensions;
IEnumerable <LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> > testExtensions;
TestPluginManager.Instance.GetSpecificTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer, ITestDiscovererCapabilities, TestDiscovererMetadata>(
TestPlatformConstants.TestAdapterEndsWithPattern,
out unfilteredTestExtensions,
out testExtensions);
// Call this again to verify that discovery is not called again.
TestPluginManager.Instance.GetSpecificTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer, ITestDiscovererCapabilities, TestDiscovererMetadata>(
TestPlatformConstants.TestAdapterEndsWithPattern,
out unfilteredTestExtensions,
out testExtensions);
Assert.IsNotNull(testExtensions);
Assert.IsTrue(testExtensions.Any());
Assert.AreEqual(2, discoveryCount);
}
public TestExtensionManagerTests()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestExtensionManagerTests));
messageLogger = TestSessionMessageLogger.Instance;
TestPluginManager.Instance.GetSpecificTestExtensions <TestLoggerPluginInformation, ITestLogger, ITestLoggerCapabilities, TestLoggerMetadata>
(TestPlatformConstants.TestLoggerEndsWithPattern, out unfilteredTestExtensions, out filteredTestExtensions);
}
public void DiscoverTestsShouldCollectMetrics()
{
var mockMetricsCollector = new Mock <IMetricsCollection>();
var dict = new Dictionary <string, object>();
dict.Add("DummyMessage", "DummyValue");
mockMetricsCollector.Setup(mc => mc.Metrics).Returns(dict);
this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(mockMetricsCollector.Object);
TestPluginCacheHelper.SetupMockExtensions(
new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string>
{
typeof(DiscoveryManagerTests).GetTypeInfo().Assembly.Location
};
var mockLogger = new Mock <ITestDiscoveryEventsHandler2>();
var criteria = new DiscoveryCriteria(sources, 1, null);
// Act.
this.discoveryManager.DiscoverTests(criteria, mockLogger.Object);
// Verify.
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.DiscoveryState, It.IsAny <string>()), Times.Once);
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.TotalTestsDiscovered, It.IsAny <object>()), Times.Once);
}
public void DiscoverTestsShouldInitializeExtensionsWithExistingExtensionsOnly()
{
var inputExtensions = new[] { "abc.TestAdapter.dll", "def.TestAdapter.dll", "xyz.TestAdapter.dll" };
var expectedOutputPaths = new[] { "abc.TestAdapter.dll", "xyz.TestAdapter.dll" };
TestPluginCacheHelper.SetupMockAdditionalPathExtensions(inputExtensions);
this.mockRequestSender.Setup(s => s.WaitForRequestHandlerConnection(It.IsAny <int>(), It.IsAny <CancellationToken>())).Returns(true);
this.mockTestHostManager.Setup(th => th.GetTestPlatformExtensions(It.IsAny <IEnumerable <string> >(), It.IsAny <IEnumerable <string> >())).Returns((IEnumerable <string> sources, IEnumerable <string> extensions) =>
{
return(extensions.Select(extension => { return Path.GetFileName(extension); }));
});
this.mockFileHelper.Setup(fh => fh.Exists(It.IsAny <string>())).Returns((string extensionPath) =>
{
return(!extensionPath.Contains("def.TestAdapter.dll"));
});
this.mockFileHelper.Setup(fh => fh.Exists("def.TestAdapter.dll")).Returns(false);
this.mockFileHelper.Setup(fh => fh.Exists("xyz.TestAdapter.dll")).Returns(true);
var mockTestDiscoveryEventHandler = new Mock <ITestDiscoveryEventsHandler2>();
this.testDiscoveryManager.DiscoverTests(this.discoveryCriteria, mockTestDiscoveryEventHandler.Object);
this.mockRequestSender.Verify(s => s.InitializeDiscovery(expectedOutputPaths), Times.Once);
}
public void CreateShouldDiscoverDiscovererExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestDiscoveryExtensionManagerTests));
var extensionManager = TestDiscoveryExtensionManager.Create();
Assert.IsNotNull(extensionManager.Discoverers);
Assert.IsTrue(extensionManager.Discoverers.Any());
}
public void CreateRunSettingsShouldReturnValidRunSettings()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(RunSettingsUtilitiesTests));
string runsettings = @"<RunSettings><RunConfiguration><ResultsDirectory>.\TestResults</ResultsDirectory></RunConfiguration ><DummyMSTest><FORCEDLEGACYMODE>true</FORCEDLEGACYMODE></DummyMSTest></RunSettings>";
var result = RunSettingsUtilities.CreateAndInitializeRunSettings(runsettings);
Assert.AreEqual("<DummyMSTest><FORCEDLEGACYMODE>true</FORCEDLEGACYMODE></DummyMSTest>", DummyMsTestSetingsProvider.StringToVerify);
TestPluginCacheHelper.ResetExtensionsCache();
}
public void CreateShouldDiscoverExecutorExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestExecutorExtensionManagerTests));
var extensionManager = TestExecutorExtensionManager.Create();
Assert.IsNotNull(extensionManager.TestExtensions);
Assert.IsTrue(extensionManager.TestExtensions.Count() > 0);
}
public void GetTestExtensionsShouldReturnExtensionsInAssembly()
{
TestPluginCacheHelper.SetupMockAdditionalPathExtensions(typeof(TestPluginCacheTests));
TestPluginCache.Instance.GetTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer>(typeof(TestPluginCacheTests).GetTypeInfo().Assembly.Location);
Assert.IsNotNull(TestPluginCache.Instance.TestExtensions);
Assert.IsTrue(TestPluginCache.Instance.TestExtensions.TestDiscoverers.Count > 0);
}
public void CreateShouldDiscoverSettingsProviderExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(SettingsProviderExtensionManagerTests));
var extensionManager = SettingsProviderExtensionManager.Create();
Assert.IsNotNull(extensionManager.SettingsProvidersMap);
Assert.IsTrue(extensionManager.SettingsProvidersMap.Count > 0);
}
public void DiscoverTestExtensionsShouldSetCachedBoolToTrue()
{
TestPluginCacheHelper.SetupMockAdditionalPathExtensions(typeof(TestPluginCacheTests));
TestPluginCache.Instance.DiscoverTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer>(TestPlatformConstants.TestAdapterEndsWithPattern);
Assert.IsTrue(TestPluginCache.Instance.TestExtensions.AreTestDiscoverersCached);
Assert.IsTrue(TestPluginCache.Instance.TestExtensions.AreTestExtensionsCached <TestDiscovererPluginInformation>());
}
public void TryGetTestExtensionShouldThrowExceptionWithNullUri()
{
testExtensionManager = new DummyTestExtensionManager(unfilteredTestExtensions, filteredTestExtensions, messageLogger);
TestPluginCacheHelper.SetupMockAdditionalPathExtensions(typeof(TestExtensionManagerTests));
Assert.ThrowsException <ArgumentNullException>(() =>
{
var result = testExtensionManager.TryGetTestExtension(default(Uri));
}
);
}
public TestEngineTests()
{
TestPluginCacheHelper.SetupMockExtensions(new[] { typeof(TestEngineTests).GetTypeInfo().Assembly.Location }, () => { });
this.mockProcessHelper = new Mock <IProcessHelper>();
this.testableTestRuntimeProvider = new TestableRuntimeProvider(true);
this.mockRequestData = new Mock <IRequestData>();
this.mockMetricsCollection = new Mock <IMetricsCollection>();
this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(this.mockMetricsCollection.Object);
this.mockRequestData.Setup(rd => rd.ProtocolConfig).Returns(this.protocolConfig);
}
public void CreateShouldCacheDiscoveredExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestExecutorExtensionManagerTests), () => { });
var extensionManager = TestExecutorExtensionManager.Create();
TestExecutorExtensionManager.Create();
Assert.IsNotNull(extensionManager.TestExtensions);
Assert.IsTrue(extensionManager.TestExtensions.Any());
}
public void CreateShouldCacheDiscoveredExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestDiscoveryExtensionManagerTests), () => { });
var extensionManager = TestDiscoveryExtensionManager.Create();
TestDiscoveryExtensionManager.Create();
Assert.IsNotNull(extensionManager.Discoverers);
Assert.IsTrue(extensionManager.Discoverers.Count() > 0);
}
public void DiscoverTestExtensionsShouldDiscoverExtensionsFromExtensionsFolder()
{
TestPluginCacheHelper.SetupMockAdditionalPathExtensions(typeof(TestPluginCacheTests));
TestPluginCache.Instance.DiscoverTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer>(TestPlatformConstants.TestAdapterEndsWithPattern);
Assert.IsNotNull(TestPluginCache.Instance.TestExtensions);
// Validate the discoverers to be absolutely certain.
Assert.IsTrue(TestPluginCache.Instance.TestExtensions.TestDiscoverers.Count > 0);
}
public void LoadAndInitializeShouldInitializeAllExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestExecutorExtensionManagerTests));
TestExecutorExtensionManager.LoadAndInitializeAllExtensions(false);
var allExecutors = TestExecutorExtensionManager.Create().TestExtensions;
foreach (var executor in allExecutors)
{
Assert.IsTrue(executor.IsExtensionCreated);
}
}
public void LoadAndInitializeShouldInitializeAllExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestDiscoveryExtensionManagerTests));
TestDiscoveryExtensionManager.LoadAndInitializeAllExtensions(false);
var allDiscoverers = TestDiscoveryExtensionManager.Create().Discoverers;
foreach (var discoverer in allDiscoverers)
{
Assert.IsTrue(discoverer.IsExtensionCreated);
}
}
public void GetTestExtensionsShouldReturnTestDiscovererExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestPluginManagerTests));
TestPluginManager.Instance.GetSpecificTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer, ITestDiscovererCapabilities, TestDiscovererMetadata>(
TestPlatformConstants.TestAdapterEndsWithPattern,
out var unfilteredTestExtensions,
out var testExtensions);
Assert.IsNotNull(unfilteredTestExtensions);
Assert.IsNotNull(testExtensions);
Assert.IsTrue(testExtensions.Any());
}
public void LoadAndInitializeShouldInitializeAllExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(SettingsProviderExtensionManagerTests));
SettingsProviderExtensionManager.LoadAndInitializeAllExtensions(false);
var settingsProviders = SettingsProviderExtensionManager.Create().SettingsProvidersMap.Values;
foreach (var provider in settingsProviders)
{
Assert.IsTrue(provider.IsExtensionCreated);
}
}
public DiscovererEnumeratorTests()
{
this.mockTestPlatformEventSource = new Mock <ITestPlatformEventSource>();
this.discoveryResultCache = new DiscoveryResultCache(1000, TimeSpan.FromHours(1), (tests) => { });
this.mockRequestData = new Mock <IRequestData>();
this.mockMetricsCollection = new Mock <IMetricsCollection>();
this.mockAssemblyProperties = new Mock <IAssemblyProperties>();
this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(this.mockMetricsCollection.Object);
this.discovererEnumerator = new DiscovererEnumerator(this.mockRequestData.Object, this.discoveryResultCache, this.mockTestPlatformEventSource.Object, this.mockAssemblyProperties.Object, this.cancellationTokenSource.Token);
this.runSettingsMock = new Mock <IRunSettings>();
this.messageLoggerMock = new Mock <IMessageLogger>();
TestPluginCacheHelper.SetupMockExtensions(new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
TestDiscoveryExtensionManager.Destroy();
}
public void GetTestExtensionsShouldReturnTestDiscovererExtensions()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(TestPluginManagerTests));
IEnumerable <LazyExtension <ITestDiscoverer, Dictionary <string, object> > > unfilteredTestExtensions;
IEnumerable <LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> > testExtensions;
TestPluginManager.Instance.GetSpecificTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer, ITestDiscovererCapabilities, TestDiscovererMetadata>(
TestPlatformConstants.TestAdapterEndsWithPattern,
out unfilteredTestExtensions,
out testExtensions);
Assert.IsNotNull(unfilteredTestExtensions);
Assert.IsNotNull(testExtensions);
Assert.IsTrue(testExtensions.Count() > 0);
}
public void InitializeSettingsProvidersShouldThrowIfSettingsProviderLoadThrows()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(RunSettingsTests));
var runSettings = new RunSettings();
runSettings.InitializeSettingsProviders(this.GetRunSettingsWithBadSettingsNodes());
Action action =
() => runSettings.GetSettings("BadSettings");
Assert.ThrowsException <SettingsException>(
action,
"An error occurred while initializing the settings provider named '{0}'",
"BadSettings");
}
public void InitializeSettingsProvidersShouldThrowIfNodeInRunSettingsDoesNotHaveAProvider()
{
TestPluginCacheHelper.SetupMockExtensions(typeof(RunSettingsTests));
var runSettings = new RunSettings();
runSettings.InitializeSettingsProviders(this.GetRunSettingsWithUndefinedSettingsNodes());
Action action =
() => runSettings.GetSettings("OrphanNode");
Assert.ThrowsException <SettingsException>(
action,
"Settings Provider named '{0}' was not found. The settings can not be loaded.",
"OrphanNode");
}
