public void LoadTestsShouldCallIntoOtherDiscoverersWhenCreatingOneFails()
{
try
{
TestPluginCacheTests.SetupMockExtensions(
new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string>
{
"test1.csv",
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location
};
var extensionSourceMap = new Dictionary <string, IEnumerable <string> >();
extensionSourceMap.Add("_none_", sources);
var settings = new Mock <IRunSettings>().Object;
var logger = new Mock <IMessageLogger>().Object;
string testCaseFilter = "TestFilter";
this.discovererEnumerator.LoadTests(extensionSourceMap, settings, testCaseFilter, logger);
Assert.IsTrue(DllTestDiscoverer.IsDiscoverTestCalled);
Assert.IsFalse(SingletonTestDiscoverer.IsDiscoverTestCalled);
// Also validate that the right set of arguments were passed on to the discoverer.
CollectionAssert.AreEqual(new List <string> {
sources[1]
}, DllTestDiscoverer.Sources.ToList());
Assert.AreEqual(settings, DllTestDiscoverer.DiscoveryContext.RunSettings);
Assert.AreEqual(testCaseFilter, (DllTestDiscoverer.DiscoveryContext as DiscoveryContext).FilterExpressionWrapper.FilterString);
Assert.AreEqual(logger, DllTestDiscoverer.MessageLogger);
Assert.IsNotNull(DllTestDiscoverer.DiscoverySink);
}
finally
{
this.ResetDiscoverers();
}
}
public void StartTestRunShouldRunTestsForTheProvidedTests()
{
var assemblyLocation = typeof(ExecutionManagerTests).GetTypeInfo().Assembly.Location;
var tests = new List <TestCase>
{
new TestCase("A.C.M1", new Uri(RunTestsWithSourcesTestsExecutorUri), assemblyLocation)
};
var mockTestRunEventsHandler = new Mock <ITestRunEventsHandler>();
var isExecutorCalled = false;
RunTestWithSourcesExecutor.RunTestsWithTestsCallback = (s, rc, fh) =>
{
isExecutorCalled = true;
var tr =
new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(
new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase(
"A.C.M",
new Uri(RunTestsWithSourcesTestsExecutorUri),
"A.dll"));
fh.RecordResult(tr);
};
TestPluginCacheTests.SetupMockExtensions(new string[] { assemblyLocation }, () => { });
this.executionManager.StartTestRun(tests, null, null, testExecutionContext, null, mockTestRunEventsHandler.Object);
Assert.IsTrue(isExecutorCalled);
mockTestRunEventsHandler.Verify(
treh => treh.HandleTestRunComplete(It.IsAny <TestRunCompleteEventArgs>(),
It.IsAny <TestRunChangedEventArgs>(),
It.IsAny <ICollection <AttachmentSet> >(),
It.IsAny <ICollection <string> >()), Times.Once);
// Also verify that run stats are passed through.
mockTestRunEventsHandler.Verify(treh => treh.HandleTestRunStatsChange(It.IsAny <TestRunChangedEventArgs>()), Times.Once);
}
public void LoadTestsShouldCollectMetrics()
{
var mockMetricsCollector = new Mock <IMetricsCollection>();
var dict = new Dictionary <string, object>();
dict.Add("DummyMessage", "DummyValue");
TestPluginCacheTests.SetupMockExtensions(
new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string>
{
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location,
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location
};
var extensionSourceMap = new Dictionary <string, IEnumerable <string> >();
extensionSourceMap.Add("_none_", sources);
mockMetricsCollector.Setup(mc => mc.Metrics).Returns(dict);
this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(mockMetricsCollector.Object);
string testCaseFilter = "TestFilter";
this.discovererEnumerator.LoadTests(extensionSourceMap, this.runSettingsMock.Object, testCaseFilter, this.messageLoggerMock.Object);
// Verify.
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.TimeTakenInSecByAllAdapters, It.IsAny <object>()), Times.Once);
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.NumberOfAdapterUsedToDiscoverTests, It.IsAny <object>()), Times.Once);
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.NumberOfAdapterDiscoveredDuringDiscovery, It.IsAny <object>()), Times.Once);
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.TotalTestsByAdapter + ".discoverer://manageddlldiscoverer/", It.IsAny <object>()), Times.Once);
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter + ".discoverer://manageddlldiscoverer/", It.IsAny <object>()), Times.Once);
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.TotalTestsByAdapter + ".discoverer://nativedlldiscoverer/", It.IsAny <object>()), Times.Once);
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter + ".discoverer://nativedlldiscoverer/", It.IsAny <object>()), Times.Once);
mockMetricsCollector.Verify(rd => rd.Add(TelemetryDataConstants.TimeTakenToLoadAdaptersInSec, It.IsAny <object>()), Times.Once);
}
public void DiscoverTestsShouldSendMetricsOnDiscoveryComplete()
{
var metricsCollector = new MetricsCollection();
metricsCollector.Add("DummyMessage", "DummyValue");
this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(metricsCollector);
DiscoveryCompleteEventArgs receivedDiscoveryCompleteEventArgs = null;
TestPluginCacheTests.SetupMockExtensions(
new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string>
{
typeof(DiscoveryManagerTests).GetTypeInfo().Assembly.Location
};
var mockLogger = new Mock <ITestDiscoveryEventsHandler2>();
var criteria = new DiscoveryCriteria(sources, 1, null);
mockLogger.Setup(ml => ml.HandleDiscoveryComplete(It.IsAny <DiscoveryCompleteEventArgs>(), It.IsAny <IEnumerable <TestCase> >()))
.Callback(
(DiscoveryCompleteEventArgs complete,
IEnumerable <TestCase> tests) =>
{
receivedDiscoveryCompleteEventArgs = complete;
});
// Act.
this.discoveryManager.DiscoverTests(criteria, mockLogger.Object);
// Assert
Assert.IsNotNull(receivedDiscoveryCompleteEventArgs.Metrics);
Assert.IsTrue(receivedDiscoveryCompleteEventArgs.Metrics.Any());
Assert.IsTrue(receivedDiscoveryCompleteEventArgs.Metrics.ContainsKey("DummyMessage"));
}
public void TestInit()
{
this.testableTestRunCache = new TestableTestRunCache();
this.testExecutionContext = new TestExecutionContext(
frequencyOfRunStatsChangeEvent: 100,
runStatsChangeEventTimeout: TimeSpan.MaxValue,
inIsolation: false,
keepAlive: false,
isDataCollectionEnabled: false,
areTestCaseLevelEventsRequired: false,
hasTestRun: false,
isDebug: false,
testCaseFilter: null);
this.mockTestRunEventsHandler = new Mock <ITestRunEventsHandler>();
this.requestData = new RequestData(new NoOpMetricsCollection());
TestPluginCacheTests.SetupMockExtensions(
new string[] { typeof(RunTestsWithSourcesTests).GetTypeInfo().Assembly.Location },
() => { });
TestPluginCache.Instance.DiscoverTestExtensions <TestExecutorPluginInformation, ITestExecutor>(TestPlatformConstants.TestAdapterEndsWithPattern);
TestPluginCache.Instance.DiscoverTestExtensions <TestDiscovererPluginInformation, ITestDiscoverer>(TestPlatformConstants.TestAdapterEndsWithPattern);
}
public void LoadTestsShouldReportWarningOnNoDiscoverers()
{
TestPluginCacheTests.SetupMockExtensions(
new string[] { typeof(TestPluginCache).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string> {
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location
};
var mockLogger = new Mock <IMessageLogger>();
var extensionSourceMap = new Dictionary <string, IEnumerable <string> >();
extensionSourceMap.Add("_none_", sources);
this.discovererEnumerator.LoadTests(extensionSourceMap, new Mock <IRunSettings>().Object, mockLogger.Object);
mockLogger.Verify(
l =>
l.SendMessage(
TestMessageLevel.Warning,
"No test discoverer is registered to perform discovery of test cases. Register a test discoverer and try again."),
Times.Once);
}
public void InitializeSettingsProvidersShouldLoadSettingsIntoMultipleSettingsProviders()
{
TestPluginCacheTests.SetupMockExtensions();
var runSettings = new RunSettings();
runSettings.InitializeSettingsProviders(this.GetRunSettingsWithRunConfigurationAndMSTestNode());
var rcSettingsProvider = runSettings.GetSettings("RunConfiguration");
var mstestSettingsProvider = runSettings.GetSettings("MSTest");
Assert.IsNotNull(rcSettingsProvider);
Assert.IsTrue(rcSettingsProvider is RunConfigurationSettingsProvider);
Assert.AreEqual(
"<RunConfiguration><Architecture>x86</Architecture></RunConfiguration>",
(rcSettingsProvider as RunConfigurationSettingsProvider).SettingsTree);
Assert.IsNotNull(mstestSettingsProvider);
Assert.IsTrue(mstestSettingsProvider is MSTestSettingsProvider);
Assert.AreEqual(
"<MSTest><NoAppDomain>true</NoAppDomain></MSTest>",
(mstestSettingsProvider as MSTestSettingsProvider).SettingsTree);
}
public void LoadTestsShouldReportWarningOnNoDiscoverers()
{
TestPluginCacheTests.SetupMockExtensions(
new string[] { typeof(TestPluginCache).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string> {
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location
};
var extensionSourceMap = new Dictionary <string, IEnumerable <string> >();
extensionSourceMap.Add("_none_", sources);
this.discovererEnumerator.LoadTests(extensionSourceMap, this.runSettingsMock.Object, null, this.messageLoggerMock.Object);
var messageFormat =
"No test is available in {0}. Make sure that test discoverer & executors are registered and platform & framework version settings are appropriate and try again.";
var message = string.Format(messageFormat, string.Join(" ", sources));
this.messageLoggerMock.Verify(
l =>
l.SendMessage(TestMessageLevel.Warning, message), Times.Once);
}
public BaseRunTestsTests()
{
this.testExecutionContext = new TestExecutionContext(
frequencyOfRunStatsChangeEvent: 100,
runStatsChangeEventTimeout: TimeSpan.MaxValue,
inIsolation: false,
keepAlive: false,
isDataCollectionEnabled: false,
areTestCaseLevelEventsRequired: false,
hasTestRun: false,
isDebug: false,
testCaseFilter: string.Empty,
filterOptions: null);
this.mockTestRunEventsHandler = new Mock <ITestRunEventsHandler>();
this.mockTestPlatformEventSource = new Mock <ITestPlatformEventSource>();
this.mockRequestData = new Mock <IRequestData>();
this.mockMetricsCollection = new Mock <IMetricsCollection>();
this.mockThread = new Mock <IThread>();
this.mockDataSerializer = new Mock <IDataSerializer>();
this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(this.mockMetricsCollection.Object);
this.runTestsInstance = new TestableBaseRunTests(
this.mockRequestData.Object,
null,
null,
this.testExecutionContext,
null,
this.mockTestRunEventsHandler.Object,
this.mockTestPlatformEventSource.Object,
null,
new PlatformThread(),
this.mockDataSerializer.Object);
TestPluginCacheTests.SetupMockExtensions(new string[] { typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location }, () => { });
}
public void LoadTestsShouldNotCallIntoDiscoverersWhenCancelled()
{
// Setup
string[] extensions = new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location };
TestPluginCacheTests.SetupMockExtensions(extensions, () => { });
var dllsources = new List <string>
{
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location,
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location
};
var jsonsources = new List <string>
{
"test1.json",
"test2.json"
};
var sources = new List <string>(dllsources);
sources.AddRange(jsonsources);
var extensionSourceMap = new Dictionary <string, IEnumerable <string> >
{
{ "_none_", sources }
};
// Act
this.cancellationTokenSource.Cancel();
var runSettings = this.runSettingsMock.Object;
string testCaseFilter = "TestFilter";
this.discovererEnumerator.LoadTests(extensionSourceMap, runSettings, testCaseFilter, this.messageLoggerMock.Object);
// Validate
Assert.IsFalse(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled);
Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled);
this.messageLoggerMock.Verify(logger => logger.SendMessage(TestMessageLevel.Warning, "Discovery of tests cancelled."), Times.Once);
}
public void LoadTestsShouldCallIntoADiscovererThatMatchesTheSources()
{
try
{
TestPluginCacheTests.SetupMockExtensions(
new string[] { typeof(DiscovererEnumeratorTests).GetTypeInfo().Assembly.Location },
() => { });
var sources = new List <string>
{
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location,
typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location
};
var extensionSourceMap = new Dictionary <string, IEnumerable <string> >();
extensionSourceMap.Add("_none_", sources);
var settings = new Mock <IRunSettings>().Object;
var logger = new Mock <IMessageLogger>().Object;
this.discovererEnumerator.LoadTests(extensionSourceMap, settings, logger);
Assert.IsTrue(DllTestDiscoverer.IsDiscoverTestCalled);
Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled);
// Also validate that the right set of arguments were passed on to the discoverer.
CollectionAssert.AreEqual(sources, DllTestDiscoverer.Sources.ToList());
Assert.AreEqual(settings, DllTestDiscoverer.DiscoveryContext.RunSettings);
Assert.AreEqual(logger, DllTestDiscoverer.MessageLogger);
Assert.IsNotNull(DllTestDiscoverer.DiscoverySink);
}
finally
{
this.ResetDiscoverers();
}
}
public void InitializeShouldInitializeExtensionsIfPresent()
{
// Make sure TestPlugincache is refreshed.
TestPluginCache.Instance = null;
try
{
var extensions = new[] { "c:\\e1.dll", "c:\\e2.dll" };
TestPluginCacheTests.SetupMockAdditionalPathExtensions(extensions);
this.mockRequestSender.Setup(s => s.WaitForRequestHandlerConnection(It.IsAny <int>())).Returns(true);
this.testExecutionManager.Initialize();
// Also verify that we have waited for client connection.
this.mockRequestSender.Verify(s => s.WaitForRequestHandlerConnection(It.IsAny <int>()), Times.Once);
this.mockRequestSender.Verify(
s => s.InitializeExecution(extensions, true),
Times.Once);
}
finally
{
TestPluginCache.Instance = null;
}
}
public void DiscoverTestsShouldInitializeExtensionsIfPresent()
{
// Make sure TestPlugincache is refreshed.
TestPluginCache.Instance = null;
try
{
var extensions = new[] { "c:\\e1.dll", "c:\\e2.dll" };
// Setup Mocks.
TestPluginCacheTests.SetupMockAdditionalPathExtensions(extensions);
this.mockRequestSender.Setup(s => s.WaitForRequestHandlerConnection(It.IsAny <int>())).Returns(true);
this.mockTestHostManager.Setup(th => th.GetTestPlatformExtensions(It.IsAny <IEnumerable <string> >(), It.IsAny <IEnumerable <string> >())).Returns(new[] { "c:\\e1.dll", "c:\\e2.dll" });
this.testDiscoveryManager.DiscoverTests(this.discoveryCriteria, null);
// Also verify that we have waited for client connection.
this.mockRequestSender.Verify(s => s.InitializeDiscovery(extensions), Times.Once);
}
finally
{
TestPluginCache.Instance = null;
}
}
public void RunTestsShouldInvokeTheTestExecutorIfAdapterAssemblyIsUnknown()
{
var executorUriExtensionMap = new List <Tuple <Uri, string> >
{
new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), Microsoft.VisualStudio.TestPlatform.ObjectModel.Constants.UnspecifiedAdapterPath)
};
LazyExtension <ITestExecutor, ITestExecutorCapabilities> receivedExecutor = null;
// Setup mocks.
this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); };
this.runTestsInstance.InvokeExecutorCallback =
(executor, executorUriExtensionTuple, runContext, frameworkHandle) =>
{
receivedExecutor = executor;
};
TestPluginCacheTests.SetupMockExtensions(
new string[] { typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location },
() => { });
this.runTestsInstance.RunTests();
Assert.IsNotNull(receivedExecutor);
Assert.AreEqual(BaseRunTestsExecutorUri, receivedExecutor.Metadata.ExtensionUri);
}
public void TestCleanup()
{
TestPluginCacheTests.ResetExtensionsCache();
TestSessionMessageLogger.Instance = null;
}
public void Cleanup()
{
TestExecutorExtensionManager.Destroy();
TestPluginCacheTests.ResetExtensionsCache();
}
public TestHostProviderManagerTests()
{
TestPluginCacheTests.SetupMockExtensions();
}
public void Initialize()
{
TestPluginCacheTests.SetupMockExtensions();
}
public TestEngineTests()
{
TestPluginCacheTests.SetupMockExtensions(new[] { typeof(TestEngineTests).GetTypeInfo().Assembly.Location }, () => { });
this.testEngine = new TestableTestEngine();
this.testableTestRuntimeProvider = new TestableRuntimeProvider(true);
}
public void TestCleanup()
{
RunTestWithSourcesExecutor.RunTestsWithSourcesCallback = null;
TestPluginCacheTests.ResetExtensionsCache();
}
