private void LoadTestsFromAnExtension(string extensionAssembly, IEnumerable <string> sources, IRunSettings settings, string testCaseFilter, IMessageLogger logger)
{
// Stopwatch to collect metrics
var timeStart = DateTime.UtcNow;
var discovererToSourcesMap = GetDiscovererToSourcesMap(extensionAssembly, sources, logger, this.assemblyProperties);
var totalAdapterLoadTIme = DateTime.UtcNow - timeStart;
// Collecting Data Point for TimeTaken to Load Adapters
this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenToLoadAdaptersInSec, totalAdapterLoadTIme.TotalSeconds);
// Warning is logged for in the inner function
if (discovererToSourcesMap == null || !discovererToSourcesMap.Any())
{
return;
}
double totalTimeTakenByAdapters = 0;
double totalAdaptersUsed = 0;
try
{
// Collecting Total Number of Adapters Discovered in Machine
this.requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAdapterDiscoveredDuringDiscovery, discovererToSourcesMap.Keys.Count());
var context = new DiscoveryContext {
RunSettings = settings
};
context.FilterExpressionWrapper = !string.IsNullOrEmpty(testCaseFilter) ? new FilterExpressionWrapper(testCaseFilter) : null;
// Set on the logger the TreatAdapterErrorAsWarning setting from runsettings.
this.SetAdapterLoggingSettings(logger, settings);
var discoverySink = new TestCaseDiscoverySink(this.discoveryResultCache);
foreach (var discoverer in discovererToSourcesMap.Keys)
{
if (this.cancellationToken.IsCancellationRequested)
{
EqtTrace.Info("DiscovererEnumerator.LoadTestsFromAnExtension: Cancellation Requested. Aborting the discovery");
LogTestsDiscoveryCancellation(logger);
return;
}
var result = this.DiscoverTestsFromSingleDiscoverer(discoverer, discovererToSourcesMap[discoverer], context, discoverySink, logger, cancellationToken);
totalAdaptersUsed += result.TotalAdaptersUsed;
totalTimeTakenByAdapters += result.TotalTimeSpentInAdapaters;
}
if (this.discoveryResultCache.TotalDiscoveredTests == 0)
{
LogWarningOnNoTestsDiscovered(sources, testCaseFilter, logger);
}
}
finally
{
this.CollectTelemetryAtEnd(totalTimeTakenByAdapters, totalAdaptersUsed);
}
}
private DiscoveryResult DiscoverTestsFromSingleDiscoverer(
LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> discoverer,
IEnumerable <string> sources,
DiscoveryContext context,
TestCaseDiscoverySink discoverySink,
IMessageLogger logger,
CancellationToken cancellationToken)
{
var result = new DiscoveryResult();
if (DiscovererEnumerator.TryToLoadDiscoverer(discoverer, logger, out var discovererType) == false)
{
// Fail to instantiate the discoverer type.
return(result);
}
// on instantiated successfully, get tests
try
{
EqtTrace.Verbose(
"DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Loading tests for {0}",
discoverer.Value.GetType().FullName);
var currentTotalTests = this.discoveryResultCache.TotalDiscoveredTests;
var newTimeStart = DateTime.UtcNow;
this.testPlatformEventSource.AdapterDiscoveryStart(discoverer.Metadata.DefaultExecutorUri.AbsoluteUri);
foreach (var testSource in sources)
{
if (cancellationToken.IsCancellationRequested)
{
EqtTrace.Info("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Cancellation Requested. Aborting the discovery");
break;
}
discoverer.Value.DiscoverTests(new[] { testSource }, context, logger, discoverySink);
}
var totalAdapterRunTime = DateTime.UtcNow - newTimeStart;
this.testPlatformEventSource.AdapterDiscoveryStop(this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests);
// Record Total Tests Discovered By each Discoverer.
var totalTestsDiscoveredByCurrentDiscoverer = this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests;
this.requestData.MetricsCollection.Add(
string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsByAdapter,
discoverer.Metadata.DefaultExecutorUri), totalTestsDiscoveredByCurrentDiscoverer);
result.TotalAdaptersUsed++;
EqtTrace.Verbose("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Done loading tests for {0}",
discoverer.Value.GetType().FullName);
var discovererFromDeprecatedLocations = DiscovererEnumerator.IsDiscovererFromDeprecatedLocations(discoverer);
if (discovererFromDeprecatedLocations)
{
logger.SendMessage(TestMessageLevel.Warning,
string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DeprecatedAdapterPath));
}
// Collecting Data Point for Time Taken to Discover Tests by each Adapter
this.requestData.MetricsCollection.Add($"{TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter}.{discoverer.Metadata.DefaultExecutorUri}", totalAdapterRunTime.TotalSeconds);
result.TotalTimeSpentInAdapaters += totalAdapterRunTime.TotalSeconds;
}
catch (Exception e)
{
var message = string.Format(CultureInfo.CurrentUICulture, CrossPlatEngineResources.ExceptionFromLoadTests, discovererType.Name, e.Message);
logger.SendMessage(TestMessageLevel.Error, message);
EqtTrace.Error("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: {0} ", e);
}
return(result);
}
private void LoadTestsFromAnExtension(string extensionAssembly, IEnumerable <string> sources, IRunSettings settings, string testCaseFilter, IMessageLogger logger)
{
double totalAdaptersUsed = 0;
// Stopwatch to collect metrics
var timeStart = DateTime.UtcNow;
var discovererToSourcesMap = GetDiscovererToSourcesMap(extensionAssembly, sources, logger);
var totalAdapterLoadTIme = DateTime.UtcNow - timeStart;
// Collecting Data Point for TimeTaken to Load Adapters
this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenToLoadAdaptersInSec, totalAdapterLoadTIme.TotalSeconds);
// Warning is logged for in the inner function
if (discovererToSourcesMap == null || !discovererToSourcesMap.Any())
{
return;
}
// Collecting Total Number of Adapters Discovered in Machine
this.requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAdapterDiscoveredDuringDiscovery, discovererToSourcesMap.Keys.Count());
var context = new DiscoveryContext {
RunSettings = settings
};
context.FilterExpressionWrapper = !string.IsNullOrEmpty(testCaseFilter) ? new FilterExpressionWrapper(testCaseFilter) : null;
// Set on the logger the TreatAdapterErrorAsWarning setting from runsettings.
this.SetAdapterLoggingSettings(logger, settings);
var discoverySink = new TestCaseDiscoverySink(this.discoveryResultCache);
double totalTimeTakenByAdapters = 0;
foreach (var discoverer in discovererToSourcesMap.Keys)
{
Type discovererType = null;
// See if discoverer can be instantiated successfully else move next.
try
{
discovererType = discoverer.Value.GetType();
}
catch (Exception e)
{
var mesage = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.DiscovererInstantiationException,
e.Message);
logger.SendMessage(TestMessageLevel.Warning, mesage);
EqtTrace.Error("DiscovererEnumerator.LoadTestsFromAnExtension: {0} ", e);
continue;
}
// if instantiated successfully, get tests
try
{
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose(
"DiscoveryContext.LoadTests: Loading tests for {0}",
discoverer.Value.GetType().FullName);
}
var currentTotalTests = this.discoveryResultCache.TotalDiscoveredTests;
var newTimeStart = DateTime.UtcNow;
this.testPlatformEventSource.AdapterDiscoveryStart(discoverer.Metadata.DefaultExecutorUri.AbsoluteUri);
discoverer.Value.DiscoverTests(discovererToSourcesMap[discoverer], context, logger, discoverySink);
var totalAdapterRunTime = DateTime.UtcNow - newTimeStart;
this.testPlatformEventSource.AdapterDiscoveryStop(this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests);
// Collecting Total Tests Discovered By each Adapter.
if (this.discoveryResultCache.TotalDiscoveredTests > currentTotalTests)
{
var totalDiscoveredTests = this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests;
this.requestData.MetricsCollection.Add(string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsByAdapter, discoverer.Metadata.DefaultExecutorUri), totalDiscoveredTests);
totalAdaptersUsed++;
}
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose(
"DiscoveryContext.LoadTests: Done loading tests for {0}",
discoverer.Value.GetType().FullName);
}
// Collecting Data Point for Time Taken to Discover Tests by each Adapter
this.requestData.MetricsCollection.Add(string.Format("{0}.{1}", TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter, discoverer.Metadata.DefaultExecutorUri), totalAdapterRunTime.TotalSeconds);
totalTimeTakenByAdapters += totalAdapterRunTime.TotalSeconds;
}
catch (Exception e)
{
var message = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.ExceptionFromLoadTests,
discovererType.Name,
e.Message);
logger.SendMessage(TestMessageLevel.Error, message);
EqtTrace.Error("DiscovererEnumerator.LoadTestsFromAnExtension: {0} ", e);
}
}
// Collecting Total Time Taken by Adapters
this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenInSecByAllAdapters, totalTimeTakenByAdapters);
// Collecting Total Adapters Used to Discover tests
this.requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAdapterUsedToDiscoverTests, totalAdaptersUsed);
}
private void DiscoverTestsFromSingleDiscoverer(
LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities> discoverer,
Dictionary <LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>, IEnumerable <string> > discovererToSourcesMap,
DiscoveryContext context,
TestCaseDiscoverySink discoverySink,
IMessageLogger logger,
ref double totalAdaptersUsed,
ref double totalTimeTakenByAdapters)
{
if (DiscovererEnumerator.TryToLoadDiscoverer(discoverer, logger, out var discovererType) == false)
{
// Fail to instantiate the discoverer type.
return;
}
// on instantiated successfully, get tests
try
{
EqtTrace.Verbose(
"DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Loading tests for {0}",
discoverer.Value.GetType().FullName);
if (discoverer.Metadata.DefaultExecutorUri == null)
{
throw new Exception($@"DefaultExecutorUri is null, did you decorate the discoverer class with [DefaultExecutorUri()] attribute? For example [DefaultExecutorUri(""executor://example.testadapter"")].");
}
var currentTotalTests = this.discoveryResultCache.TotalDiscoveredTests;
var newTimeStart = DateTime.UtcNow;
this.testPlatformEventSource.AdapterDiscoveryStart(discoverer.Metadata.DefaultExecutorUri.AbsoluteUri);
discoverer.Value.DiscoverTests(discovererToSourcesMap[discoverer], context, logger, discoverySink);
var totalAdapterRunTime = DateTime.UtcNow - newTimeStart;
this.testPlatformEventSource.AdapterDiscoveryStop(this.discoveryResultCache.TotalDiscoveredTests -
currentTotalTests);
// Record Total Tests Discovered By each Discoverer.
var totalTestsDiscoveredByCurrentDiscoverer = this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests;
this.requestData.MetricsCollection.Add(
string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsByAdapter,
discoverer.Metadata.DefaultExecutorUri), totalTestsDiscoveredByCurrentDiscoverer);
totalAdaptersUsed++;
EqtTrace.Verbose("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Done loading tests for {0}",
discoverer.Value.GetType().FullName);
var discovererFromDeprecatedLocations = DiscovererEnumerator.IsDiscovererFromDeprecatedLocations(discoverer);
if (discovererFromDeprecatedLocations)
{
logger.SendMessage(TestMessageLevel.Warning,
string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DeprecatedAdapterPath));
}
// Collecting Data Point for Time Taken to Discover Tests by each Adapter
this.requestData.MetricsCollection.Add(
string.Format("{0}.{1}", TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter,
discoverer.Metadata.DefaultExecutorUri), totalAdapterRunTime.TotalSeconds);
totalTimeTakenByAdapters += totalAdapterRunTime.TotalSeconds;
}
catch (Exception e)
{
var message = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.ExceptionFromLoadTests,
discovererType.Name,
e.Message);
logger.SendMessage(TestMessageLevel.Error, message);
EqtTrace.Error("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: {0} ", e);
}
}
private void LoadTestsFromAnExtension(string extensionAssembly, IEnumerable <string> sources, IRunSettings settings, IMessageLogger logger)
{
var discovererToSourcesMap = GetDiscovererToSourcesMap(extensionAssembly, sources, logger);
// Warning is logged for in the inner function
if (discovererToSourcesMap == null || discovererToSourcesMap.Count() == 0)
{
return;
}
var context = new DiscoveryContext {
RunSettings = settings
};
// Set on the logger the TreatAdapterErrorAsWarning setting from runsettings.
this.SetAdapterLoggingSettings(logger, settings);
var discoverySink = new TestCaseDiscoverySink(this.discoveryResultCache);
foreach (var discoverer in discovererToSourcesMap.Keys)
{
Type discovererType = null;
// See if discoverer can be instantiated successfully else move next.
try
{
discovererType = discoverer.Value.GetType();
}
catch (Exception e)
{
var mesage = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.DiscovererInstantiationException,
e.Message);
logger.SendMessage(TestMessageLevel.Warning, mesage);
EqtTrace.Error(e);
continue;
}
// if instantiated successfully, get tests
try
{
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose(
"DiscoveryContext.LoadTests: Loading tests for {0}",
discoverer.Value.GetType().FullName);
}
var currentTotalTests = this.discoveryResultCache.TotalDiscoveredTests;
this.testPlatformEventSource.AdapterDiscoveryStart(discoverer.Metadata.DefaultExecutorUri.AbsoluteUri);
discoverer.Value.DiscoverTests(discovererToSourcesMap[discoverer], context, logger, discoverySink);
this.testPlatformEventSource.AdapterDiscoveryStop(this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests);
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose(
"DiscoveryContext.LoadTests: Done loading tests for {0}",
discoverer.Value.GetType().FullName);
}
}
catch (Exception e)
{
var message = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.ExceptionFromLoadTests,
discovererType.Name,
e.Message);
logger.SendMessage(TestMessageLevel.Error, message);
EqtTrace.Error(e);
}
}
}
