/// <summary>
/// Notifies the cache of a new test result from the adapter.
/// </summary>
/// <param name="testResult"> The test result. </param>
public void OnNewTestResult(TestResult testResult)
{
lock (this.syncObject)
{
this.totalExecutedTests++;
this.testResults.Add(testResult);
MSTestV1TelemetryHelper.AddTelemetry(testResult, this.AdapterTelemetry);
long count;
if (this.runStats.TryGetValue(testResult.Outcome, out count))
{
count++;
}
else
{
count = 1;
}
this.runStats[testResult.Outcome] = count;
this.RemoveInProgress(testResult);
this.CheckForCacheHit();
}
}
private bool RunTestInternalWithExecutors(IEnumerable <Tuple <Uri, string> > executorUriExtensionMap, long totalTests)
{
// Collecting Total Number of Adapters Discovered in Machine.
this.requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAdapterDiscoveredDuringExecution, executorUriExtensionMap.Count());
var attachedToTestHost = false;
var executorCache = new Dictionary <string, LazyExtension <ITestExecutor, ITestExecutorCapabilities> >();
foreach (var executorUriExtensionTuple in executorUriExtensionMap)
{
// Avoid processing the same executor twice.
if (executorCache.ContainsKey(executorUriExtensionTuple.Item1.AbsoluteUri))
{
continue;
}
// Get the extension manager.
var extensionManager = this.GetExecutorExtensionManager(executorUriExtensionTuple.Item2);
// Look up the executor.
var executor = extensionManager.TryGetTestExtension(executorUriExtensionTuple.Item1);
if (executor == null)
{
// Commenting this out because of a compatibility issue with Microsoft.Dotnet.ProjectModel released on nuGet.org.
// this.activeExecutor = null;
// var runtimeVersion = string.Concat(PlatformServices.Default.Runtime.RuntimeType, " ",
// PlatformServices.Default.Runtime.RuntimeVersion);
var runtimeVersion = " ";
this.TestRunEventsHandler?.HandleLogMessage(
TestMessageLevel.Warning,
string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.NoMatchingExecutor,
executorUriExtensionTuple.Item1.AbsoluteUri,
runtimeVersion));
continue;
}
// Cache the executor.
executorCache.Add(executorUriExtensionTuple.Item1.AbsoluteUri, executor);
// Check if we actually have to attach to the default test host.
if (!this.runContext.IsBeingDebugged || attachedToTestHost)
{
// We already know we should attach to the default test host, simply continue.
continue;
}
// If there's at least one adapter in the filtered adapters list that doesn't
// implement the new test executor interface, we should attach to the default test
// host by default.
// Same goes if all adapters implement the new test executor interface but at
// least one of them needs the test platform to attach to the default test host.
if (!(executor.Value is ITestExecutor2) ||
this.ShouldAttachDebuggerToTestHost(executor, executorUriExtensionTuple, this.runContext))
{
EqtTrace.Verbose("Attaching to default test host.");
attachedToTestHost = true;
var pid = Process.GetCurrentProcess().Id;
if (!this.frameworkHandle.AttachDebuggerToProcess(pid))
{
EqtTrace.Warning(
string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.AttachDebuggerToDefaultTestHostFailure,
pid));
}
}
}
// Call the executor for each group of tests.
var exceptionsHitDuringRunTests = false;
var executorsFromDeprecatedLocations = false;
double totalTimeTakenByAdapters = 0;
foreach (var executorUriExtensionTuple in executorUriExtensionMap)
{
var executorUri = executorUriExtensionTuple.Item1.AbsoluteUri;
// Get the executor from the cache.
if (!executorCache.TryGetValue(executorUri, out var executor))
{
continue;
}
try
{
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose(
"BaseRunTests.RunTestInternalWithExecutors: Running tests for {0}",
executor.Metadata.ExtensionUri);
}
// set the active executor
this.activeExecutor = executor.Value;
// If test run cancellation is requested, skip the next executor
if (this.isCancellationRequested)
{
break;
}
var timeStartNow = DateTime.UtcNow;
var currentTotalTests = this.testRunCache.TotalExecutedTests;
this.testPlatformEventSource.AdapterExecutionStart(executorUri);
// Run the tests.
if (this.NotRequiredSTAThread() || !this.TryToRunInSTAThread(() => this.InvokeExecutor(executor, executorUriExtensionTuple, this.runContext, this.frameworkHandle), true))
{
this.InvokeExecutor(executor, executorUriExtensionTuple, this.runContext, this.frameworkHandle);
}
this.testPlatformEventSource.AdapterExecutionStop(this.testRunCache.TotalExecutedTests - currentTotalTests);
var totalTimeTaken = DateTime.UtcNow - timeStartNow;
// Identify whether the executor did run any tests at all
if (this.testRunCache.TotalExecutedTests > totalTests)
{
this.executorUrisThatRanTests.Add(executorUri);
// Collecting Total Tests Ran by each Adapter
var totalTestRun = this.testRunCache.TotalExecutedTests - totalTests;
this.requestData.MetricsCollection.Add(string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsRanByAdapter, executorUri), totalTestRun);
// Only enable this for MSTestV1 telemetry for now, this might become more generic later.
if (MSTestV1TelemetryHelper.IsMSTestV1Adapter(executorUri))
{
foreach (var adapterMetrics in this.testRunCache.AdapterTelemetry.Keys.Where(k => k.StartsWith(executorUri)))
{
var value = this.testRunCache.AdapterTelemetry[adapterMetrics];
this.requestData.MetricsCollection.Add(string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsRunByMSTestv1, adapterMetrics), value);
}
}
if (!CrossPlatEngine.Constants.DefaultAdapters.Contains(executor.Metadata.ExtensionUri, StringComparer.OrdinalIgnoreCase))
{
var executorLocation = executor.Value.GetType().GetTypeInfo().Assembly.GetAssemblyLocation();
executorsFromDeprecatedLocations |= Path.GetDirectoryName(executorLocation).Equals(CrossPlatEngine.Constants.DefaultAdapterLocation);
}
totalTests = this.testRunCache.TotalExecutedTests;
}
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose(
"BaseRunTests.RunTestInternalWithExecutors: Completed running tests for {0}",
executor.Metadata.ExtensionUri);
}
// Collecting Time Taken by each executor Uri
this.requestData.MetricsCollection.Add(string.Format("{0}.{1}", TelemetryDataConstants.TimeTakenToRunTestsByAnAdapter, executorUri), totalTimeTaken.TotalSeconds);
totalTimeTakenByAdapters += totalTimeTaken.TotalSeconds;
}
catch (Exception e)
{
string exceptionMessage = (e is UnauthorizedAccessException)
? string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.AccessDenied, e.Message)
: ExceptionUtilities.GetExceptionMessage(e);
exceptionsHitDuringRunTests = true;
if (EqtTrace.IsErrorEnabled)
{
EqtTrace.Error(
"BaseRunTests.RunTestInternalWithExecutors: An exception occurred while invoking executor {0}. {1}.",
executorUriExtensionTuple.Item1,
e);
}
this.TestRunEventsHandler?.HandleLogMessage(
TestMessageLevel.Error,
string.Format(
CultureInfo.CurrentCulture,
CrossPlatEngineResources.ExceptionFromRunTests,
executorUriExtensionTuple.Item1,
exceptionMessage));
}
finally
{
this.activeExecutor = null;
}
}
// Collecting Total Time Taken by Adapters
this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenByAllAdaptersInSec, totalTimeTakenByAdapters);
if (executorsFromDeprecatedLocations)
{
this.TestRunEventsHandler?.HandleLogMessage(TestMessageLevel.Warning, string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DeprecatedAdapterPath));
}
return(exceptionsHitDuringRunTests);
}