/// <inheritdoc />
public void SendTestRunStatistics(TestRunChangedEventArgs testRunChangedArgs)
{
var data = this.dataSerializer.SerializePayload(MessageType.TestRunStatsChange, testRunChangedArgs, this.protocolVersion);
this.channel.Send(data);
}
/// <summary>
/// Handles Partial Run Complete event coming from a specific concurrent proxy exceution manager
/// Each concurrent proxy execution manager will signal the parallel execution manager when its complete
/// </summary>
/// <param name="proxyExecutionManager">Concurrent Execution manager that completed the run</param>
/// <param name="testRunCompleteArgs">RunCompleteArgs for the concurrent run</param>
/// <param name="lastChunkArgs">LastChunk testresults for the concurrent run</param>
/// <param name="runContextAttachments">RunAttachments for the concurrent run</param>
/// <param name="executorUris">ExecutorURIs of the adapters involved in executing the tests</param>
/// <returns>True if parallel run is complete</returns>
public bool HandlePartialRunComplete(
IProxyExecutionManager proxyExecutionManager,
TestRunCompleteEventArgs testRunCompleteArgs,
TestRunChangedEventArgs lastChunkArgs,
ICollection <AttachmentSet> runContextAttachments,
ICollection <string> executorUris)
{
var allRunsCompleted = false;
lock (this.executionStatusLockObject)
{
// Each concurrent Executor calls this method
// So, we need to keep track of total runcomplete calls
this.runCompletedClients++;
if (testRunCompleteArgs.IsCanceled)
{
allRunsCompleted = this.runCompletedClients == this.runStartedClients;
}
else
{
allRunsCompleted = this.runCompletedClients == this.availableTestSources;
}
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("ParallelProxyExecutionManager: HandlePartialRunComplete: Total completed clients = {0}, Run complete = {1}, Run canceled: {2}.", this.runCompletedClients, allRunsCompleted, testRunCompleteArgs.IsCanceled);
}
}
// verify that all executors are done with the execution and there are no more sources/testcases to execute
if (allRunsCompleted)
{
// Reset enumerators
this.sourceEnumerator = null;
this.testCaseListEnumerator = null;
this.currentRunDataAggregator = null;
this.currentRunEventsHandler = null;
// Dispose concurrent executors
// Do not do the cleanuptask in the current thread as we will unncessarily add to execution time
this.lastParallelRunCleanUpTask = Task.Run(() =>
{
this.UpdateParallelLevel(0);
});
return(true);
}
// In case of DataCollection we only start dc.exe on initialize, & close once the TestRun is complete,
// So instead of resuing ProxyExecutionManager, we will close it here, & create new PEMWDC
// In Case of Abort, clean old one and create new proxyExecutionManager in place of old one.
if (!this.SharedHosts || testRunCompleteArgs.IsAborted || (proxyExecutionManager is ProxyExecutionManagerWithDataCollection))
{
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose("ParallelProxyExecutionManager: HandlePartialRunComplete: Replace execution manager. Shared: {0}, Aborted: {1}.", this.SharedHosts, testRunCompleteArgs.IsAborted);
}
this.RemoveManager(proxyExecutionManager);
proxyExecutionManager = CreateNewConcurrentManager();
var parallelEventsHandler = this.GetEventsHandler(proxyExecutionManager);
this.AddManager(proxyExecutionManager, parallelEventsHandler);
}
// If cancel is triggered for any one run, there is no reason to fetch next source
// and queue another test run
if (!testRunCompleteArgs.IsCanceled)
{
this.StartTestRunOnConcurrentManager(proxyExecutionManager);
}
return(false);
}
/// <inheritdoc/>
public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris)
{
this.baseTestRunEventsHandler.HandleTestRunComplete(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris);
}
/// <inheritdoc/>
public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs)
{
this.baseTestRunEventsHandler.HandleTestRunStatsChange(testRunChangedArgs);
}
/// <summary>
/// Invoked when test run is complete
/// </summary>
public void HandleTestRunComplete(TestRunCompleteEventArgs runCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris)
{
if (runCompleteArgs == null)
{
throw new ArgumentNullException(nameof(runCompleteArgs));
}
bool isAborted = runCompleteArgs.IsAborted;
bool isCanceled = runCompleteArgs.IsCanceled;
EqtTrace.Verbose("TestRunRequest:TestRunComplete: Starting. IsAborted:{0} IsCanceled:{1}.", isAborted, isCanceled);
lock (this.syncObject)
{
// If this object is disposed, dont do anything
if (this.disposed)
{
EqtTrace.Warning("TestRunRequest.TestRunComplete: Ignoring as the object is disposed.");
return;
}
if (this.runCompletionEvent.WaitOne(0))
{
EqtTrace.Info("TestRunRequest:TestRunComplete:Ignoring duplicate event. IsAborted:{0} IsCanceled:{1}.", isAborted, isCanceled);
return;
}
// Disposing off the resources held by the execution manager so that the test host process can shut down.
this.ExecutionManager?.Close();
try
{
this.runRequestTimeTracker.Stop();
if (lastChunkArgs != null)
{
// Raised the changed event also
this.OnRunStatsChange.SafeInvoke(this, lastChunkArgs, "TestRun.RunStatsChanged");
}
TestRunCompleteEventArgs runCompletedEvent =
new TestRunCompleteEventArgs(
runCompleteArgs.TestRunStatistics,
runCompleteArgs.IsCanceled,
runCompleteArgs.IsAborted,
runCompleteArgs.Error,
runContextAttachments as Collection <AttachmentSet>,
this.runRequestTimeTracker.Elapsed);
// Ignore the time sent (runCompleteArgs.ElapsedTimeInRunningTests)
// by either engines - as both calculate at different points
// If we use them, it would be an incorrect comparison between TAEF and Rocksteady
this.OnRunCompletion.SafeInvoke(this, runCompletedEvent, "TestRun.TestRunComplete");
}
finally
{
if (isCanceled)
{
this.State = TestRunState.Canceled;
}
else if (isAborted)
{
this.State = TestRunState.Aborted;
}
else
{
this.State = TestRunState.Completed;
}
// Notify the waiting handle that run is complete
this.runCompletionEvent.Set();
var executionTotalTimeTaken = DateTime.UtcNow - this.executionStartTime;
// Fill in the time taken to complete the run
this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenInSecForRun, executionTotalTimeTaken.TotalSeconds.ToString());
// Fill in the Metrics From Test Host Process
var metrics = runCompleteArgs.Metrics;
if (metrics != null && metrics.Count != 0)
{
foreach (var metric in metrics)
{
this.requestData.MetricsCollection.Add(metric.Key, metric.Value);
}
}
}
EqtTrace.Info("TestRunRequest:TestRunComplete: Completed.");
}
}
/// <summary>
/// The handle test run complete.
/// </summary>
/// <param name="testRunCompleteArgs">
/// The test run complete args.
/// </param>
/// <param name="lastChunkArgs">
/// The last chunk args.
/// </param>
/// <param name="runContextAttachments">
/// The run context attachments.
/// </param>
/// <param name="executorUris">
/// The executor uris.
/// </param>
public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris)
{
if (this.dataCollectionAttachmentSets != null && this.dataCollectionAttachmentSets.Any())
{
runContextAttachments = GetCombinedAttachmentSets(this.dataCollectionAttachmentSets, runContextAttachments);
}
this.testRunEventsHandler.HandleTestRunComplete(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris);
}
/// <inheritdoc/>
public void SendTestRunStatistics(TestRunChangedEventArgs testRunChangedArgs)
{
this.communicationManager.SendMessage(MessageType.TestRunStatsChange, testRunChangedArgs, this.protocolVersion);
}
public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris)
{
if (lastChunkArgs != null)
{
foreach (TestResult testResult in lastChunkArgs.NewTestResults)
{
switch (testResult.Outcome)
{
case TestOutcome.Passed:
this.PassedTests.Add(testResult);
break;
case TestOutcome.Failed:
this.FailedTests.Add(testResult);
break;
case TestOutcome.Skipped:
this.SkippedTests.Add(testResult);
break;
}
}
}
}
/// <summary>
/// Handle test run complete.
/// </summary>
/// <param name="testRunCompleteArgs"> The test run complete args. </param>
/// <param name="lastChunkArgs"> The last chunk args. </param>
/// <param name="runContextAttachments"> The run context attachments. </param>
/// <param name="executorUris"> The executor uris. </param>
public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris)
{
EqtTrace.Info("Sending test run complete");
this.requestHandler.SendExecutionComplete(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris);
}
/// <summary>
/// Handle test run stats change.
/// </summary>
/// <param name="testRunChangedArgs"> The test run changed args. </param>
public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs)
{
EqtTrace.Info("Sending test run statistics");
this.requestHandler.SendTestRunStatistics(testRunChangedArgs);
}
public void RunTestsShouldRaiseTestRunComplete()
{
TestRunCompleteEventArgs receivedRunCompleteArgs = null;
TestRunChangedEventArgs receivedRunStatusArgs = null;
ICollection <AttachmentSet> receivedattachments = null;
ICollection <string> receivedExecutorUris = null;
var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location;
var executorUriExtensionMap = new List <Tuple <Uri, string> >
{
new Tuple <Uri, string>(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation),
new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), assemblyLocation)
};
// Setup mocks.
this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); };
this.runTestsInstance.InvokeExecutorCallback =
(executor, executorUriExtensionTuple, runContext, frameworkHandle) =>
{
var testCase = new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll");
var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase);
this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult);
};
this.mockTestRunEventsHandler.Setup(
treh =>
treh.HandleTestRunComplete(
It.IsAny <TestRunCompleteEventArgs>(),
It.IsAny <TestRunChangedEventArgs>(),
It.IsAny <ICollection <AttachmentSet> >(),
It.IsAny <ICollection <string> >()))
.Callback(
(TestRunCompleteEventArgs complete,
TestRunChangedEventArgs stats,
ICollection <AttachmentSet> attachments,
ICollection <string> executorUris) =>
{
receivedRunCompleteArgs = complete;
receivedRunStatusArgs = stats;
receivedattachments = attachments;
receivedExecutorUris = executorUris;
});
// Act.
this.runTestsInstance.RunTests();
// Test run complete assertions.
Assert.IsNotNull(receivedRunCompleteArgs);
Assert.IsNull(receivedRunCompleteArgs.Error);
Assert.IsFalse(receivedRunCompleteArgs.IsAborted);
Assert.AreEqual(this.runTestsInstance.GetTestRunCache.TestRunStatistics.ExecutedTests, receivedRunCompleteArgs.TestRunStatistics.ExecutedTests);
// Test run changed event assertions
Assert.IsNotNull(receivedRunStatusArgs);
Assert.AreEqual(this.runTestsInstance.GetTestRunCache.TestRunStatistics.ExecutedTests, receivedRunStatusArgs.TestRunStatistics.ExecutedTests);
Assert.IsNotNull(receivedRunStatusArgs.NewTestResults);
Assert.IsTrue(receivedRunStatusArgs.NewTestResults.Count() > 0);
Assert.IsTrue(receivedRunStatusArgs.ActiveTests == null || receivedRunStatusArgs.ActiveTests.Count() == 0);
// Attachments
Assert.IsNotNull(receivedattachments);
// Executor Uris
var expectedUris = new string[] { BadBaseRunTestsExecutorUri.ToLower(), BaseRunTestsExecutorUri.ToLower() };
CollectionAssert.AreEqual(expectedUris, receivedExecutorUris.ToArray());
}
/// <summary>
/// Handles the Run Complete event from a parallel proxy manager
/// </summary>
public void HandleTestRunComplete(
TestRunCompleteEventArgs testRunCompleteArgs,
TestRunChangedEventArgs lastChunkArgs,
ICollection <AttachmentSet> runContextAttachments,
ICollection <string> executorUris)
{
// we get run complete events from each executor process
// so we cannot "complete" the actual executor operation until all sources/testcases are consumed
// We should not block last chunk results while we aggregate overall run data
if (lastChunkArgs != null)
{
ConvertToRawMessageAndSend(MessageType.TestRunStatsChange, lastChunkArgs);
HandleTestRunStatsChange(lastChunkArgs);
}
// Update runstats, executorUris, etc.
// we need this data when we send the final runcomplete
runDataAggregator.Aggregate(
testRunCompleteArgs.TestRunStatistics,
executorUris,
testRunCompleteArgs.Error,
testRunCompleteArgs.ElapsedTimeInRunningTests,
testRunCompleteArgs.IsAborted,
testRunCompleteArgs.IsCanceled,
runContextAttachments,
testRunCompleteArgs.AttachmentSets);
// Do not send TestRunComplete to actual test run handler
// We need to see if there are still sources to execute - let the parallel manager decide
var parallelRunComplete = this.parallelProxyExecutionManager.HandlePartialRunComplete(
this.proxyExecutionManager,
testRunCompleteArgs,
null, // lastChunk should be null as we already sent this data above
runContextAttachments,
executorUris);
if (parallelRunComplete)
{
// todo : Merge Code Coverage files here
var completedArgs = new TestRunCompleteEventArgs(runDataAggregator.GetAggregatedRunStats(),
runDataAggregator.IsCanceled,
runDataAggregator.IsAborted,
runDataAggregator.GetAggregatedException(),
new Collection <AttachmentSet>(runDataAggregator.RunCompleteArgsAttachments),
runDataAggregator.ElapsedTime);
// In case of sequential execution - RawMessage would have contained a 'TestRunCompletePayload' object
// To send a rawmessge - we need to create rawmessage from an aggregated payload object
var testRunCompletePayload = new TestRunCompletePayload()
{
ExecutorUris = runDataAggregator.ExecutorUris,
LastRunTests = null,
RunAttachments = runDataAggregator.RunContextAttachments,
TestRunCompleteArgs = completedArgs
};
// we have to send rawmessages as we block the runcomplete actual raw messages
ConvertToRawMessageAndSend(MessageType.ExecutionComplete, testRunCompletePayload);
// send actual test runcomplete to clients
this.actualRunEventsHandler.HandleTestRunComplete(
completedArgs, null, runDataAggregator.RunContextAttachments, runDataAggregator.ExecutorUris);
}
}
public void RunTestsShouldUpdateTestCasesWithPackageWhenCacheIsHitIfProvided()
{
var package = "x.apprecipe";
this.testExecutionContext.FrequencyOfRunStatsChangeEvent = 2;
TestRunChangedEventArgs receivedRunStatusArgs = null;
this.runTestsInstance = new TestableBaseRunTests(
null,
package,
testExecutionContext,
null,
this.mockTestRunEventsHandler.Object,
this.mockTestPlatformEventSource.Object,
this.mockRequestData.Object);
var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location;
var executorUriExtensionMap = new List <Tuple <Uri, string> >
{
new Tuple <Uri, string>(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation),
new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), assemblyLocation)
};
// Setup mocks.
this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); };
this.runTestsInstance.InvokeExecutorCallback =
(executor, executorUriExtensionTuple, runContext, frameworkHandle) =>
{
var testCase = new TestCase("x.y.z1", new Uri("uri://dummy"), "x.dll");
var inProgTestCase = new TestCase("x.y.z2", new Uri("uri://dummy"), "x.dll");
var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase);
this.runTestsInstance.GetTestRunCache.OnTestStarted(inProgTestCase);
this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult);
};
this.mockTestRunEventsHandler.Setup(treh => treh.HandleTestRunStatsChange(It.IsAny <TestRunChangedEventArgs>()))
.Callback((TestRunChangedEventArgs stats) =>
{
receivedRunStatusArgs = stats;
});
// Act.
this.runTestsInstance.RunTests();
// Test run changed event assertions
Assert.IsNotNull(receivedRunStatusArgs.NewTestResults);
Assert.IsTrue(receivedRunStatusArgs.NewTestResults.Count() > 0);
Assert.IsNotNull(receivedRunStatusArgs.ActiveTests);
Assert.AreEqual(receivedRunStatusArgs.ActiveTests.Count(), 1);
// verify TC.Source is updated with package
foreach (var tr in receivedRunStatusArgs.NewTestResults)
{
Assert.AreEqual(tr.TestCase.Source, package);
}
foreach (var tc in receivedRunStatusArgs.ActiveTests)
{
Assert.AreEqual(tc.Source, package);
}
}
