/// <inheritdoc /> public void SendTestRunStatistics(TestRunChangedEventArgs testRunChangedArgs) { var data = this.dataSerializer.SerializePayload(MessageType.TestRunStatsChange, testRunChangedArgs, this.protocolVersion); this.channel.Send(data); }
/// <summary> /// Handles Partial Run Complete event coming from a specific concurrent proxy exceution manager /// Each concurrent proxy execution manager will signal the parallel execution manager when its complete /// </summary> /// <param name="proxyExecutionManager">Concurrent Execution manager that completed the run</param> /// <param name="testRunCompleteArgs">RunCompleteArgs for the concurrent run</param> /// <param name="lastChunkArgs">LastChunk testresults for the concurrent run</param> /// <param name="runContextAttachments">RunAttachments for the concurrent run</param> /// <param name="executorUris">ExecutorURIs of the adapters involved in executing the tests</param> /// <returns>True if parallel run is complete</returns> public bool HandlePartialRunComplete( IProxyExecutionManager proxyExecutionManager, TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris) { var allRunsCompleted = false; lock (this.executionStatusLockObject) { // Each concurrent Executor calls this method // So, we need to keep track of total runcomplete calls this.runCompletedClients++; if (testRunCompleteArgs.IsCanceled) { allRunsCompleted = this.runCompletedClients == this.runStartedClients; } else { allRunsCompleted = this.runCompletedClients == this.availableTestSources; } if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("ParallelProxyExecutionManager: HandlePartialRunComplete: Total completed clients = {0}, Run complete = {1}, Run canceled: {2}.", this.runCompletedClients, allRunsCompleted, testRunCompleteArgs.IsCanceled); } } // verify that all executors are done with the execution and there are no more sources/testcases to execute if (allRunsCompleted) { // Reset enumerators this.sourceEnumerator = null; this.testCaseListEnumerator = null; this.currentRunDataAggregator = null; this.currentRunEventsHandler = null; // Dispose concurrent executors // Do not do the cleanuptask in the current thread as we will unncessarily add to execution time this.lastParallelRunCleanUpTask = Task.Run(() => { this.UpdateParallelLevel(0); }); return(true); } // In case of DataCollection we only start dc.exe on initialize, & close once the TestRun is complete, // So instead of resuing ProxyExecutionManager, we will close it here, & create new PEMWDC // In Case of Abort, clean old one and create new proxyExecutionManager in place of old one. if (!this.SharedHosts || testRunCompleteArgs.IsAborted || (proxyExecutionManager is ProxyExecutionManagerWithDataCollection)) { if (EqtTrace.IsVerboseEnabled) { EqtTrace.Verbose("ParallelProxyExecutionManager: HandlePartialRunComplete: Replace execution manager. Shared: {0}, Aborted: {1}.", this.SharedHosts, testRunCompleteArgs.IsAborted); } this.RemoveManager(proxyExecutionManager); proxyExecutionManager = CreateNewConcurrentManager(); var parallelEventsHandler = this.GetEventsHandler(proxyExecutionManager); this.AddManager(proxyExecutionManager, parallelEventsHandler); } // If cancel is triggered for any one run, there is no reason to fetch next source // and queue another test run if (!testRunCompleteArgs.IsCanceled) { this.StartTestRunOnConcurrentManager(proxyExecutionManager); } return(false); }
/// <inheritdoc/> public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris) { this.baseTestRunEventsHandler.HandleTestRunComplete(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris); }
/// <inheritdoc/> public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs) { this.baseTestRunEventsHandler.HandleTestRunStatsChange(testRunChangedArgs); }
/// <summary> /// Invoked when test run is complete /// </summary> public void HandleTestRunComplete(TestRunCompleteEventArgs runCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris) { if (runCompleteArgs == null) { throw new ArgumentNullException(nameof(runCompleteArgs)); } bool isAborted = runCompleteArgs.IsAborted; bool isCanceled = runCompleteArgs.IsCanceled; EqtTrace.Verbose("TestRunRequest:TestRunComplete: Starting. IsAborted:{0} IsCanceled:{1}.", isAborted, isCanceled); lock (this.syncObject) { // If this object is disposed, dont do anything if (this.disposed) { EqtTrace.Warning("TestRunRequest.TestRunComplete: Ignoring as the object is disposed."); return; } if (this.runCompletionEvent.WaitOne(0)) { EqtTrace.Info("TestRunRequest:TestRunComplete:Ignoring duplicate event. IsAborted:{0} IsCanceled:{1}.", isAborted, isCanceled); return; } // Disposing off the resources held by the execution manager so that the test host process can shut down. this.ExecutionManager?.Close(); try { this.runRequestTimeTracker.Stop(); if (lastChunkArgs != null) { // Raised the changed event also this.OnRunStatsChange.SafeInvoke(this, lastChunkArgs, "TestRun.RunStatsChanged"); } TestRunCompleteEventArgs runCompletedEvent = new TestRunCompleteEventArgs( runCompleteArgs.TestRunStatistics, runCompleteArgs.IsCanceled, runCompleteArgs.IsAborted, runCompleteArgs.Error, runContextAttachments as Collection <AttachmentSet>, this.runRequestTimeTracker.Elapsed); // Ignore the time sent (runCompleteArgs.ElapsedTimeInRunningTests) // by either engines - as both calculate at different points // If we use them, it would be an incorrect comparison between TAEF and Rocksteady this.OnRunCompletion.SafeInvoke(this, runCompletedEvent, "TestRun.TestRunComplete"); } finally { if (isCanceled) { this.State = TestRunState.Canceled; } else if (isAborted) { this.State = TestRunState.Aborted; } else { this.State = TestRunState.Completed; } // Notify the waiting handle that run is complete this.runCompletionEvent.Set(); var executionTotalTimeTaken = DateTime.UtcNow - this.executionStartTime; // Fill in the time taken to complete the run this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenInSecForRun, executionTotalTimeTaken.TotalSeconds.ToString()); // Fill in the Metrics From Test Host Process var metrics = runCompleteArgs.Metrics; if (metrics != null && metrics.Count != 0) { foreach (var metric in metrics) { this.requestData.MetricsCollection.Add(metric.Key, metric.Value); } } } EqtTrace.Info("TestRunRequest:TestRunComplete: Completed."); } }
/// <summary> /// The handle test run complete. /// </summary> /// <param name="testRunCompleteArgs"> /// The test run complete args. /// </param> /// <param name="lastChunkArgs"> /// The last chunk args. /// </param> /// <param name="runContextAttachments"> /// The run context attachments. /// </param> /// <param name="executorUris"> /// The executor uris. /// </param> public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris) { if (this.dataCollectionAttachmentSets != null && this.dataCollectionAttachmentSets.Any()) { runContextAttachments = GetCombinedAttachmentSets(this.dataCollectionAttachmentSets, runContextAttachments); } this.testRunEventsHandler.HandleTestRunComplete(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris); }
/// <inheritdoc/> public void SendTestRunStatistics(TestRunChangedEventArgs testRunChangedArgs) { this.communicationManager.SendMessage(MessageType.TestRunStatsChange, testRunChangedArgs, this.protocolVersion); }
public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris) { if (lastChunkArgs != null) { foreach (TestResult testResult in lastChunkArgs.NewTestResults) { switch (testResult.Outcome) { case TestOutcome.Passed: this.PassedTests.Add(testResult); break; case TestOutcome.Failed: this.FailedTests.Add(testResult); break; case TestOutcome.Skipped: this.SkippedTests.Add(testResult); break; } } } }
/// <summary> /// Handle test run complete. /// </summary> /// <param name="testRunCompleteArgs"> The test run complete args. </param> /// <param name="lastChunkArgs"> The last chunk args. </param> /// <param name="runContextAttachments"> The run context attachments. </param> /// <param name="executorUris"> The executor uris. </param> public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris) { EqtTrace.Info("Sending test run complete"); this.requestHandler.SendExecutionComplete(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris); }
/// <summary> /// Handle test run stats change. /// </summary> /// <param name="testRunChangedArgs"> The test run changed args. </param> public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs) { EqtTrace.Info("Sending test run statistics"); this.requestHandler.SendTestRunStatistics(testRunChangedArgs); }
public void RunTestsShouldRaiseTestRunComplete() { TestRunCompleteEventArgs receivedRunCompleteArgs = null; TestRunChangedEventArgs receivedRunStatusArgs = null; ICollection <AttachmentSet> receivedattachments = null; ICollection <string> receivedExecutorUris = null; var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; var executorUriExtensionMap = new List <Tuple <Uri, string> > { new Tuple <Uri, string>(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation), new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), assemblyLocation) }; // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); }; this.runTestsInstance.InvokeExecutorCallback = (executor, executorUriExtensionTuple, runContext, frameworkHandle) => { var testCase = new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll"); var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult); }; this.mockTestRunEventsHandler.Setup( treh => treh.HandleTestRunComplete( It.IsAny <TestRunCompleteEventArgs>(), It.IsAny <TestRunChangedEventArgs>(), It.IsAny <ICollection <AttachmentSet> >(), It.IsAny <ICollection <string> >())) .Callback( (TestRunCompleteEventArgs complete, TestRunChangedEventArgs stats, ICollection <AttachmentSet> attachments, ICollection <string> executorUris) => { receivedRunCompleteArgs = complete; receivedRunStatusArgs = stats; receivedattachments = attachments; receivedExecutorUris = executorUris; }); // Act. this.runTestsInstance.RunTests(); // Test run complete assertions. Assert.IsNotNull(receivedRunCompleteArgs); Assert.IsNull(receivedRunCompleteArgs.Error); Assert.IsFalse(receivedRunCompleteArgs.IsAborted); Assert.AreEqual(this.runTestsInstance.GetTestRunCache.TestRunStatistics.ExecutedTests, receivedRunCompleteArgs.TestRunStatistics.ExecutedTests); // Test run changed event assertions Assert.IsNotNull(receivedRunStatusArgs); Assert.AreEqual(this.runTestsInstance.GetTestRunCache.TestRunStatistics.ExecutedTests, receivedRunStatusArgs.TestRunStatistics.ExecutedTests); Assert.IsNotNull(receivedRunStatusArgs.NewTestResults); Assert.IsTrue(receivedRunStatusArgs.NewTestResults.Count() > 0); Assert.IsTrue(receivedRunStatusArgs.ActiveTests == null || receivedRunStatusArgs.ActiveTests.Count() == 0); // Attachments Assert.IsNotNull(receivedattachments); // Executor Uris var expectedUris = new string[] { BadBaseRunTestsExecutorUri.ToLower(), BaseRunTestsExecutorUri.ToLower() }; CollectionAssert.AreEqual(expectedUris, receivedExecutorUris.ToArray()); }
/// <summary> /// Handles the Run Complete event from a parallel proxy manager /// </summary> public void HandleTestRunComplete( TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection <AttachmentSet> runContextAttachments, ICollection <string> executorUris) { // we get run complete events from each executor process // so we cannot "complete" the actual executor operation until all sources/testcases are consumed // We should not block last chunk results while we aggregate overall run data if (lastChunkArgs != null) { ConvertToRawMessageAndSend(MessageType.TestRunStatsChange, lastChunkArgs); HandleTestRunStatsChange(lastChunkArgs); } // Update runstats, executorUris, etc. // we need this data when we send the final runcomplete runDataAggregator.Aggregate( testRunCompleteArgs.TestRunStatistics, executorUris, testRunCompleteArgs.Error, testRunCompleteArgs.ElapsedTimeInRunningTests, testRunCompleteArgs.IsAborted, testRunCompleteArgs.IsCanceled, runContextAttachments, testRunCompleteArgs.AttachmentSets); // Do not send TestRunComplete to actual test run handler // We need to see if there are still sources to execute - let the parallel manager decide var parallelRunComplete = this.parallelProxyExecutionManager.HandlePartialRunComplete( this.proxyExecutionManager, testRunCompleteArgs, null, // lastChunk should be null as we already sent this data above runContextAttachments, executorUris); if (parallelRunComplete) { // todo : Merge Code Coverage files here var completedArgs = new TestRunCompleteEventArgs(runDataAggregator.GetAggregatedRunStats(), runDataAggregator.IsCanceled, runDataAggregator.IsAborted, runDataAggregator.GetAggregatedException(), new Collection <AttachmentSet>(runDataAggregator.RunCompleteArgsAttachments), runDataAggregator.ElapsedTime); // In case of sequential execution - RawMessage would have contained a 'TestRunCompletePayload' object // To send a rawmessge - we need to create rawmessage from an aggregated payload object var testRunCompletePayload = new TestRunCompletePayload() { ExecutorUris = runDataAggregator.ExecutorUris, LastRunTests = null, RunAttachments = runDataAggregator.RunContextAttachments, TestRunCompleteArgs = completedArgs }; // we have to send rawmessages as we block the runcomplete actual raw messages ConvertToRawMessageAndSend(MessageType.ExecutionComplete, testRunCompletePayload); // send actual test runcomplete to clients this.actualRunEventsHandler.HandleTestRunComplete( completedArgs, null, runDataAggregator.RunContextAttachments, runDataAggregator.ExecutorUris); } }
public void RunTestsShouldUpdateTestCasesWithPackageWhenCacheIsHitIfProvided() { var package = "x.apprecipe"; this.testExecutionContext.FrequencyOfRunStatsChangeEvent = 2; TestRunChangedEventArgs receivedRunStatusArgs = null; this.runTestsInstance = new TestableBaseRunTests( null, package, testExecutionContext, null, this.mockTestRunEventsHandler.Object, this.mockTestPlatformEventSource.Object, this.mockRequestData.Object); var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; var executorUriExtensionMap = new List <Tuple <Uri, string> > { new Tuple <Uri, string>(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation), new Tuple <Uri, string>(new Uri(BaseRunTestsExecutorUri), assemblyLocation) }; // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return(executorUriExtensionMap); }; this.runTestsInstance.InvokeExecutorCallback = (executor, executorUriExtensionTuple, runContext, frameworkHandle) => { var testCase = new TestCase("x.y.z1", new Uri("uri://dummy"), "x.dll"); var inProgTestCase = new TestCase("x.y.z2", new Uri("uri://dummy"), "x.dll"); var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); this.runTestsInstance.GetTestRunCache.OnTestStarted(inProgTestCase); this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult); }; this.mockTestRunEventsHandler.Setup(treh => treh.HandleTestRunStatsChange(It.IsAny <TestRunChangedEventArgs>())) .Callback((TestRunChangedEventArgs stats) => { receivedRunStatusArgs = stats; }); // Act. this.runTestsInstance.RunTests(); // Test run changed event assertions Assert.IsNotNull(receivedRunStatusArgs.NewTestResults); Assert.IsTrue(receivedRunStatusArgs.NewTestResults.Count() > 0); Assert.IsNotNull(receivedRunStatusArgs.ActiveTests); Assert.AreEqual(receivedRunStatusArgs.ActiveTests.Count(), 1); // verify TC.Source is updated with package foreach (var tr in receivedRunStatusArgs.NewTestResults) { Assert.AreEqual(tr.TestCase.Source, package); } foreach (var tc in receivedRunStatusArgs.ActiveTests) { Assert.AreEqual(tc.Source, package); } }