public async Task RunTestsForCurrentAgentAsync(string testAgentTag, int testAgentRunTimeout) { if (await IsEligibleToStartTestAgentRunAsync(testAgentTag).ConfigureAwait(false)) { var newTestAgentRun = await GetFirstNewTestAgentRunForCurrentTestAgentAsync().ConfigureAwait(false); if (newTestAgentRun != null) { await _testRunLogService.CreateTestRunLogAsync($"Test agent with tag {testAgentTag} starts tests execution on machine {_environmentService.MachineName}.", newTestAgentRun.TestRunId).ConfigureAwait(false); await _testAgentStateSwitcher.SetTestAgentAsRunningTestsAsync(testAgentTag).ConfigureAwait(false); var cancellationTokenSource = new CancellationTokenSource(); var cancellationTokenSourceLastAvailable = new CancellationTokenSource(); var testAgentUpdateAvailabilityTask = _taskProvider.StartNewLongRunningRepeating( cancellationTokenSourceLastAvailable, () => { UpdateTestAgentLastAvailable(newTestAgentRun.TestAgentRunId); }, 15000); var executeTestAgentRunTask = _taskProvider.StartNewLongRunning( (c) => { ExecuteTestAgentRunAsync(newTestAgentRun, testAgentRunTimeout, cancellationTokenSource).Wait(); }, cancellationTokenSource); var checkTestRunnerLastAvailableTask = _taskProvider.StartNewLongRunningRepeating( cancellationTokenSource, () => { if (executeTestAgentRunTask.IsCompleted) { cancellationTokenSource.Cancel(); return; } if (executeTestAgentRunTask.IsFaulted) { cancellationTokenSource.Cancel(); return; } CheckTestRunnerStatus(newTestAgentRun.TestAgentRunId, cancellationTokenSource); }, 15000); checkTestRunnerLastAvailableTask.Wait(); cancellationTokenSourceLastAvailable.Cancel(); testAgentUpdateAvailabilityTask.Wait(); try { // DEBUG: Turn-off if debugging _consoleProvider.Clear(); if (_wasTestAgentRunCompleted) { await _testRunLogService.CreateTestRunLogAsync($"Test agent with tag {testAgentTag} finished tests execution on machine {_environmentService.MachineName}.", newTestAgentRun.TestRunId).ConfigureAwait(false); await _testRunLogService.CreateTestRunLogAsync($"Test agent with tag {testAgentTag} starts waiting for new jobs on machine {_environmentService.MachineName}.", newTestAgentRun.TestRunId).ConfigureAwait(false); } else { SendTestAgentRunExceptionToRunner(newTestAgentRun, executeTestAgentRunTask); // TODO: Move logic to be executed on Test Agent Run Abort- extension. await _testRunLogService .CreateTestRunLogAsync( $"Test agent with tag {testAgentTag} starts waiting for new jobs on machine {_environmentService.MachineName}.", newTestAgentRun.TestRunId).ConfigureAwait(false); _consoleProvider.WriteLine($"Test agent run aborted."); _consoleProvider.WriteLine($"Test agent with tag {testAgentTag} starts waiting for new jobs on machine {_environmentService.MachineName}."); var cts = new CancellationTokenSource(); var waitForTestRunToCompleteTask = _taskProvider.StartNewLongRunningRepeating( cts, () => { if (IsTestRunCompleted(newTestAgentRun.TestRunId).Result) { cts.Cancel(); } }, 5000); waitForTestRunToCompleteTask.Wait(); } } finally { await _testAgentStateSwitcher.SetTestAgentAsActiveAsync(testAgentTag).ConfigureAwait(false); } } } }
public async Task <string> ExecuteTestsWithRetryAsync( string testTechnology, string originalRunTestResults, string distributedTestsList, string workingDir, Guid testRunId, string testsLibraryPath, string assemblyName, bool runInParallel, string nativeArguments, int testAgentRunTimeout, int retriesCount, double threshold, bool isTimeBasedBalance, bool sameMachineByClass, CancellationTokenSource cancellationTokenSource) { _currentTestRunId = testRunId; _nativeTestsRunner = _pluginService.GetNativeTestRunnerService(testTechnology); var testRun = await _testRunRepository.GetAsync(testRunId).ConfigureAwait(false); var distributedTestCases = _jsonSerializer.Deserialize <List <TestCase> >(distributedTestsList); if (string.IsNullOrEmpty(originalRunTestResults)) { return(null); } var originalTestRun = _nativeTestsRunner.DeserializeTestResults(originalRunTestResults); var failedTests = _testResultsService.GetAllNotPassedTests(testTechnology, originalRunTestResults); double failedTestsPercentage = _testResultsService.CalculatedFailedTestsPercentage(failedTests, distributedTestCases); await _testRunLogService.CreateTestRunLogAsync($"failedTestsPercentage= {failedTestsPercentage} < threshold = {threshold}", testRunId).ConfigureAwait(false); if (failedTests.Count > 0 && failedTestsPercentage < threshold) { await _testRunLogService.CreateTestRunLogAsync($"The failed test % {failedTestsPercentage} < threshold % {threshold}. The failed tests will be retried {retriesCount} times.", testRunId).ConfigureAwait(false); for (int i = 0; i < retriesCount; i++) { if (cancellationTokenSource.Token.IsCancellationRequested) { return(null); } await _testRunLogService.CreateTestRunLogAsync($"Start failed tests retry number {i + 1}.", testRunId).ConfigureAwait(false); string retriedTestTestResults; if (failedTests.Count > 0) { await _testRunLogService.CreateTestRunLogAsync($"{failedTests.Count} tests will be retried.", testRunId).ConfigureAwait(false); retriedTestTestResults = await ExecuteTestsWithNativeRunnerAsync(workingDir, testsLibraryPath, assemblyName, runInParallel, testRun.MaxParallelProcessesCount, nativeArguments, testAgentRunTimeout, isTimeBasedBalance, sameMachineByClass, failedTests, cancellationTokenSource).ConfigureAwait(false); var passedTests = _nativeTestsRunner.GetAllPassesTests(retriedTestTestResults); _nativeTestsRunner.UpdatePassedTests(passedTests, originalTestRun); _nativeTestsRunner.UpdateResultsSummary(originalTestRun); } else { break; } if (!string.IsNullOrEmpty(retriedTestTestResults)) { failedTests = _testResultsService.GetAllNotPassedTests(testTechnology, retriedTestTestResults); } } } else { await _testRunLogService.CreateTestRunLogAsync($"Percentage of failed tests {failedTestsPercentage} is over threshold {threshold}, will not retry tests.", testRunId).ConfigureAwait(false); } string retriedRunTestResults = _nativeTestsRunner.SerializeTestResults(originalTestRun); return(retriedRunTestResults); }