public static string GetTempFileName()
{
if (_provider == null)
{
return(Path.GetTempFileName());
}
return(_provider.GetTempFileName());
}
public string CreateLocalTestListFileAsync(string testList)
{
if (string.IsNullOrEmpty(testList))
{
throw new ArgumentNullException(TestListNullExceptionMessage);
}
var testListContent = testList;
var testListFilePath = _pathProvider.GetTempFileName();
_fileProvider.WriteAllText(testListFilePath, testListContent);
return(testListFilePath);
}
private async Task ExecuteTestAgentRunAsync(TestAgentRunDto testAgentRun, int testAgentRunTimeout, CancellationTokenSource cancellationTokenSource)
{
_pluginService.ExecuteAllTestAgentPluginsPreTestRunLogic();
testAgentRun.Status = TestAgentRunStatus.InProgress;
await CreateEnvironmentVariablesForCustomArgumentsAsync(testAgentRun.TestRunId).ConfigureAwait(false);
await _testAgentRunRepository.UpdateAsync(testAgentRun.TestAgentRunId, testAgentRun).ConfigureAwait(false);
var testRun = await _testRunRepository.GetAsync(testAgentRun.TestRunId).ConfigureAwait(false);
var tempExecutionFolder = _pathProvider.Combine(_pathProvider.GetTempFolderPath(), _guidService.NewGuid().ToString());
var tempZipFileName = _pathProvider.GetTempFileName();
var testRunOutput = await _testRunOutputServiceClient.GetTestRunOutputByTestRunIdAsync(testRun.TestRunId).ConfigureAwait(false);
if (testRunOutput == null)
{
// DEBUG:
await _testRunLogService.CreateTestRunLogAsync("The test run output cannot be null.", testAgentRun.TestRunId).ConfigureAwait(false);
throw new ArgumentException("The test run output cannot be null.");
}
_fileProvider.WriteAllBytes(tempZipFileName, testRunOutput.TestOutputFilesPackage);
_fileProvider.ExtractZip(tempZipFileName, tempExecutionFolder);
var testRunTestsAssemblyPath = _pathProvider.Combine(tempExecutionFolder, testRun.TestAssemblyName);
if (cancellationTokenSource.IsCancellationRequested)
{
return;
}
var testsResults = await _nativeTestsRunner.ExecuteTestsAsync(
testRun.TestTechnology,
testAgentRun.TestList,
tempExecutionFolder,
testAgentRun.TestRunId,
testRunTestsAssemblyPath,
testRun.TestAssemblyName,
testRun.RunInParallel,
testRun.NativeArguments,
testAgentRunTimeout,
testRun.IsTimeBasedBalance,
testRun.SameMachineByClass,
cancellationTokenSource).ConfigureAwait(false);
var retriedTestResults = string.Empty;
if (testRun.RetriesCount > 0)
{
retriedTestResults = await _nativeTestsRunner.ExecuteTestsWithRetryAsync(
testRun.TestTechnology,
testsResults,
testAgentRun.TestList,
tempExecutionFolder,
testAgentRun.TestRunId,
testRunTestsAssemblyPath,
testRun.TestAssemblyName,
testRun.RunInParallel,
testRun.NativeArguments,
testAgentRunTimeout,
testRun.RetriesCount,
testRun.Threshold,
testRun.IsTimeBasedBalance,
testRun.SameMachineByClass,
cancellationTokenSource).ConfigureAwait(false);
}
if (cancellationTokenSource.IsCancellationRequested)
{
return;
}
await CompleteTestAgentRunAsync(testAgentRun.TestAgentRunId, testsResults, retriedTestResults).ConfigureAwait(false);
_pluginService.ExecuteAllTestAgentPluginsPostTestRunLogic();
DeleteTempExecutionFolder(tempExecutionFolder);
}
public async Task <bool> ExecuteAsync(TestRunSettings testRunSettings)
{
if (!_pathProvider.IsFilePathValid(testRunSettings.ResultsFilePath))
{
throw new ArgumentException($"The specified test results file path is not valid. Specified path = {testRunSettings.ResultsFilePath}");
}
_pluginService.ExecuteAllTestRunnerPluginsPreTestRunLogic();
await _testRunsCleanerServiceClient.DeleteOldTestRunsDataAsync();
await _testCasesHistoryService.DeleteOlderTestCasesHistoryAsync();
var activeTestAgents = await _testAgentService.GetAllActiveTestAgentsByTagAsync(testRunSettings.AgentTag);
await _testAgentService.SetAllActiveAgentsToVerifyTheirStatusAsync(testRunSettings.AgentTag);
await _testAgentService.WaitAllActiveAgentsToVerifyTheirStatusAsync(activeTestAgents);
var availableTestAgents = await _testAgentService.GetAllActiveTestAgentsByTagAsync(testRunSettings.AgentTag);
bool wasSuccessfulRun = false;
if (availableTestAgents.Count > 0)
{
var tempFilePath = _pathProvider.GetTempFileName();
_fileProvider.Delete(tempFilePath);
_fileProvider.CreateZip(testRunSettings.OutputFilesLocation, tempFilePath);
var zipData = _fileProvider.ReadAllBytes(tempFilePath);
var testRunId = await _testRunProvider.CreateNewTestRunAsync(_pathProvider.GetFileName(testRunSettings.TestLibraryPath), zipData, testRunSettings.RetriesCount, testRunSettings.Threshold, testRunSettings.RunInParallel, testRunSettings.MaxParallelProcessesCount, testRunSettings.NativeArguments, testRunSettings.TestTechnology, testRunSettings.TimeBasedBalance, testRunSettings.CustomArguments);
_testCasesProvider = _pluginService.GetNativeTestsRunnerTestCasesPluginService(testRunSettings.TestTechnology);
var allTestCases = _testCasesProvider.ExtractAllTestCasesFromTestLibrary(testRunSettings.TestLibraryPath);
var filteredTestCases = _testCasesFilterService.FilterCases(allTestCases, testRunSettings.TestsFilter);
var distributedTestsLists = testRunSettings.TimeBasedBalance ?
await _testsTimesBasedDistributeService.GenerateDistributionListsAsync(availableTestAgents.Count, filteredTestCases) :
_testCountsBasedDistributeService.GenerateDistributionLists(availableTestAgents.Count, filteredTestCases);
var testAgentRuns = await _testAgentRunProvider.CreateNewTestAgentRunsAsync(testRunId, availableTestAgents, distributedTestsLists);
try
{
// TODO: pass ExecutionFrequency from args console?
await _testAgentRunProvider.WaitForTestAgentRunsToFinishAsync(testAgentRuns, testRunSettings.TestRunTimeout, ExecutionFrequency);
// DEBUG:
////_consoleProvider.WriteLine("AFTER WaitForTestAgentRunsToFinishAsync");
_consoleProvider.WriteLine(TestAgentRunsHasFinished);
}
catch (TimeoutException)
{
_consoleProvider.WriteLine(string.Format(TestRunHasTimedOut, testRunSettings.TestRunTimeout));
await _testAgentRunProvider.AbortAllTestAgentRunsInTestRunAsync(testRunId);
}
var areThereAbortedTestAgentRuns = await _testAgentRunProvider.AreThereAbortedTestAgentRunsAsync(testRunId);
if (!areThereAbortedTestAgentRuns)
{
// DEBUG:
////_consoleProvider.WriteLine("START COMPLEETING TEST RUN");
await _testRunProvider.CompleteTestRunAsync(testRunId, TestRunStatus.Completed);
wasSuccessfulRun = true;
// DEBUG:
_consoleProvider.WriteLine("TEST RUN COMPLETED");
await _testResultsService.SaveTestResultsForCurrentRunAsync(testRunSettings.TestTechnology, testRunSettings.ResultsFilePath, testRunSettings.RetriedResultsFilePath, testRunId);
try
{
await _testRunsCleanerServiceClient.DeleteOldTestRunDataByTestRunIdAsync(testRunId);
}
catch (Exception e)
{
_consoleProvider.WriteLine(e.ToString());
}
}
else
{
_consoleProvider.WriteLine("Test Run Aborted!");
await _testRunProvider.CompleteTestRunAsync(testRunId, TestRunStatus.Aborted);
}
}
else
{
_consoleProvider.WriteLine(NoTestAgentsAreAvailable);
}
_pluginService.ExecuteAllTestRunnerPluginsPostTestRunLogic();
return(wasSuccessfulRun);
}
private async Task <string> ExecuteTestsWithNativeRunnerAsync(
string workingDir,
string testsLibraryPath,
string assemblyName,
bool runInParallel,
int maxParallelProcessesCount,
string nativeArguments,
int testAgentRunTimeout,
bool isTimeBasedBalance,
bool sameMachineByClass,
List <TestCase> distributedTestCases,
CancellationTokenSource outerCancellationTokenSource)
{
var outputFilesDir = _pathProvider.GetDirectoryName(testsLibraryPath);
var availableCores = runInParallel ? maxParallelProcessesCount : 1;
// Merge logic of CreateRunFilterArgument here. remove inner foreach
var listOfDistributedTestCases = _nativeTestsRunner.SplitTestCases(availableCores, sameMachineByClass, distributedTestCases);
var testRunsToBeMerged = new List <object>();
var testRunProcesses = new List <Process>();
var resultsFiles = new List <string>();
var processCreationTime = _dateTimeProvider.GetCurrentTime();
foreach (var distributedTestCasesList in listOfDistributedTestCases)
{
var currentTestResultsFilePath = _pathProvider.GetTempFileName();
resultsFiles.Add(currentTestResultsFilePath);
var arguments = _nativeTestsRunner.BuildNativeRunnerArguments(
assemblyName,
testsLibraryPath,
distributedTestCasesList,
currentTestResultsFilePath,
outputFilesDir,
nativeArguments);
var currentProcess = _processStarter.InitializeProcess(
_nativeTestsRunner.RunnerFile,
workingDir,
arguments,
LogStandardOutput,
LogErrorOutput);
testRunProcesses.Add(currentProcess);
}
var innerCancellationTokenSource = new CancellationTokenSource();
var waitForNativeRunnerProcessesToFinishTask = _taskProvider.StartNewLongRunning(
(c) =>
{
var ranProcesses = new List <int>();
do
{
var coresCount = availableCores;
if (runInParallel)
{
foreach (var process in testRunProcesses)
{
if (coresCount == 0)
{
break;
}
if (!ranProcesses.Contains(process.GetHashCode()))
{
_processStarter.StartProcess(process, LogStandardOutput, LogErrorOutput);
Thread.Sleep(500);
ranProcesses.Add(process.GetHashCode());
coresCount--;
}
}
}
foreach (var process in testRunProcesses)
{
if (outerCancellationTokenSource.Token.IsCancellationRequested)
{
return;
}
if (!runInParallel)
{
// Start processes one by one, otherwise they are started all upfront.
_processStarter.StartProcess(process, LogStandardOutput, LogErrorOutput);
ranProcesses.Add(process.GetHashCode());
}
if (ranProcesses.Contains(process.GetHashCode()))
{
_processStarter.WaitForProcessToFinish(testAgentRunTimeout, process);
}
}
}while (ranProcesses.Count != testRunProcesses.Count);
},
innerCancellationTokenSource);
var checkCancellationRequestsTask = _taskProvider.StartNewLongRunningRepeating(
innerCancellationTokenSource,
() =>
{
if (waitForNativeRunnerProcessesToFinishTask.IsCompleted || waitForNativeRunnerProcessesToFinishTask.IsFaulted)
{
if (waitForNativeRunnerProcessesToFinishTask.IsFaulted)
{
_testRunLogService.CreateTestRunLogAsync($"waitForNativeRunnerProcessesToFinishTask FAULTED- {waitForNativeRunnerProcessesToFinishTask.Exception}", _currentTestRunId).Wait();
}
innerCancellationTokenSource.Cancel();
}
else if (outerCancellationTokenSource.Token.IsCancellationRequested)
{
foreach (var processName in _nativeTestsRunner.RunnerProcessesNamesToKill)
{
var processes = Process.GetProcessesByName(processName);
foreach (var process in processes)
{
try
{
if (process.StartTime > processCreationTime)
{
process.Kill();
process.WaitForExit();
}
}
catch (Exception e)
{
_consoleProvider.WriteLine(e.ToString());
}
}
}
}
},
500);
checkCancellationRequestsTask.Wait();
string result = null;
if (!outerCancellationTokenSource.Token.IsCancellationRequested)
{
foreach (var testResultFile in resultsFiles)
{
if (_fileProvider.Exists(testResultFile))
{
var testTestResults = _fileProvider.ReadAllText(testResultFile);
var currentTestRun = _nativeTestsRunner.DeserializeTestResults(testTestResults);
testRunsToBeMerged.Add(currentTestRun);
}
else if (_directoryProvider.Exists(testResultFile))
{
// TODO: Added this because of the Protractor Plugin, since it produces folder instead of file.
// Fix it later with something more generic solution. Maybe, we have to save the test results to plugin folder and
// add a plugin method for deleting them at the end.
var files = _directoryProvider.GetFiles(testResultFile);
if (!files.Any())
{
throw new Exception("No test results' file was produced for test run");
}
var firstFile = _directoryProvider.GetFiles(testResultFile).First();
var testTestResults = _fileProvider.ReadAllText(firstFile);
var currentTestRun = _nativeTestsRunner.DeserializeTestResults(testTestResults);
testRunsToBeMerged.Add(currentTestRun);
}
else
{
throw new Exception("No test results' file was produced for test run");
}
}
var mergedTestRun = _nativeTestsRunner.MergeTestResults(testRunsToBeMerged);
if (isTimeBasedBalance)
{
var testCaseRuns = _nativeTestsRunner.UpdateTestCasesHistory(mergedTestRun, assemblyName);
await _testCasesHistoryService.UpdateTestCaseExecutionHistoryAsync(testCaseRuns).ConfigureAwait(false);
}
try
{
_nativeTestsRunner.ExecutePostRunActions();
}
catch (Exception ex)
{
_testRunLogService.CreateTestRunLogAsync($"There was a problem executing ExecutePostRunActions on {Environment.MachineName}. Exception: {ex}", _currentTestRunId).Wait();
}
result = mergedTestRun;
}
return(result);
}
private async Task <string> ExecuteTestsWithNativeRunnerAsync(
string workingDir,
string testsLibraryPath,
string assemblyName,
bool runInParallel,
int maxParallelProcessesCount,
string nativeArguments,
int testAgentRunTimeout,
bool isTimeBasedBalance,
List <TestCase> distributedTestCases,
CancellationTokenSource outerCancellationTokenSource)
{
string outputFilesDir = _pathProvider.GetDirectoryName(testsLibraryPath);
////var updatedMaxParallelProcessesCount = maxParallelProcessesCount > 1 ? maxParallelProcessesCount - 1 : maxParallelProcessesCount;
int availableCores = runInParallel ? maxParallelProcessesCount : 1;
// Merge logic of CreateRunFilterArgument here. remove inner foreach
var listOfDistributedTestCases = _nativeTestsRunner.SplitTestCases(distributedTestCases, availableCores);
var testRunsToBeMerged = new List <object>();
var testRunProcesses = new List <Process>();
var resultsFiles = new List <string>();
DateTime processCreationTime = _dateTimeProvider.GetCurrentTime();
// DEBUG:
////await _testRunLogService.CreateTestRunLogAsync($"Number of tests to be executed- {distributedTestCases.Count}", _currentTestRunId);
// DEBUG:
////await _testRunLogService.CreateTestRunLogAsync($"Number of listOfDistributedTestCases- {listOfDistributedTestCases.Count()}", _currentTestRunId);
int index = 1;
foreach (var distributedTestCasesList in listOfDistributedTestCases)
{
// DEBUG:
////await _testRunLogService.CreateTestRunLogAsync($"{index++} distributedTestCasesList items- {distributedTestCasesList.Count()}", _currentTestRunId);
var currentTestResultsFilePath = _pathProvider.GetTempFileName();
resultsFiles.Add(currentTestResultsFilePath);
var arguments = _nativeTestsRunner.BuildNativeRunnerArguments(
assemblyName,
testsLibraryPath,
distributedTestCasesList,
currentTestResultsFilePath,
outputFilesDir,
nativeArguments);
var currentProcess = _processStarter.InitializeProcess(
_nativeTestsRunner.RunnerFile,
workingDir,
arguments,
LogStandardOutput,
LogErrorOutput);
testRunProcesses.Add(currentProcess);
}
var innerCancellationTokenSource = new CancellationTokenSource();
// DEBUG:
////await _testRunLogService.CreateTestRunLogAsync($"The native runs will be executed in parallel- {runInParallel}. Available core on machine {Environment.MachineName}- {availableCores}", _currentTestRunId);
////await _testRunLogService.CreateTestRunLogAsync($"Number of native test runner process to be run- {testRunProcesses.Count}", _currentTestRunId);
var waitForNativeRunnerProcessesToFinishTask = _taskProvider.StartNewLongRunning(
(c) =>
{
var ranProcesses = new List <int>();
do
{
var coresCount = availableCores;
if (runInParallel)
{
foreach (var process in testRunProcesses)
{
if (coresCount == 0)
{
break;
}
if (!ranProcesses.Contains(process.GetHashCode()))
{
_processStarter.StartProcess(process, LogStandardOutput, LogErrorOutput);
Thread.Sleep(1000);
ranProcesses.Add(process.GetHashCode());
coresCount--;
}
}
}
// Do not start all processes upfront
// if parallel here start all of them?
// run in task and pass cancelation token. If is canceled kill all processes. DO IT for NUNIT TOO
foreach (var process in testRunProcesses)
{
if (outerCancellationTokenSource.Token.IsCancellationRequested)
{
return;
}
if (!runInParallel)
{
// Start processes one by one, otherwise they are started all upfront.
_processStarter.StartProcess(process, LogStandardOutput, LogErrorOutput);
ranProcesses.Add(process.GetHashCode());
}
if (ranProcesses.Contains(process.GetHashCode()))
{
_processStarter.WaitForProcessToFinish(testAgentRunTimeout, process);
}
}
// DEBUG:
////_testRunLogService.CreateTestRunLogAsync($"ranProcesses.Count {ranProcesses.Count} testRunProcesses.Count {testRunProcesses.Count} {_dateTimeProvider.GetCurrentTime()}", _currentTestRunId).Wait();
}while (ranProcesses.Count != testRunProcesses.Count);
},
innerCancellationTokenSource);
var checkCancellationRequestsTask = _taskProvider.StartNewLongRunningRepeating(
innerCancellationTokenSource,
() =>
{
if (waitForNativeRunnerProcessesToFinishTask.IsCompleted || waitForNativeRunnerProcessesToFinishTask.IsFaulted)
{
if (waitForNativeRunnerProcessesToFinishTask.IsFaulted)
{
_testRunLogService.CreateTestRunLogAsync($"waitForNativeRunnerProcessesToFinishTask FAULTED- {waitForNativeRunnerProcessesToFinishTask.Exception}", _currentTestRunId).Wait();
}
innerCancellationTokenSource.Cancel();
return;
}
else if (outerCancellationTokenSource.Token.IsCancellationRequested)
{
foreach (var processName in _nativeTestsRunner.RunnerProcessesNamesToKill)
{
var processes = Process.GetProcessesByName(processName);
foreach (var process in processes)
{
try
{
if (process.StartTime > processCreationTime)
{
process.Kill();
process.WaitForExit();
}
}
catch (Exception e)
{
_consoleProvider.WriteLine(e.ToString());
}
}
}
return;
}
},
5000);
checkCancellationRequestsTask.Wait();
string result = null;
if (!outerCancellationTokenSource.Token.IsCancellationRequested)
{
// DEBUG:
_testRunLogService.CreateTestRunLogAsync($"START MERGING RESULTS- on machine {Environment.MachineName}", _currentTestRunId).Wait();
foreach (var testResultFile in resultsFiles)
{
if (_fileProvider.Exists(testResultFile))
{
var testTestResults = _fileProvider.ReadAllText(testResultFile);
var currentTestRun = _nativeTestsRunner.DeserializeTestResults(testTestResults);
testRunsToBeMerged.Add(currentTestRun);
}
else
{
throw new Exception("No test results' file was produced for test run");
}
}
var mergedTestRun = _nativeTestsRunner.MergeTestResults(testRunsToBeMerged);
if (isTimeBasedBalance)
{
// DEBUG:
var startTime = _dateTimeProvider.GetCurrentTime();
_testRunLogService.CreateTestRunLogAsync($"START updating test case history- on machine {Environment.MachineName}", _currentTestRunId).Wait();
var testCaseRuns = _nativeTestsRunner.UpdateTestCasesHistory(mergedTestRun, assemblyName);
await _testCasesHistoryService.UpdateTestCaseExecutionHistoryAsync(testCaseRuns);
// DEBUG:
var endTime = _dateTimeProvider.GetCurrentTime();
_testRunLogService.CreateTestRunLogAsync($"END updating test case history- on machine {Environment.MachineName} for {(endTime - startTime).Seconds} seconds", _currentTestRunId).Wait();
}
result = mergedTestRun;
}
return(result);
}