Beispiel #1
0
 public static string GetTempFileName()
 {
     if (_provider == null)
     {
         return(Path.GetTempFileName());
     }
     return(_provider.GetTempFileName());
 }
Beispiel #2
0
        public string CreateLocalTestListFileAsync(string testList)
        {
            if (string.IsNullOrEmpty(testList))
            {
                throw new ArgumentNullException(TestListNullExceptionMessage);
            }

            var testListContent  = testList;
            var testListFilePath = _pathProvider.GetTempFileName();

            _fileProvider.WriteAllText(testListFilePath, testListContent);

            return(testListFilePath);
        }
Beispiel #3
0
        private async Task ExecuteTestAgentRunAsync(TestAgentRunDto testAgentRun, int testAgentRunTimeout, CancellationTokenSource cancellationTokenSource)
        {
            _pluginService.ExecuteAllTestAgentPluginsPreTestRunLogic();

            testAgentRun.Status = TestAgentRunStatus.InProgress;
            await CreateEnvironmentVariablesForCustomArgumentsAsync(testAgentRun.TestRunId).ConfigureAwait(false);

            await _testAgentRunRepository.UpdateAsync(testAgentRun.TestAgentRunId, testAgentRun).ConfigureAwait(false);

            var testRun = await _testRunRepository.GetAsync(testAgentRun.TestRunId).ConfigureAwait(false);

            var tempExecutionFolder = _pathProvider.Combine(_pathProvider.GetTempFolderPath(), _guidService.NewGuid().ToString());
            var tempZipFileName     = _pathProvider.GetTempFileName();
            var testRunOutput       = await _testRunOutputServiceClient.GetTestRunOutputByTestRunIdAsync(testRun.TestRunId).ConfigureAwait(false);

            if (testRunOutput == null)
            {
                // DEBUG:
                await _testRunLogService.CreateTestRunLogAsync("The test run output cannot be null.", testAgentRun.TestRunId).ConfigureAwait(false);

                throw new ArgumentException("The test run output cannot be null.");
            }

            _fileProvider.WriteAllBytes(tempZipFileName, testRunOutput.TestOutputFilesPackage);
            _fileProvider.ExtractZip(tempZipFileName, tempExecutionFolder);
            var testRunTestsAssemblyPath = _pathProvider.Combine(tempExecutionFolder, testRun.TestAssemblyName);

            if (cancellationTokenSource.IsCancellationRequested)
            {
                return;
            }

            var testsResults = await _nativeTestsRunner.ExecuteTestsAsync(
                testRun.TestTechnology,
                testAgentRun.TestList,
                tempExecutionFolder,
                testAgentRun.TestRunId,
                testRunTestsAssemblyPath,
                testRun.TestAssemblyName,
                testRun.RunInParallel,
                testRun.NativeArguments,
                testAgentRunTimeout,
                testRun.IsTimeBasedBalance,
                testRun.SameMachineByClass,
                cancellationTokenSource).ConfigureAwait(false);

            var retriedTestResults = string.Empty;

            if (testRun.RetriesCount > 0)
            {
                retriedTestResults = await _nativeTestsRunner.ExecuteTestsWithRetryAsync(
                    testRun.TestTechnology,
                    testsResults,
                    testAgentRun.TestList,
                    tempExecutionFolder,
                    testAgentRun.TestRunId,
                    testRunTestsAssemblyPath,
                    testRun.TestAssemblyName,
                    testRun.RunInParallel,
                    testRun.NativeArguments,
                    testAgentRunTimeout,
                    testRun.RetriesCount,
                    testRun.Threshold,
                    testRun.IsTimeBasedBalance,
                    testRun.SameMachineByClass,
                    cancellationTokenSource).ConfigureAwait(false);
            }

            if (cancellationTokenSource.IsCancellationRequested)
            {
                return;
            }

            await CompleteTestAgentRunAsync(testAgentRun.TestAgentRunId, testsResults, retriedTestResults).ConfigureAwait(false);

            _pluginService.ExecuteAllTestAgentPluginsPostTestRunLogic();

            DeleteTempExecutionFolder(tempExecutionFolder);
        }
Beispiel #4
0
        public async Task <bool> ExecuteAsync(TestRunSettings testRunSettings)
        {
            if (!_pathProvider.IsFilePathValid(testRunSettings.ResultsFilePath))
            {
                throw new ArgumentException($"The specified test results file path is not valid. Specified path = {testRunSettings.ResultsFilePath}");
            }

            _pluginService.ExecuteAllTestRunnerPluginsPreTestRunLogic();

            await _testRunsCleanerServiceClient.DeleteOldTestRunsDataAsync();

            await _testCasesHistoryService.DeleteOlderTestCasesHistoryAsync();

            var activeTestAgents = await _testAgentService.GetAllActiveTestAgentsByTagAsync(testRunSettings.AgentTag);

            await _testAgentService.SetAllActiveAgentsToVerifyTheirStatusAsync(testRunSettings.AgentTag);

            await _testAgentService.WaitAllActiveAgentsToVerifyTheirStatusAsync(activeTestAgents);

            var availableTestAgents = await _testAgentService.GetAllActiveTestAgentsByTagAsync(testRunSettings.AgentTag);

            bool wasSuccessfulRun = false;

            if (availableTestAgents.Count > 0)
            {
                var tempFilePath = _pathProvider.GetTempFileName();
                _fileProvider.Delete(tempFilePath);
                _fileProvider.CreateZip(testRunSettings.OutputFilesLocation, tempFilePath);
                var zipData = _fileProvider.ReadAllBytes(tempFilePath);

                var testRunId = await _testRunProvider.CreateNewTestRunAsync(_pathProvider.GetFileName(testRunSettings.TestLibraryPath), zipData, testRunSettings.RetriesCount, testRunSettings.Threshold, testRunSettings.RunInParallel, testRunSettings.MaxParallelProcessesCount, testRunSettings.NativeArguments, testRunSettings.TestTechnology, testRunSettings.TimeBasedBalance, testRunSettings.CustomArguments);

                _testCasesProvider = _pluginService.GetNativeTestsRunnerTestCasesPluginService(testRunSettings.TestTechnology);
                var allTestCases      = _testCasesProvider.ExtractAllTestCasesFromTestLibrary(testRunSettings.TestLibraryPath);
                var filteredTestCases = _testCasesFilterService.FilterCases(allTestCases, testRunSettings.TestsFilter);

                var distributedTestsLists = testRunSettings.TimeBasedBalance ?
                                            await _testsTimesBasedDistributeService.GenerateDistributionListsAsync(availableTestAgents.Count, filteredTestCases) :
                                            _testCountsBasedDistributeService.GenerateDistributionLists(availableTestAgents.Count, filteredTestCases);

                var testAgentRuns = await _testAgentRunProvider.CreateNewTestAgentRunsAsync(testRunId, availableTestAgents, distributedTestsLists);

                try
                {
                    // TODO: pass ExecutionFrequency from args console?
                    await _testAgentRunProvider.WaitForTestAgentRunsToFinishAsync(testAgentRuns, testRunSettings.TestRunTimeout, ExecutionFrequency);

                    // DEBUG:
                    ////_consoleProvider.WriteLine("AFTER WaitForTestAgentRunsToFinishAsync");
                    _consoleProvider.WriteLine(TestAgentRunsHasFinished);
                }
                catch (TimeoutException)
                {
                    _consoleProvider.WriteLine(string.Format(TestRunHasTimedOut, testRunSettings.TestRunTimeout));
                    await _testAgentRunProvider.AbortAllTestAgentRunsInTestRunAsync(testRunId);
                }

                var areThereAbortedTestAgentRuns = await _testAgentRunProvider.AreThereAbortedTestAgentRunsAsync(testRunId);

                if (!areThereAbortedTestAgentRuns)
                {
                    // DEBUG:
                    ////_consoleProvider.WriteLine("START COMPLEETING TEST RUN");
                    await _testRunProvider.CompleteTestRunAsync(testRunId, TestRunStatus.Completed);

                    wasSuccessfulRun = true;

                    // DEBUG:
                    _consoleProvider.WriteLine("TEST RUN COMPLETED");
                    await _testResultsService.SaveTestResultsForCurrentRunAsync(testRunSettings.TestTechnology, testRunSettings.ResultsFilePath, testRunSettings.RetriedResultsFilePath, testRunId);

                    try
                    {
                        await _testRunsCleanerServiceClient.DeleteOldTestRunDataByTestRunIdAsync(testRunId);
                    }
                    catch (Exception e)
                    {
                        _consoleProvider.WriteLine(e.ToString());
                    }
                }
                else
                {
                    _consoleProvider.WriteLine("Test Run Aborted!");
                    await _testRunProvider.CompleteTestRunAsync(testRunId, TestRunStatus.Aborted);
                }
            }
            else
            {
                _consoleProvider.WriteLine(NoTestAgentsAreAvailable);
            }

            _pluginService.ExecuteAllTestRunnerPluginsPostTestRunLogic();

            return(wasSuccessfulRun);
        }
Beispiel #5
0
        private async Task <string> ExecuteTestsWithNativeRunnerAsync(
            string workingDir,
            string testsLibraryPath,
            string assemblyName,
            bool runInParallel,
            int maxParallelProcessesCount,
            string nativeArguments,
            int testAgentRunTimeout,
            bool isTimeBasedBalance,
            bool sameMachineByClass,
            List <TestCase> distributedTestCases,
            CancellationTokenSource outerCancellationTokenSource)
        {
            var outputFilesDir = _pathProvider.GetDirectoryName(testsLibraryPath);
            var availableCores = runInParallel ? maxParallelProcessesCount : 1;

            // Merge logic of CreateRunFilterArgument here. remove inner foreach
            var listOfDistributedTestCases = _nativeTestsRunner.SplitTestCases(availableCores, sameMachineByClass, distributedTestCases);
            var testRunsToBeMerged         = new List <object>();
            var testRunProcesses           = new List <Process>();
            var resultsFiles = new List <string>();

            var processCreationTime = _dateTimeProvider.GetCurrentTime();

            foreach (var distributedTestCasesList in listOfDistributedTestCases)
            {
                var currentTestResultsFilePath = _pathProvider.GetTempFileName();
                resultsFiles.Add(currentTestResultsFilePath);
                var arguments = _nativeTestsRunner.BuildNativeRunnerArguments(
                    assemblyName,
                    testsLibraryPath,
                    distributedTestCasesList,
                    currentTestResultsFilePath,
                    outputFilesDir,
                    nativeArguments);

                var currentProcess = _processStarter.InitializeProcess(
                    _nativeTestsRunner.RunnerFile,
                    workingDir,
                    arguments,
                    LogStandardOutput,
                    LogErrorOutput);
                testRunProcesses.Add(currentProcess);
            }

            var innerCancellationTokenSource = new CancellationTokenSource();

            var waitForNativeRunnerProcessesToFinishTask = _taskProvider.StartNewLongRunning(
                (c) =>
            {
                var ranProcesses = new List <int>();
                do
                {
                    var coresCount = availableCores;
                    if (runInParallel)
                    {
                        foreach (var process in testRunProcesses)
                        {
                            if (coresCount == 0)
                            {
                                break;
                            }

                            if (!ranProcesses.Contains(process.GetHashCode()))
                            {
                                _processStarter.StartProcess(process, LogStandardOutput, LogErrorOutput);
                                Thread.Sleep(500);
                                ranProcesses.Add(process.GetHashCode());
                                coresCount--;
                            }
                        }
                    }

                    foreach (var process in testRunProcesses)
                    {
                        if (outerCancellationTokenSource.Token.IsCancellationRequested)
                        {
                            return;
                        }

                        if (!runInParallel)
                        {
                            // Start processes one by one, otherwise they are started all upfront.
                            _processStarter.StartProcess(process, LogStandardOutput, LogErrorOutput);
                            ranProcesses.Add(process.GetHashCode());
                        }

                        if (ranProcesses.Contains(process.GetHashCode()))
                        {
                            _processStarter.WaitForProcessToFinish(testAgentRunTimeout, process);
                        }
                    }
                }while (ranProcesses.Count != testRunProcesses.Count);
            },
                innerCancellationTokenSource);

            var checkCancellationRequestsTask = _taskProvider.StartNewLongRunningRepeating(
                innerCancellationTokenSource,
                () =>
            {
                if (waitForNativeRunnerProcessesToFinishTask.IsCompleted || waitForNativeRunnerProcessesToFinishTask.IsFaulted)
                {
                    if (waitForNativeRunnerProcessesToFinishTask.IsFaulted)
                    {
                        _testRunLogService.CreateTestRunLogAsync($"waitForNativeRunnerProcessesToFinishTask FAULTED- {waitForNativeRunnerProcessesToFinishTask.Exception}", _currentTestRunId).Wait();
                    }

                    innerCancellationTokenSource.Cancel();
                }
                else if (outerCancellationTokenSource.Token.IsCancellationRequested)
                {
                    foreach (var processName in _nativeTestsRunner.RunnerProcessesNamesToKill)
                    {
                        var processes = Process.GetProcessesByName(processName);
                        foreach (var process in processes)
                        {
                            try
                            {
                                if (process.StartTime > processCreationTime)
                                {
                                    process.Kill();
                                    process.WaitForExit();
                                }
                            }
                            catch (Exception e)
                            {
                                _consoleProvider.WriteLine(e.ToString());
                            }
                        }
                    }
                }
            },
                500);

            checkCancellationRequestsTask.Wait();

            string result = null;

            if (!outerCancellationTokenSource.Token.IsCancellationRequested)
            {
                foreach (var testResultFile in resultsFiles)
                {
                    if (_fileProvider.Exists(testResultFile))
                    {
                        var testTestResults = _fileProvider.ReadAllText(testResultFile);
                        var currentTestRun  = _nativeTestsRunner.DeserializeTestResults(testTestResults);
                        testRunsToBeMerged.Add(currentTestRun);
                    }
                    else if (_directoryProvider.Exists(testResultFile))
                    {
                        // TODO: Added this because of the Protractor Plugin, since it produces folder instead of file.
                        // Fix it later with something more generic solution. Maybe, we have to save the test results to plugin folder and
                        // add a plugin method for deleting them at the end.
                        var files = _directoryProvider.GetFiles(testResultFile);
                        if (!files.Any())
                        {
                            throw new Exception("No test results' file was produced for test run");
                        }

                        var firstFile       = _directoryProvider.GetFiles(testResultFile).First();
                        var testTestResults = _fileProvider.ReadAllText(firstFile);
                        var currentTestRun  = _nativeTestsRunner.DeserializeTestResults(testTestResults);
                        testRunsToBeMerged.Add(currentTestRun);
                    }
                    else
                    {
                        throw new Exception("No test results' file was produced for test run");
                    }
                }

                var mergedTestRun = _nativeTestsRunner.MergeTestResults(testRunsToBeMerged);
                if (isTimeBasedBalance)
                {
                    var testCaseRuns = _nativeTestsRunner.UpdateTestCasesHistory(mergedTestRun, assemblyName);
                    await _testCasesHistoryService.UpdateTestCaseExecutionHistoryAsync(testCaseRuns).ConfigureAwait(false);
                }

                try
                {
                    _nativeTestsRunner.ExecutePostRunActions();
                }
                catch (Exception ex)
                {
                    _testRunLogService.CreateTestRunLogAsync($"There was a problem executing ExecutePostRunActions on {Environment.MachineName}. Exception: {ex}", _currentTestRunId).Wait();
                }

                result = mergedTestRun;
            }

            return(result);
        }
        private async Task <string> ExecuteTestsWithNativeRunnerAsync(
            string workingDir,
            string testsLibraryPath,
            string assemblyName,
            bool runInParallel,
            int maxParallelProcessesCount,
            string nativeArguments,
            int testAgentRunTimeout,
            bool isTimeBasedBalance,
            List <TestCase> distributedTestCases,
            CancellationTokenSource outerCancellationTokenSource)
        {
            string outputFilesDir = _pathProvider.GetDirectoryName(testsLibraryPath);
            ////var updatedMaxParallelProcessesCount = maxParallelProcessesCount > 1 ? maxParallelProcessesCount - 1 : maxParallelProcessesCount;
            int availableCores = runInParallel ? maxParallelProcessesCount : 1;

            // Merge logic of CreateRunFilterArgument here. remove inner foreach
            var listOfDistributedTestCases = _nativeTestsRunner.SplitTestCases(distributedTestCases, availableCores);
            var testRunsToBeMerged         = new List <object>();

            var testRunProcesses = new List <Process>();
            var resultsFiles     = new List <string>();

            DateTime processCreationTime = _dateTimeProvider.GetCurrentTime();

            // DEBUG:
            ////await _testRunLogService.CreateTestRunLogAsync($"Number of tests to be executed- {distributedTestCases.Count}", _currentTestRunId);

            // DEBUG:
            ////await _testRunLogService.CreateTestRunLogAsync($"Number of listOfDistributedTestCases- {listOfDistributedTestCases.Count()}", _currentTestRunId);
            int index = 1;

            foreach (var distributedTestCasesList in listOfDistributedTestCases)
            {
                // DEBUG:
                ////await _testRunLogService.CreateTestRunLogAsync($"{index++} distributedTestCasesList items- {distributedTestCasesList.Count()}", _currentTestRunId);
                var currentTestResultsFilePath = _pathProvider.GetTempFileName();
                resultsFiles.Add(currentTestResultsFilePath);
                var arguments = _nativeTestsRunner.BuildNativeRunnerArguments(
                    assemblyName,
                    testsLibraryPath,
                    distributedTestCasesList,
                    currentTestResultsFilePath,
                    outputFilesDir,
                    nativeArguments);

                var currentProcess = _processStarter.InitializeProcess(
                    _nativeTestsRunner.RunnerFile,
                    workingDir,
                    arguments,
                    LogStandardOutput,
                    LogErrorOutput);
                testRunProcesses.Add(currentProcess);
            }

            var innerCancellationTokenSource = new CancellationTokenSource();

            // DEBUG:
            ////await _testRunLogService.CreateTestRunLogAsync($"The native runs will be executed in parallel- {runInParallel}. Available core on machine {Environment.MachineName}- {availableCores}", _currentTestRunId);
            ////await _testRunLogService.CreateTestRunLogAsync($"Number of native test runner process to be run- {testRunProcesses.Count}", _currentTestRunId);
            var waitForNativeRunnerProcessesToFinishTask = _taskProvider.StartNewLongRunning(
                (c) =>
            {
                var ranProcesses = new List <int>();
                do
                {
                    var coresCount = availableCores;
                    if (runInParallel)
                    {
                        foreach (var process in testRunProcesses)
                        {
                            if (coresCount == 0)
                            {
                                break;
                            }

                            if (!ranProcesses.Contains(process.GetHashCode()))
                            {
                                _processStarter.StartProcess(process, LogStandardOutput, LogErrorOutput);
                                Thread.Sleep(1000);
                                ranProcesses.Add(process.GetHashCode());
                                coresCount--;
                            }
                        }
                    }

                    // Do not start all processes upfront
                    // if parallel here start all of them?
                    // run in task and pass cancelation token. If is canceled kill all processes. DO IT for NUNIT TOO
                    foreach (var process in testRunProcesses)
                    {
                        if (outerCancellationTokenSource.Token.IsCancellationRequested)
                        {
                            return;
                        }

                        if (!runInParallel)
                        {
                            // Start processes one by one, otherwise they are started all upfront.
                            _processStarter.StartProcess(process, LogStandardOutput, LogErrorOutput);
                            ranProcesses.Add(process.GetHashCode());
                        }

                        if (ranProcesses.Contains(process.GetHashCode()))
                        {
                            _processStarter.WaitForProcessToFinish(testAgentRunTimeout, process);
                        }
                    }

                    // DEBUG:
                    ////_testRunLogService.CreateTestRunLogAsync($"ranProcesses.Count {ranProcesses.Count} testRunProcesses.Count {testRunProcesses.Count} {_dateTimeProvider.GetCurrentTime()}", _currentTestRunId).Wait();
                }while (ranProcesses.Count != testRunProcesses.Count);
            },
                innerCancellationTokenSource);

            var checkCancellationRequestsTask = _taskProvider.StartNewLongRunningRepeating(
                innerCancellationTokenSource,
                () =>
            {
                if (waitForNativeRunnerProcessesToFinishTask.IsCompleted || waitForNativeRunnerProcessesToFinishTask.IsFaulted)
                {
                    if (waitForNativeRunnerProcessesToFinishTask.IsFaulted)
                    {
                        _testRunLogService.CreateTestRunLogAsync($"waitForNativeRunnerProcessesToFinishTask FAULTED- {waitForNativeRunnerProcessesToFinishTask.Exception}", _currentTestRunId).Wait();
                    }

                    innerCancellationTokenSource.Cancel();
                    return;
                }
                else if (outerCancellationTokenSource.Token.IsCancellationRequested)
                {
                    foreach (var processName in _nativeTestsRunner.RunnerProcessesNamesToKill)
                    {
                        var processes = Process.GetProcessesByName(processName);
                        foreach (var process in processes)
                        {
                            try
                            {
                                if (process.StartTime > processCreationTime)
                                {
                                    process.Kill();
                                    process.WaitForExit();
                                }
                            }
                            catch (Exception e)
                            {
                                _consoleProvider.WriteLine(e.ToString());
                            }
                        }
                    }

                    return;
                }
            },
                5000);

            checkCancellationRequestsTask.Wait();

            string result = null;

            if (!outerCancellationTokenSource.Token.IsCancellationRequested)
            {
                // DEBUG:
                _testRunLogService.CreateTestRunLogAsync($"START MERGING RESULTS- on machine {Environment.MachineName}", _currentTestRunId).Wait();
                foreach (var testResultFile in resultsFiles)
                {
                    if (_fileProvider.Exists(testResultFile))
                    {
                        var testTestResults = _fileProvider.ReadAllText(testResultFile);
                        var currentTestRun  = _nativeTestsRunner.DeserializeTestResults(testTestResults);
                        testRunsToBeMerged.Add(currentTestRun);
                    }
                    else
                    {
                        throw new Exception("No test results' file was produced for test run");
                    }
                }

                var mergedTestRun = _nativeTestsRunner.MergeTestResults(testRunsToBeMerged);
                if (isTimeBasedBalance)
                {
                    // DEBUG:
                    var startTime = _dateTimeProvider.GetCurrentTime();
                    _testRunLogService.CreateTestRunLogAsync($"START updating test case history- on machine {Environment.MachineName}", _currentTestRunId).Wait();
                    var testCaseRuns = _nativeTestsRunner.UpdateTestCasesHistory(mergedTestRun, assemblyName);
                    await _testCasesHistoryService.UpdateTestCaseExecutionHistoryAsync(testCaseRuns);

                    // DEBUG:
                    var endTime = _dateTimeProvider.GetCurrentTime();
                    _testRunLogService.CreateTestRunLogAsync($"END updating test case history- on machine {Environment.MachineName} for {(endTime - startTime).Seconds} seconds", _currentTestRunId).Wait();
                }

                result = mergedTestRun;
            }

            return(result);
        }