public async Task UpdatePipelines(string pipelineName, int count = 10) { try { _logger.LogInformation($"Starting: Update Pipeline '{pipelineName}' for the last {count} runs"); var startPipeCounter = (await _pipelineRepository.GetLastUpdatedPipelineCounter(pipelineName) ?? 0) + 1; var updatedData = (await _pipelineProvider.GetPipelineRuns(pipelineName, startPipeCounter, count)) .ToList(); _logger.LogInformation($"Found {updatedData.Count} updated pipelines"); foreach (var pipeline in updatedData) { var existingPipeline = await _pipelineRepository.GetPipeline(pipeline.Id); _logger.LogInformation($"Analyzing pipeline {pipeline.Name} instance {pipeline.Counter}"); var ignoredJobs = new List <PipelineJob>(); foreach (var job in pipeline.PipelineJobs) { var existingJob = existingPipeline?.PipelineJobs?.FirstOrDefault(x => x.Id == job.Id); if (existingJob != null && existingJob.LogStatus != LogStatus.NotFound && existingJob.LogStatus != LogStatus.Unknown) { _logger.LogInformation($"Skipping stage {job.StageName} - job {job.Name}, Already analyzed."); ignoredJobs.Add(job); continue; } _logger.LogInformation($"Analyzing stage {job.StageName} - job {job.Name}"); var result = await _pipelineProvider.GetJobResult(pipelineName, pipeline.Counter, job.StageName, job.StageCounter, job.Name); if (result == null) { job.LogStatus = LogStatus.NotFound; continue; } if (result.StartTime.HasValue) { job.StartDate = result.StartTime.Value; } if (result.EndTime.HasValue) { job.EndDate = result.EndTime.Value; } job.Result = result.Status; job.Agent = result.Agent; job.LogStatus = LogStatus.NoParserFound; var testLookup = new Dictionary <string, Test>(); foreach (var task in result.Tasks) { if (task.Log == null) { continue; } foreach (var parser in _testLogParsers) { try { var tests = parser.Parse <Test>(task.Log); if (tests == null) { continue; //Log isn't recognized by the parser } if (!tests.Any()) { _logger.LogWarning( $"Job log doesn't have any test results! Job: {job.Name}"); } job.LogStatus = LogStatus.Parsed; foreach (var test in tests) { //Avoid test result duplication test.PipelineJobId = job.Id; if (testLookup.ContainsKey(test.Name)) { testLookup[test.Name] = test; _logger.LogWarning( $"Duplicate test detected in the same run Test: '{test.Name}'"); continue; } testLookup.Add(test.Name, test); } job.TestType = parser.TestType; break; } catch (Exception e) { _logger.LogCritical(e, parser.GetType().Name + " Parsing error"); } } } job.Tests = testLookup.Values.ToList(); if (job.LogStatus == LogStatus.Parsed && job.Tests.Count == 0) { job.LogStatus = LogStatus.NotTestsFound; } } //Remove all non-updated jobs foreach (var job in ignoredJobs) { pipeline.PipelineJobs.Remove(job); } } if (updatedData.Count == 0) { _logger.LogInformation("Pipeline is up to date!"); return; } _logger.LogInformation( $"Saving pipeline instances {updatedData.Min(x => x.Counter)} to {updatedData.Max(x => x.Counter)}"); using (var scope = new TransactionScope(TransactionScopeOption.Required, new TransactionOptions { IsolationLevel = IsolationLevel.ReadCommitted }, TransactionScopeAsyncFlowOption.Enabled)) { await _pipelineRepository.UpdatePipelines(updatedData); await _testRepository.Insert(updatedData.SelectMany(x => x.PipelineJobs.SelectMany(j => j.Tests ?? new List <Test>(0))).ToList()); scope.Complete(); } _logger.LogInformation("Saving completed"); } catch (Exception e) { _logger.LogCritical($"Critical error trying to update pipeline {pipelineName}" + e); } }