/// <summary> /// Populate the build table by processing the given message. This function doesn't handle /// any build state semantics. Instead it just processes the build and updates the build /// result tables. /// </summary> public static async Task PopulateBuildData( [QueueTrigger(QueueNames.ProcessBuild)] string message, [Table(TableNames.BuildState)] CloudTable buildStateTable, [Table(TableNames.BuildStateKey)] CloudTable buildStateKeyTable, [Table(TableNames.BuildResultDate)] CloudTable buildResultDateTable, [Table(TableNames.BuildResultExact)] CloudTable buildResultExactTable, [Table(TableNames.BuildFailureDate)] CloudTable buildFailureDateTable, [Table(TableNames.BuildFailureExact)] CloudTable buildFailureExactTable, [Table(TableNames.CounterBuilds)] CloudTable counterBuildsTable, [Table(TableNames.ViewNameDate)] CloudTable viewNameDateTable, [Queue(QueueNames.ProcessBuild)] CloudQueue processBuildQueue, [Queue(QueueNames.EmailBuild)] CloudQueue emailBuildQueue, TextWriter logger, CancellationToken cancellationToken) { var buildIdJson = (BuildStateMessage)JsonConvert.DeserializeObject(message, typeof(BuildStateMessage)); var client = StateUtil.CreateJenkinsClient(buildIdJson.BoundBuildId); var populator = new BuildTablePopulator( buildResultDateTable: buildResultDateTable, buildResultExactTable: buildResultExactTable, buildFailureDateTable: buildFailureDateTable, buildFailureExactTable: buildFailureExactTable, viewNameDateTable: viewNameDateTable, buildCounterUtil: CounterUtilFactory.Create<BuildCounterEntity>(counterBuildsTable), client: client, textWriter: logger); var stateUtil = new StateUtil( buildStateKeyTable: buildStateKeyTable, buildStateTable: buildStateTable, processBuildQueue: processBuildQueue, emailBuildQueue: emailBuildQueue, logger: logger); await stateUtil.Populate(buildIdJson, populator, cancellationToken); }
public BuildTablePopulatorTests() { var account = Util.GetStorageAccount(); var tableClient = account.CreateCloudTableClient(); _restClient = new MockRestClient(); var client = new JenkinsClient(new Uri("http://test.com"), _restClient.Client); _buildFailureExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildFailureExact); _buildResultExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildResultExact); _populator = new BuildTablePopulator( tableClient, client: client, factory: new CounterUtilFactory(), textWriter: new StringWriter()); }
/// <summary> /// The build is determined to be missing. Finish the build according to that. /// </summary> internal async Task<bool> PopulateMissing(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken) { try { await populator.PopulateBuildMissing(entity.BoundBuildId); entity.IsBuildFinished = true; entity.IsDataComplete = true; entity.Error = "Build missing"; await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken); return true; } catch (Exception ex) { // This is frankly the best possible outcome. This is the worst state we can have for a build // so any other thread giving a result can't be worse. _logger.WriteLine($"Error populating build {entity.BuildId} as missing {ex}"); return false; } }
/// <summary> /// Populate the given build and update the unprocessed table accordingly. If there is no /// existing entity in the unprocessed table, this won't add one. It will only update existing /// ones. /// </summary> internal async Task Populate(BuildStateMessage message, BuildTablePopulator populator, CancellationToken cancellationToken) { var buildId = message.BuildId; var entityKey = BuildStateEntity.GetEntityKey(message.BuildStateKey, message.BoundBuildId); var entity = await AzureUtil.QueryAsync<BuildStateEntity>(_buildStateTable, entityKey, cancellationToken); var completed = await PopulateCore(entity, populator, cancellationToken); // Unable to complete the build, consider this is a 404 missing that we need to handle. if (!completed && entity.BuildMissingCount > MissingBuildLimit) { completed = await PopulateMissing(entity, populator, cancellationToken); } if (completed) { return; } var isDone = (DateTimeOffset.UtcNow - entity.BuildStateKey.DateTime).TotalDays > DayWindow; if (isDone) { await EnqueueEmailBuild(entity.BuildStateKey, entity.BoundBuildId, cancellationToken); } else { // Wait an hour to retry. Hope that a bug fix is uploaded or jenkins gets back into a good state. await EnqueueProcessBuild(entity.BuildStateKey, entity.BoundBuildId, TimeSpan.FromHours(1), cancellationToken); } }
internal async Task<bool> PopulateCore(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken) { var buildId = entity.BoundBuildId; var key = entity.BuildStateKey; await CheckFinished(entity, cancellationToken); // Don't process the build unless it's known to have finished. if (!entity.IsBuildFinished) { _logger.WriteLine($"Build {buildId.JobId} isn't finished yet"); return false; } // The build was completely populated by a previous message. No more work needed. if (entity.IsDataComplete) { _logger.WriteLine($"Build {buildId.JobId} is already populated"); return true; } try { _logger.WriteLine($"Populating {buildId.JobId} ... "); await populator.PopulateBuild(buildId); _logger.WriteLine($"Updating the build data state .."); entity.IsDataComplete = true; entity.Error = null; entity.ETag = "*"; await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity), cancellationToken); _logger.WriteLine($"Completed"); return true; } catch (Exception e) { _logger.WriteLine($"Failed"); _logger.WriteLine(e); await CheckForMissingBuild(entity, cancellationToken); try { entity.Error = $"{e.Message} - {e.StackTrace.Take(1000)}"; await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity)); } catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 412) { // It's possible the enity was updated in parallel. That's okay. This table // is meant as an approximation of the build state and always moving towards complete. } return false; } }