public void PreferHostRaw() { var jobId = JobId.ParseName("test"); var host = new Uri("https://example.com"); var entity = new BuildStateEntity() { JobName = jobId.Name, BuildNumber = 42, HostRaw = host.ToString(), HostName = "ignore" }; Assert.Equal(host, entity.BoundBuildId.Host); }
public void FallbackToHostName() { var jobId = JobId.ParseName("test"); var host = new Uri("http://example.com"); var entity = new BuildStateEntity() { JobName = jobId.Name, BuildNumber = 42, HostRaw = null, HostName = "example.com" }; Assert.Equal(host, entity.BoundBuildId.Host); }
internal async Task ProcessBuildEvent(BuildEventMessageJson message, CancellationToken cancellationToken) { var isBuildFinished = message.Phase == "FINALIZED"; var key = await GetOrCreateBuildStateKey(message.BoundBuildId); var entityKey = BuildStateEntity.GetEntityKey(key, message.BoundBuildId); // Ensure there is an entry in the build state table for this build. var entity = await AzureUtil.QueryAsync<BuildStateEntity>(_buildStateTable, entityKey, cancellationToken); if (entity == null || entity.IsBuildFinished != isBuildFinished) { entity = new BuildStateEntity(key, message.BoundBuildId, isBuildFinished); await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken); } // Enqueue a message to process the build. Insert a delay if the build isn't finished yet so that // we don't unnecessarily ask Jenkins for information. var delay = isBuildFinished ? (TimeSpan?)null : TimeSpan.FromMinutes(30); await EnqueueProcessBuild(key, message.BoundBuildId, delay, cancellationToken); }
private static void AppendEmailText(BuildStateEntity entity, StringBuilder textBuilder, StringBuilder htmlBuilder) { var boundBuildId = entity.BoundBuildId; var buildId = boundBuildId.BuildId; textBuilder.Append($"Failed to process build: {boundBuildId.GetBuildUri(useHttps: false)}"); textBuilder.Append($"Error: {entity.Error}"); htmlBuilder.Append($@"<div>"); htmlBuilder.Append($@"<div>Build <a href=""{boundBuildId.GetBuildUri(useHttps: false)}"">{buildId.JobName} {buildId.Number}</a></div>"); htmlBuilder.Append($@"<div>Error: {WebUtility.HtmlEncode(entity.Error)}</div>"); htmlBuilder.Append($@"</div>"); }
private async Task<bool> IsBuildTemporarilyMissing(BuildStateEntity entity) { var buildId = entity.BoundBuildId; try { var client = new RestClient(buildId.Host); var request = new RestRequest(buildId.BuildUri.PathAndQuery, Method.GET); var response = await client.ExecuteTaskAsync(request); return response.StatusCode == HttpStatusCode.NotFound; } catch (Exception ex) { _logger.WriteLine($"Error checking for 404 on {buildId} {ex}"); return false; } }
/// <summary> /// This is called when we get an exception processing a build. This accounts for the case that a /// build is missing. Can happen during Jenkins restart, build archiving, etc ... /// /// This is fundamentally a heuristic. It's interpreting 404 essentially as permanently missing vs. /// Jenkins is just down for a period of time. This is understood and accounted for as best as possible. /// </summary> private async Task CheckForMissingBuild(BuildStateEntity entity, CancellationToken cancellationToken) { var isMissing = await IsBuildTemporarilyMissing(entity); if (!isMissing) { return; } entity.BuildMissingCount++; try { await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken); } catch { // Possible to be updated in parallel. Always moving to a final state so that's fine. } }
private async Task CheckFinished(BuildStateEntity entity, CancellationToken cancellationToken) { if (entity.IsBuildFinished) { return; } try { _logger.WriteLine($"Checking to see if {entity.BuildId} has completed"); var client = CreateJenkinsClient(entity.BoundBuildId); var buildInfo = await client.GetBuildInfoAsync(entity.BuildId); if (buildInfo.State != BuildState.Running) { entity.IsBuildFinished = true; await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity), cancellationToken); } } catch (Exception ex) { await CheckForMissingBuild(entity, cancellationToken); _logger.WriteLine($"Unable to query job state {ex.Message}"); } }
/// <summary> /// The build is determined to be missing. Finish the build according to that. /// </summary> internal async Task<bool> PopulateMissing(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken) { try { await populator.PopulateBuildMissing(entity.BoundBuildId); entity.IsBuildFinished = true; entity.IsDataComplete = true; entity.Error = "Build missing"; await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken); return true; } catch (Exception ex) { // This is frankly the best possible outcome. This is the worst state we can have for a build // so any other thread giving a result can't be worse. _logger.WriteLine($"Error populating build {entity.BuildId} as missing {ex}"); return false; } }
internal async Task<bool> PopulateCore(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken) { var buildId = entity.BoundBuildId; var key = entity.BuildStateKey; await CheckFinished(entity, cancellationToken); // Don't process the build unless it's known to have finished. if (!entity.IsBuildFinished) { _logger.WriteLine($"Build {buildId.JobId} isn't finished yet"); return false; } // The build was completely populated by a previous message. No more work needed. if (entity.IsDataComplete) { _logger.WriteLine($"Build {buildId.JobId} is already populated"); return true; } try { _logger.WriteLine($"Populating {buildId.JobId} ... "); await populator.PopulateBuild(buildId); _logger.WriteLine($"Updating the build data state .."); entity.IsDataComplete = true; entity.Error = null; entity.ETag = "*"; await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity), cancellationToken); _logger.WriteLine($"Completed"); return true; } catch (Exception e) { _logger.WriteLine($"Failed"); _logger.WriteLine(e); await CheckForMissingBuild(entity, cancellationToken); try { entity.Error = $"{e.Message} - {e.StackTrace.Take(1000)}"; await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity)); } catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 412) { // It's possible the enity was updated in parallel. That's okay. This table // is meant as an approximation of the build state and always moving towards complete. } return false; } }