public void PreferHostRaw()
{
var jobId = JobId.ParseName("test");
var host = new Uri("https://example.com");
var entity = new BuildStateEntity()
{
JobName = jobId.Name,
BuildNumber = 42,
HostRaw = host.ToString(),
HostName = "ignore"
};
Assert.Equal(host, entity.BoundBuildId.Host);
}
public void FallbackToHostName()
{
var jobId = JobId.ParseName("test");
var host = new Uri("http://example.com");
var entity = new BuildStateEntity()
{
JobName = jobId.Name,
BuildNumber = 42,
HostRaw = null,
HostName = "example.com"
};
Assert.Equal(host, entity.BoundBuildId.Host);
}
internal async Task ProcessBuildEvent(BuildEventMessageJson message, CancellationToken cancellationToken)
{
var isBuildFinished = message.Phase == "FINALIZED";
var key = await GetOrCreateBuildStateKey(message.BoundBuildId);
var entityKey = BuildStateEntity.GetEntityKey(key, message.BoundBuildId);
// Ensure there is an entry in the build state table for this build.
var entity = await AzureUtil.QueryAsync<BuildStateEntity>(_buildStateTable, entityKey, cancellationToken);
if (entity == null || entity.IsBuildFinished != isBuildFinished)
{
entity = new BuildStateEntity(key, message.BoundBuildId, isBuildFinished);
await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken);
}
// Enqueue a message to process the build. Insert a delay if the build isn't finished yet so that
// we don't unnecessarily ask Jenkins for information.
var delay = isBuildFinished ? (TimeSpan?)null : TimeSpan.FromMinutes(30);
await EnqueueProcessBuild(key, message.BoundBuildId, delay, cancellationToken);
}
private static void AppendEmailText(BuildStateEntity entity, StringBuilder textBuilder, StringBuilder htmlBuilder)
{
var boundBuildId = entity.BoundBuildId;
var buildId = boundBuildId.BuildId;
textBuilder.Append($"Failed to process build: {boundBuildId.GetBuildUri(useHttps: false)}");
textBuilder.Append($"Error: {entity.Error}");
htmlBuilder.Append($@"<div>");
htmlBuilder.Append($@"<div>Build <a href=""{boundBuildId.GetBuildUri(useHttps: false)}"">{buildId.JobName} {buildId.Number}</a></div>");
htmlBuilder.Append($@"<div>Error: {WebUtility.HtmlEncode(entity.Error)}</div>");
htmlBuilder.Append($@"</div>");
}
private async Task<bool> IsBuildTemporarilyMissing(BuildStateEntity entity)
{
var buildId = entity.BoundBuildId;
try
{
var client = new RestClient(buildId.Host);
var request = new RestRequest(buildId.BuildUri.PathAndQuery, Method.GET);
var response = await client.ExecuteTaskAsync(request);
return response.StatusCode == HttpStatusCode.NotFound;
}
catch (Exception ex)
{
_logger.WriteLine($"Error checking for 404 on {buildId} {ex}");
return false;
}
}
/// <summary>
/// This is called when we get an exception processing a build. This accounts for the case that a
/// build is missing. Can happen during Jenkins restart, build archiving, etc ...
///
/// This is fundamentally a heuristic. It's interpreting 404 essentially as permanently missing vs.
/// Jenkins is just down for a period of time. This is understood and accounted for as best as possible.
/// </summary>
private async Task CheckForMissingBuild(BuildStateEntity entity, CancellationToken cancellationToken)
{
var isMissing = await IsBuildTemporarilyMissing(entity);
if (!isMissing)
{
return;
}
entity.BuildMissingCount++;
try
{
await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken);
}
catch
{
// Possible to be updated in parallel. Always moving to a final state so that's fine.
}
}
private async Task CheckFinished(BuildStateEntity entity, CancellationToken cancellationToken)
{
if (entity.IsBuildFinished)
{
return;
}
try
{
_logger.WriteLine($"Checking to see if {entity.BuildId} has completed");
var client = CreateJenkinsClient(entity.BoundBuildId);
var buildInfo = await client.GetBuildInfoAsync(entity.BuildId);
if (buildInfo.State != BuildState.Running)
{
entity.IsBuildFinished = true;
await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity), cancellationToken);
}
}
catch (Exception ex)
{
await CheckForMissingBuild(entity, cancellationToken);
_logger.WriteLine($"Unable to query job state {ex.Message}");
}
}
/// <summary>
/// The build is determined to be missing. Finish the build according to that.
/// </summary>
internal async Task<bool> PopulateMissing(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken)
{
try
{
await populator.PopulateBuildMissing(entity.BoundBuildId);
entity.IsBuildFinished = true;
entity.IsDataComplete = true;
entity.Error = "Build missing";
await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken);
return true;
}
catch (Exception ex)
{
// This is frankly the best possible outcome. This is the worst state we can have for a build
// so any other thread giving a result can't be worse.
_logger.WriteLine($"Error populating build {entity.BuildId} as missing {ex}");
return false;
}
}
internal async Task<bool> PopulateCore(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken)
{
var buildId = entity.BoundBuildId;
var key = entity.BuildStateKey;
await CheckFinished(entity, cancellationToken);
// Don't process the build unless it's known to have finished.
if (!entity.IsBuildFinished)
{
_logger.WriteLine($"Build {buildId.JobId} isn't finished yet");
return false;
}
// The build was completely populated by a previous message. No more work needed.
if (entity.IsDataComplete)
{
_logger.WriteLine($"Build {buildId.JobId} is already populated");
return true;
}
try
{
_logger.WriteLine($"Populating {buildId.JobId} ... ");
await populator.PopulateBuild(buildId);
_logger.WriteLine($"Updating the build data state ..");
entity.IsDataComplete = true;
entity.Error = null;
entity.ETag = "*";
await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity), cancellationToken);
_logger.WriteLine($"Completed");
return true;
}
catch (Exception e)
{
_logger.WriteLine($"Failed");
_logger.WriteLine(e);
await CheckForMissingBuild(entity, cancellationToken);
try
{
entity.Error = $"{e.Message} - {e.StackTrace.Take(1000)}";
await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity));
}
catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 412)
{
// It's possible the enity was updated in parallel. That's okay. This table
// is meant as an approximation of the build state and always moving towards complete.
}
return false;
}
}
