/// <summary>
/// Populate the build table by processing the given message. This function doesn't handle
/// any build state semantics. Instead it just processes the build and updates the build
/// result tables.
/// </summary>
public static async Task PopulateBuildData(
[QueueTrigger(QueueNames.ProcessBuild)] string message,
[Table(TableNames.BuildState)] CloudTable buildStateTable,
[Table(TableNames.BuildStateKey)] CloudTable buildStateKeyTable,
[Table(TableNames.BuildResultDate)] CloudTable buildResultDateTable,
[Table(TableNames.BuildResultExact)] CloudTable buildResultExactTable,
[Table(TableNames.BuildFailureDate)] CloudTable buildFailureDateTable,
[Table(TableNames.BuildFailureExact)] CloudTable buildFailureExactTable,
[Table(TableNames.CounterBuilds)] CloudTable counterBuildsTable,
[Table(TableNames.ViewNameDate)] CloudTable viewNameDateTable,
[Queue(QueueNames.ProcessBuild)] CloudQueue processBuildQueue,
[Queue(QueueNames.EmailBuild)] CloudQueue emailBuildQueue,
TextWriter logger,
CancellationToken cancellationToken)
{
var buildIdJson = (BuildStateMessage)JsonConvert.DeserializeObject(message, typeof(BuildStateMessage));
var client = StateUtil.CreateJenkinsClient(buildIdJson.BoundBuildId);
var populator = new BuildTablePopulator(
buildResultDateTable: buildResultDateTable,
buildResultExactTable: buildResultExactTable,
buildFailureDateTable: buildFailureDateTable,
buildFailureExactTable: buildFailureExactTable,
viewNameDateTable: viewNameDateTable,
buildCounterUtil: CounterUtilFactory.Create<BuildCounterEntity>(counterBuildsTable),
client: client,
textWriter: logger);
var stateUtil = new StateUtil(
buildStateKeyTable: buildStateKeyTable,
buildStateTable: buildStateTable,
processBuildQueue: processBuildQueue,
emailBuildQueue: emailBuildQueue,
logger: logger);
await stateUtil.Populate(buildIdJson, populator, cancellationToken);
}
public BuildTablePopulatorTests()
{
var account = Util.GetStorageAccount();
var tableClient = account.CreateCloudTableClient();
_restClient = new MockRestClient();
var client = new JenkinsClient(new Uri("http://test.com"), _restClient.Client);
_buildFailureExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildFailureExact);
_buildResultExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildResultExact);
_populator = new BuildTablePopulator(
tableClient,
client: client,
factory: new CounterUtilFactory(),
textWriter: new StringWriter());
}
/// <summary>
/// The build is determined to be missing. Finish the build according to that.
/// </summary>
internal async Task<bool> PopulateMissing(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken)
{
try
{
await populator.PopulateBuildMissing(entity.BoundBuildId);
entity.IsBuildFinished = true;
entity.IsDataComplete = true;
entity.Error = "Build missing";
await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken);
return true;
}
catch (Exception ex)
{
// This is frankly the best possible outcome. This is the worst state we can have for a build
// so any other thread giving a result can't be worse.
_logger.WriteLine($"Error populating build {entity.BuildId} as missing {ex}");
return false;
}
}
/// <summary>
/// Populate the given build and update the unprocessed table accordingly. If there is no
/// existing entity in the unprocessed table, this won't add one. It will only update existing
/// ones.
/// </summary>
internal async Task Populate(BuildStateMessage message, BuildTablePopulator populator, CancellationToken cancellationToken)
{
var buildId = message.BuildId;
var entityKey = BuildStateEntity.GetEntityKey(message.BuildStateKey, message.BoundBuildId);
var entity = await AzureUtil.QueryAsync<BuildStateEntity>(_buildStateTable, entityKey, cancellationToken);
var completed = await PopulateCore(entity, populator, cancellationToken);
// Unable to complete the build, consider this is a 404 missing that we need to handle.
if (!completed && entity.BuildMissingCount > MissingBuildLimit)
{
completed = await PopulateMissing(entity, populator, cancellationToken);
}
if (completed)
{
return;
}
var isDone = (DateTimeOffset.UtcNow - entity.BuildStateKey.DateTime).TotalDays > DayWindow;
if (isDone)
{
await EnqueueEmailBuild(entity.BuildStateKey, entity.BoundBuildId, cancellationToken);
}
else
{
// Wait an hour to retry. Hope that a bug fix is uploaded or jenkins gets back into a good state.
await EnqueueProcessBuild(entity.BuildStateKey, entity.BoundBuildId, TimeSpan.FromHours(1), cancellationToken);
}
}
internal async Task<bool> PopulateCore(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken)
{
var buildId = entity.BoundBuildId;
var key = entity.BuildStateKey;
await CheckFinished(entity, cancellationToken);
// Don't process the build unless it's known to have finished.
if (!entity.IsBuildFinished)
{
_logger.WriteLine($"Build {buildId.JobId} isn't finished yet");
return false;
}
// The build was completely populated by a previous message. No more work needed.
if (entity.IsDataComplete)
{
_logger.WriteLine($"Build {buildId.JobId} is already populated");
return true;
}
try
{
_logger.WriteLine($"Populating {buildId.JobId} ... ");
await populator.PopulateBuild(buildId);
_logger.WriteLine($"Updating the build data state ..");
entity.IsDataComplete = true;
entity.Error = null;
entity.ETag = "*";
await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity), cancellationToken);
_logger.WriteLine($"Completed");
return true;
}
catch (Exception e)
{
_logger.WriteLine($"Failed");
_logger.WriteLine(e);
await CheckForMissingBuild(entity, cancellationToken);
try
{
entity.Error = $"{e.Message} - {e.StackTrace.Take(1000)}";
await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity));
}
catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 412)
{
// It's possible the enity was updated in parallel. That's okay. This table
// is meant as an approximation of the build state and always moving towards complete.
}
return false;
}
}