private static async Task DrainPoisonQueue()
{
var account = GetStorageAccount();
var client = account.CreateCloudQueueClient();
var queue = client.GetQueueReference($"{AzureConstants.QueueNames.BuildEvent}-poison");
var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CreateClient(), Console.Out);
var set = new HashSet <BuildId>();
do
{
var message = await queue.GetMessageAsync();
var obj = JObject.Parse(message.AsString);
var jobPath = obj.Value <string>("jobName");
var number = obj.Value <int>("number");
var buildId = new BuildId(number, JenkinsUtil.ConvertPathToJobId(jobPath));
if (!set.Add(buildId))
{
continue;
}
await populator.PopulateBuild(buildId);
await queue.DeleteMessageAsync(message);
} while (true);
}
private static async Task CheckUnknown()
{
var account = GetStorageAccount();
var buildUtil = new BuildUtil(account);
var date = DateTimeOffset.UtcNow - TimeSpan.FromDays(1);
var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CreateClient(), Console.Out);
var table = account.CreateCloudTableClient().GetTableReference(AzureConstants.TableNames.BuildResultDate);
foreach (var entity in buildUtil.GetBuildResultsByKindName(date, BuildResultClassification.Unknown.Name, AzureUtil.ViewNameAll))
{
var entityDate = DateKey.Parse(entity.PartitionKey);
var before = new DateKey(entityDate.Date.AddDays(-1));
var after = new DateKey(entityDate.Date.AddDays(1));
var op = TableOperation.Retrieve(before.Key, entity.RowKey);
var result = await table.ExecuteAsync(op);
if (result.Result != null)
{
await table.ExecuteAsync(TableOperation.Delete(entity));
continue;
}
op = TableOperation.Retrieve(after.Key, entity.RowKey);
result = await table.ExecuteAsync(op);
if (result.Result != null)
{
await table.ExecuteAsync(TableOperation.Delete(entity));
continue;
}
}
}
/// <summary>
/// Populate the build table by processing the given message. This function doesn't handle
/// any build state semantics. Instead it just processes the build and updates the build
/// result tables.
/// </summary>
public static async Task PopulateBuildData(
[QueueTrigger(QueueNames.ProcessBuild)] string message,
[Table(TableNames.BuildState)] CloudTable buildStateTable,
[Table(TableNames.BuildStateKey)] CloudTable buildStateKeyTable,
[Table(TableNames.BuildResultDate)] CloudTable buildResultDateTable,
[Table(TableNames.BuildResultExact)] CloudTable buildResultExactTable,
[Table(TableNames.BuildFailureDate)] CloudTable buildFailureDateTable,
[Table(TableNames.BuildFailureExact)] CloudTable buildFailureExactTable,
[Table(TableNames.CounterBuilds)] CloudTable counterBuildsTable,
[Table(TableNames.ViewNameDate)] CloudTable viewNameDateTable,
[Queue(QueueNames.ProcessBuild)] CloudQueue processBuildQueue,
[Queue(QueueNames.EmailBuild)] CloudQueue emailBuildQueue,
TextWriter logger,
CancellationToken cancellationToken)
{
var buildIdJson = (BuildStateMessage)JsonConvert.DeserializeObject(message, typeof(BuildStateMessage));
var client = StateUtil.CreateJenkinsClient(buildIdJson.BoundBuildId);
var populator = new BuildTablePopulator(
buildResultDateTable: buildResultDateTable,
buildResultExactTable: buildResultExactTable,
buildFailureDateTable: buildFailureDateTable,
buildFailureExactTable: buildFailureExactTable,
viewNameDateTable: viewNameDateTable,
buildCounterUtil: CounterUtilFactory.Create <BuildCounterEntity>(counterBuildsTable),
client: client,
textWriter: logger);
var stateUtil = new StateUtil(
buildStateKeyTable: buildStateKeyTable,
buildStateTable: buildStateTable,
processBuildQueue: processBuildQueue,
emailBuildQueue: emailBuildQueue,
logger: logger);
await stateUtil.Populate(buildIdJson, populator, cancellationToken);
}
/// <summary>
/// Populate the given build and update the unprocessed table accordingly. If there is no
/// existing entity in the unprocessed table, this won't add one. It will only update existing
/// ones.
/// </summary>
internal async Task Populate(BuildStateMessage message, BuildTablePopulator populator, CancellationToken cancellationToken)
{
var buildId = message.BuildId;
var entityKey = BuildStateEntity.GetEntityKey(message.BuildStateKey, message.BoundBuildId);
var entity = await AzureUtil.QueryAsync <BuildStateEntity>(_buildStateTable, entityKey, cancellationToken);
var completed = await PopulateCore(entity, populator, cancellationToken);
// Unable to complete the build, consider this is a 404 missing that we need to handle.
if (!completed && entity.BuildMissingCount > MissingBuildLimit)
{
completed = await PopulateMissing(entity, populator, cancellationToken);
}
if (completed)
{
return;
}
var isDone = (DateTimeOffset.UtcNow - entity.BuildStateKey.DateTime).TotalDays > DayWindow;
if (isDone)
{
await EnqueueEmailBuild(entity.BuildStateKey, entity.BoundBuildId, cancellationToken);
}
else
{
// Wait an hour to retry. Hope that a bug fix is uploaded or jenkins gets back into a good state.
await EnqueueProcessBuild(entity.BuildStateKey, entity.BoundBuildId, TimeSpan.FromHours(1), cancellationToken);
}
}
private static async Task TestFailureYesterday(int days = -1)
{
var account = GetStorageAccount();
var tableClient = account.CreateCloudTableClient();
var table = tableClient.GetTableReference(TableNames.BuildState);
var date = DateTimeOffset.UtcNow.AddDays(days);
var query = TableQueryUtil.And(
TableQueryUtil.PartitionKey(DateTimeKey.GetDateKey(date)),
TableQueryUtil.Column(nameof(BuildStateEntity.IsBuildFinished), true),
TableQueryUtil.Column(nameof(BuildStateEntity.IsDataComplete), false));
var list = await AzureUtil.QueryAsync <BuildStateEntity>(table, query);
foreach (var entity in list)
{
var populator = new BuildTablePopulator(tableClient, CounterUtilFactory, CreateClient(entity.BoundBuildId), TextWriter.Null);
try
{
Console.Write($"{entity.BuildId} ... ");
await populator.PopulateBuild(entity.BoundBuildId);
Console.WriteLine("good");
}
catch (Exception ex)
{
Console.WriteLine("ERRROR");
Console.WriteLine(ex);
}
}
}
/// <summary>
/// Function to help diagnose failures in processing.
/// </summary>
/// <returns></returns>
private static async Task TestFailure()
{
/*
* var name = "Private/Microsoft_vstest/master/Microsoft_vstest_Release_prtest";
* var number = 119;
* var host = SharedConstants.DotnetJenkinsHostName;
* var jobId = JobId.ParseName(name);
* var boundBuildId = new BoundBuildId(host, new BuildId(number, jobId));
*/
var url = "http://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/master/job/debug_windows_nt_bld/198";
var boundBuildId = BoundBuildId.Parse(url);
var buildId = boundBuildId.BuildId;
var account = GetStorageAccount();
var client = CreateClient(boundBuildId);
var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CounterUtilFactory, client, Console.Out);
try
{
await populator.PopulateBuild(boundBuildId);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
/// <summary>
/// Populate the build table by processing the given message. This function doesn't handle
/// any build state semantics. Instead it just processes the build and updates the build
/// result tables.
/// </summary>
public static async Task PopulateBuildData(
[QueueTrigger(AzureConstants.QueueNames.ProcessBuild)] string message,
[Table(AzureConstants.TableNames.UnprocessedBuild)] CloudTable unprocessedBuildTable,
[Table(AzureConstants.TableNames.BuildResultDate)] CloudTable buildResultDateTable,
[Table(AzureConstants.TableNames.BuildResultExact)] CloudTable buildResultExactTable,
[Table(AzureConstants.TableNames.BuildFailureDate)] CloudTable buildFailureDateTable,
[Table(AzureConstants.TableNames.BuildFailureExact)] CloudTable buildFailureExactTable,
[Table(AzureConstants.TableNames.ViewNameDate)] CloudTable viewNameDateTable,
TextWriter logger,
CancellationToken cancellationToken)
{
var buildIdJson = (BuildIdJson)JsonConvert.DeserializeObject(message, typeof(BuildIdJson));
var client = StateUtil.CreateJenkinsClient(buildIdJson.JenkinsUrl, buildIdJson.JobId);
var populator = new BuildTablePopulator(
buildResultDateTable: buildResultDateTable,
buildResultExactTable: buildResultExactTable,
buildFailureDateTable: buildFailureDateTable,
buildFailureExactTable: buildFailureExactTable,
viewNameDateTable: viewNameDateTable,
client: client,
textWriter: logger);
var stateUtil = new StateUtil(
unprocessedBuildTable: unprocessedBuildTable,
buildResultExact: buildResultExactTable,
logger: logger);
await stateUtil.Populate(buildIdJson.BuildId, populator, force : false, cancellationToken : cancellationToken);
}
internal async Task <bool> PopulateCore(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken)
{
var buildId = entity.BoundBuildId;
var key = entity.BuildStateKey;
await CheckFinished(entity, cancellationToken);
// Don't process the build unless it's known to have finished.
if (!entity.IsBuildFinished)
{
_logger.WriteLine($"Build {buildId.JobId} isn't finished yet");
return(false);
}
// The build was completely populated by a previous message. No more work needed.
if (entity.IsDataComplete)
{
_logger.WriteLine($"Build {buildId.JobId} is already populated");
return(true);
}
try
{
_logger.WriteLine($"Populating {buildId.JobId} ... ");
await populator.PopulateBuild(buildId);
_logger.WriteLine($"Updating the build data state ..");
entity.IsDataComplete = true;
entity.Error = null;
entity.ETag = "*";
await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity), cancellationToken);
_logger.WriteLine($"Completed");
return(true);
}
catch (Exception e)
{
_logger.WriteLine($"Failed");
_logger.WriteLine(e);
await CheckForMissingBuild(entity, cancellationToken);
try
{
entity.Error = $"{e.Message} - {e.StackTrace.Take(1000)}";
await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity));
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == 412)
{
// It's possible the enity was updated in parallel. That's okay. This table
// is meant as an approximation of the build state and always moving towards complete.
}
return(false);
}
}
private static async Task TestJob()
{
var jobUrlStr = "http://dotnet-ci.cloudapp.net/job/Private/job/dotnet_roslyn-internal/job/microupdate/job/windows_vsi_p2/8/";
var uri = new Uri(jobUrlStr);
var parts = uri.PathAndQuery.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
var jobPath = string.Join("/", parts.Take(parts.Length - 1));
var number = int.Parse(parts.Last());
var jobId = JenkinsUtil.ConvertPathToJobId(jobPath);
var buildId = new BuildId(number, jobId);
var account = GetStorageAccount();
var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CreateClient(), Console.Out);
await populator.PopulateBuild(buildId);
}
public BuildTablePopulatorTests()
{
var account = Util.GetStorageAccount();
var tableClient = account.CreateCloudTableClient();
_restClient = new MockRestClient();
var client = new JenkinsClient(SharedConstants.DotnetJenkinsUri, _restClient.Client);
_buildFailureExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildFailureExact);
_buildResultExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildResultExact);
_populator = new BuildTablePopulator(
tableClient,
client: client,
textWriter: new StringWriter());
}
public BuildTablePopulatorTests()
{
var account = Util.GetStorageAccount();
var tableClient = account.CreateCloudTableClient();
_restClient = new MockRestClient();
var client = new JenkinsClient(new Uri("http://test.com"), _restClient.Client);
_buildFailureExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildFailureExact);
_buildResultExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildResultExact);
_populator = new BuildTablePopulator(
tableClient,
client: client,
factory: new CounterUtilFactory(),
textWriter: new StringWriter());
}
private static async Task TestPopulator()
{
var account = GetStorageAccount();
var client = CreateClient(auth: false);
var populator = new BuildTablePopulator(account.CreateCloudTableClient(), client, Console.Out);
var boundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/master/job/jitstress/job/x64_checked_osx_jitstress1_flow/7/");
try
{
await populator.PopulateBuild(boundBuildId.BuildId);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
/// <summary>
/// The build is determined to be missing. Finish the build according to that.
/// </summary>
internal async Task <bool> PopulateMissing(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken)
{
try
{
await populator.PopulateBuildMissing(entity.BoundBuildId);
entity.IsBuildFinished = true;
entity.IsDataComplete = true;
entity.Error = "Build missing";
await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken);
return(true);
}
catch (Exception ex)
{
// This is frankly the best possible outcome. This is the worst state we can have for a build
// so any other thread giving a result can't be worse.
_logger.WriteLine($"Error populating build {entity.BuildId} as missing {ex}");
return(false);
}
}
/// <summary>
/// Populate the given build and update the unprocessed table accordingly. If there is no
/// existing entity in the unprocessed table, this won't add one. It will only update existing
/// ones.
/// </summary>
internal async Task Populate(BuildId buildId, BuildTablePopulator populator, bool force, CancellationToken cancellationToken)
{
var key = UnprocessedBuildEntity.GetEntityKey(buildId);
try
{
// If we are not forcing the update then check for the existence of a completed run before
// requerying Jenkins.
if (force || !(await populator.IsPopulated(buildId)))
{
await populator.PopulateBuild(buildId);
}
await AzureUtil.MaybeDeleteAsync(_unprocessedBuildTable, key, cancellationToken);
}
catch (Exception e)
{
// Update the error state for the row.
var entity = await AzureUtil.QueryAsync <UnprocessedBuildEntity>(_unprocessedBuildTable, key, cancellationToken);
if (entity != null)
{
entity.StatusText = $"{e.Message} - {e.StackTrace.Take(1000)}";
var operation = TableOperation.Replace(entity);
try
{
await _unprocessedBuildTable.ExecuteAsync(operation);
}
catch
{
// It's possible the enity was deleted / updated in parallel. That's okay. This table
// is meant as an approximation of the build state and always moving towards complete.
}
}
}
}
