private static async Task DrainPoisonQueue() { var account = GetStorageAccount(); var client = account.CreateCloudQueueClient(); var queue = client.GetQueueReference($"{AzureConstants.QueueNames.BuildEvent}-poison"); var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CreateClient(), Console.Out); var set = new HashSet <BuildId>(); do { var message = await queue.GetMessageAsync(); var obj = JObject.Parse(message.AsString); var jobPath = obj.Value <string>("jobName"); var number = obj.Value <int>("number"); var buildId = new BuildId(number, JenkinsUtil.ConvertPathToJobId(jobPath)); if (!set.Add(buildId)) { continue; } await populator.PopulateBuild(buildId); await queue.DeleteMessageAsync(message); } while (true); }
private static async Task CheckUnknown() { var account = GetStorageAccount(); var buildUtil = new BuildUtil(account); var date = DateTimeOffset.UtcNow - TimeSpan.FromDays(1); var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CreateClient(), Console.Out); var table = account.CreateCloudTableClient().GetTableReference(AzureConstants.TableNames.BuildResultDate); foreach (var entity in buildUtil.GetBuildResultsByKindName(date, BuildResultClassification.Unknown.Name, AzureUtil.ViewNameAll)) { var entityDate = DateKey.Parse(entity.PartitionKey); var before = new DateKey(entityDate.Date.AddDays(-1)); var after = new DateKey(entityDate.Date.AddDays(1)); var op = TableOperation.Retrieve(before.Key, entity.RowKey); var result = await table.ExecuteAsync(op); if (result.Result != null) { await table.ExecuteAsync(TableOperation.Delete(entity)); continue; } op = TableOperation.Retrieve(after.Key, entity.RowKey); result = await table.ExecuteAsync(op); if (result.Result != null) { await table.ExecuteAsync(TableOperation.Delete(entity)); continue; } } }
/// <summary> /// Populate the build table by processing the given message. This function doesn't handle /// any build state semantics. Instead it just processes the build and updates the build /// result tables. /// </summary> public static async Task PopulateBuildData( [QueueTrigger(QueueNames.ProcessBuild)] string message, [Table(TableNames.BuildState)] CloudTable buildStateTable, [Table(TableNames.BuildStateKey)] CloudTable buildStateKeyTable, [Table(TableNames.BuildResultDate)] CloudTable buildResultDateTable, [Table(TableNames.BuildResultExact)] CloudTable buildResultExactTable, [Table(TableNames.BuildFailureDate)] CloudTable buildFailureDateTable, [Table(TableNames.BuildFailureExact)] CloudTable buildFailureExactTable, [Table(TableNames.CounterBuilds)] CloudTable counterBuildsTable, [Table(TableNames.ViewNameDate)] CloudTable viewNameDateTable, [Queue(QueueNames.ProcessBuild)] CloudQueue processBuildQueue, [Queue(QueueNames.EmailBuild)] CloudQueue emailBuildQueue, TextWriter logger, CancellationToken cancellationToken) { var buildIdJson = (BuildStateMessage)JsonConvert.DeserializeObject(message, typeof(BuildStateMessage)); var client = StateUtil.CreateJenkinsClient(buildIdJson.BoundBuildId); var populator = new BuildTablePopulator( buildResultDateTable: buildResultDateTable, buildResultExactTable: buildResultExactTable, buildFailureDateTable: buildFailureDateTable, buildFailureExactTable: buildFailureExactTable, viewNameDateTable: viewNameDateTable, buildCounterUtil: CounterUtilFactory.Create <BuildCounterEntity>(counterBuildsTable), client: client, textWriter: logger); var stateUtil = new StateUtil( buildStateKeyTable: buildStateKeyTable, buildStateTable: buildStateTable, processBuildQueue: processBuildQueue, emailBuildQueue: emailBuildQueue, logger: logger); await stateUtil.Populate(buildIdJson, populator, cancellationToken); }
/// <summary> /// Populate the given build and update the unprocessed table accordingly. If there is no /// existing entity in the unprocessed table, this won't add one. It will only update existing /// ones. /// </summary> internal async Task Populate(BuildStateMessage message, BuildTablePopulator populator, CancellationToken cancellationToken) { var buildId = message.BuildId; var entityKey = BuildStateEntity.GetEntityKey(message.BuildStateKey, message.BoundBuildId); var entity = await AzureUtil.QueryAsync <BuildStateEntity>(_buildStateTable, entityKey, cancellationToken); var completed = await PopulateCore(entity, populator, cancellationToken); // Unable to complete the build, consider this is a 404 missing that we need to handle. if (!completed && entity.BuildMissingCount > MissingBuildLimit) { completed = await PopulateMissing(entity, populator, cancellationToken); } if (completed) { return; } var isDone = (DateTimeOffset.UtcNow - entity.BuildStateKey.DateTime).TotalDays > DayWindow; if (isDone) { await EnqueueEmailBuild(entity.BuildStateKey, entity.BoundBuildId, cancellationToken); } else { // Wait an hour to retry. Hope that a bug fix is uploaded or jenkins gets back into a good state. await EnqueueProcessBuild(entity.BuildStateKey, entity.BoundBuildId, TimeSpan.FromHours(1), cancellationToken); } }
private static async Task TestFailureYesterday(int days = -1) { var account = GetStorageAccount(); var tableClient = account.CreateCloudTableClient(); var table = tableClient.GetTableReference(TableNames.BuildState); var date = DateTimeOffset.UtcNow.AddDays(days); var query = TableQueryUtil.And( TableQueryUtil.PartitionKey(DateTimeKey.GetDateKey(date)), TableQueryUtil.Column(nameof(BuildStateEntity.IsBuildFinished), true), TableQueryUtil.Column(nameof(BuildStateEntity.IsDataComplete), false)); var list = await AzureUtil.QueryAsync <BuildStateEntity>(table, query); foreach (var entity in list) { var populator = new BuildTablePopulator(tableClient, CounterUtilFactory, CreateClient(entity.BoundBuildId), TextWriter.Null); try { Console.Write($"{entity.BuildId} ... "); await populator.PopulateBuild(entity.BoundBuildId); Console.WriteLine("good"); } catch (Exception ex) { Console.WriteLine("ERRROR"); Console.WriteLine(ex); } } }
/// <summary> /// Function to help diagnose failures in processing. /// </summary> /// <returns></returns> private static async Task TestFailure() { /* * var name = "Private/Microsoft_vstest/master/Microsoft_vstest_Release_prtest"; * var number = 119; * var host = SharedConstants.DotnetJenkinsHostName; * var jobId = JobId.ParseName(name); * var boundBuildId = new BoundBuildId(host, new BuildId(number, jobId)); */ var url = "http://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/master/job/debug_windows_nt_bld/198"; var boundBuildId = BoundBuildId.Parse(url); var buildId = boundBuildId.BuildId; var account = GetStorageAccount(); var client = CreateClient(boundBuildId); var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CounterUtilFactory, client, Console.Out); try { await populator.PopulateBuild(boundBuildId); } catch (Exception ex) { Console.WriteLine(ex); } }
/// <summary> /// Populate the build table by processing the given message. This function doesn't handle /// any build state semantics. Instead it just processes the build and updates the build /// result tables. /// </summary> public static async Task PopulateBuildData( [QueueTrigger(AzureConstants.QueueNames.ProcessBuild)] string message, [Table(AzureConstants.TableNames.UnprocessedBuild)] CloudTable unprocessedBuildTable, [Table(AzureConstants.TableNames.BuildResultDate)] CloudTable buildResultDateTable, [Table(AzureConstants.TableNames.BuildResultExact)] CloudTable buildResultExactTable, [Table(AzureConstants.TableNames.BuildFailureDate)] CloudTable buildFailureDateTable, [Table(AzureConstants.TableNames.BuildFailureExact)] CloudTable buildFailureExactTable, [Table(AzureConstants.TableNames.ViewNameDate)] CloudTable viewNameDateTable, TextWriter logger, CancellationToken cancellationToken) { var buildIdJson = (BuildIdJson)JsonConvert.DeserializeObject(message, typeof(BuildIdJson)); var client = StateUtil.CreateJenkinsClient(buildIdJson.JenkinsUrl, buildIdJson.JobId); var populator = new BuildTablePopulator( buildResultDateTable: buildResultDateTable, buildResultExactTable: buildResultExactTable, buildFailureDateTable: buildFailureDateTable, buildFailureExactTable: buildFailureExactTable, viewNameDateTable: viewNameDateTable, client: client, textWriter: logger); var stateUtil = new StateUtil( unprocessedBuildTable: unprocessedBuildTable, buildResultExact: buildResultExactTable, logger: logger); await stateUtil.Populate(buildIdJson.BuildId, populator, force : false, cancellationToken : cancellationToken); }
internal async Task <bool> PopulateCore(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken) { var buildId = entity.BoundBuildId; var key = entity.BuildStateKey; await CheckFinished(entity, cancellationToken); // Don't process the build unless it's known to have finished. if (!entity.IsBuildFinished) { _logger.WriteLine($"Build {buildId.JobId} isn't finished yet"); return(false); } // The build was completely populated by a previous message. No more work needed. if (entity.IsDataComplete) { _logger.WriteLine($"Build {buildId.JobId} is already populated"); return(true); } try { _logger.WriteLine($"Populating {buildId.JobId} ... "); await populator.PopulateBuild(buildId); _logger.WriteLine($"Updating the build data state .."); entity.IsDataComplete = true; entity.Error = null; entity.ETag = "*"; await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity), cancellationToken); _logger.WriteLine($"Completed"); return(true); } catch (Exception e) { _logger.WriteLine($"Failed"); _logger.WriteLine(e); await CheckForMissingBuild(entity, cancellationToken); try { entity.Error = $"{e.Message} - {e.StackTrace.Take(1000)}"; await _buildStateTable.ExecuteAsync(TableOperation.Replace(entity)); } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == 412) { // It's possible the enity was updated in parallel. That's okay. This table // is meant as an approximation of the build state and always moving towards complete. } return(false); } }
private static async Task TestJob() { var jobUrlStr = "http://dotnet-ci.cloudapp.net/job/Private/job/dotnet_roslyn-internal/job/microupdate/job/windows_vsi_p2/8/"; var uri = new Uri(jobUrlStr); var parts = uri.PathAndQuery.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); var jobPath = string.Join("/", parts.Take(parts.Length - 1)); var number = int.Parse(parts.Last()); var jobId = JenkinsUtil.ConvertPathToJobId(jobPath); var buildId = new BuildId(number, jobId); var account = GetStorageAccount(); var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CreateClient(), Console.Out); await populator.PopulateBuild(buildId); }
public BuildTablePopulatorTests() { var account = Util.GetStorageAccount(); var tableClient = account.CreateCloudTableClient(); _restClient = new MockRestClient(); var client = new JenkinsClient(SharedConstants.DotnetJenkinsUri, _restClient.Client); _buildFailureExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildFailureExact); _buildResultExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildResultExact); _populator = new BuildTablePopulator( tableClient, client: client, textWriter: new StringWriter()); }
public BuildTablePopulatorTests() { var account = Util.GetStorageAccount(); var tableClient = account.CreateCloudTableClient(); _restClient = new MockRestClient(); var client = new JenkinsClient(new Uri("http://test.com"), _restClient.Client); _buildFailureExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildFailureExact); _buildResultExactTable = tableClient.GetTableReference(AzureConstants.TableNames.BuildResultExact); _populator = new BuildTablePopulator( tableClient, client: client, factory: new CounterUtilFactory(), textWriter: new StringWriter()); }
private static async Task TestPopulator() { var account = GetStorageAccount(); var client = CreateClient(auth: false); var populator = new BuildTablePopulator(account.CreateCloudTableClient(), client, Console.Out); var boundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/master/job/jitstress/job/x64_checked_osx_jitstress1_flow/7/"); try { await populator.PopulateBuild(boundBuildId.BuildId); } catch (Exception ex) { Console.WriteLine(ex); } }
/// <summary> /// The build is determined to be missing. Finish the build according to that. /// </summary> internal async Task <bool> PopulateMissing(BuildStateEntity entity, BuildTablePopulator populator, CancellationToken cancellationToken) { try { await populator.PopulateBuildMissing(entity.BoundBuildId); entity.IsBuildFinished = true; entity.IsDataComplete = true; entity.Error = "Build missing"; await _buildStateTable.ExecuteAsync(TableOperation.InsertOrReplace(entity), cancellationToken); return(true); } catch (Exception ex) { // This is frankly the best possible outcome. This is the worst state we can have for a build // so any other thread giving a result can't be worse. _logger.WriteLine($"Error populating build {entity.BuildId} as missing {ex}"); return(false); } }
/// <summary> /// Populate the given build and update the unprocessed table accordingly. If there is no /// existing entity in the unprocessed table, this won't add one. It will only update existing /// ones. /// </summary> internal async Task Populate(BuildId buildId, BuildTablePopulator populator, bool force, CancellationToken cancellationToken) { var key = UnprocessedBuildEntity.GetEntityKey(buildId); try { // If we are not forcing the update then check for the existence of a completed run before // requerying Jenkins. if (force || !(await populator.IsPopulated(buildId))) { await populator.PopulateBuild(buildId); } await AzureUtil.MaybeDeleteAsync(_unprocessedBuildTable, key, cancellationToken); } catch (Exception e) { // Update the error state for the row. var entity = await AzureUtil.QueryAsync <UnprocessedBuildEntity>(_unprocessedBuildTable, key, cancellationToken); if (entity != null) { entity.StatusText = $"{e.Message} - {e.StackTrace.Take(1000)}"; var operation = TableOperation.Replace(entity); try { await _unprocessedBuildTable.ExecuteAsync(operation); } catch { // It's possible the enity was deleted / updated in parallel. That's okay. This table // is meant as an approximation of the build state and always moving towards complete. } } } }