public BuildResultEntity( BoundBuildId buildId, DateTimeOffset buildDateTime, TimeSpan duration, string jobKind, string machineName, BuildResultClassification classification, PullRequestInfo prInfo) { JobName = buildId.JobId.Name; JobKind = jobKind; ViewName = AzureUtil.GetViewName(BuildId.JobId); BuildNumber = buildId.Number; HostRaw = buildId.Host.ToString(); ClassificationKindRaw = classification.Kind.ToString(); ClassificationName = classification.Name; BuildDateTime = buildDateTime.UtcDateTime; MachineName = machineName; IsPullRequest = JobUtil.IsPullRequestJobName(buildId.JobId); DurationSeconds = (int)duration.TotalSeconds; if (prInfo != null) { PullRequestId = prInfo.Id; PullRequestAuthor = prInfo.Author; PullRequestAuthorEmail = prInfo.AuthorEmail; PullRequestUrl = prInfo.PullUrl; PullRequestSha1 = prInfo.Sha1; Debug.Assert(HasPullRequestInfo); Debug.Assert(PullRequestInfo != null); } Debug.Assert(BuildDateTime.Kind == DateTimeKind.Utc); }
public BuildStateMessage(DateTimeKey buildStateKey, BoundBuildId buildId) { BuildStateKeyRaw = buildStateKey.Key; HostRaw = buildId.Host.ToString(); JobName = buildId.JobName; BuildNumber = buildId.Number; }
/// <summary> /// Populate the <see cref="BuildResultEntity"/> structures for a build overwriting any data /// that existed before. Returns the entity if enough information was there to process the value. /// </summary> public async Task PopulateBuild(BoundBuildId buildId) { var data = await GetPopulateData(buildId); var result = data.Result; await PopulateViewName(buildId.JobId, result.BuildDateTimeOffset); await _buildResultDateTable.ExecuteAsync(TableOperation.InsertOrReplace(result.CopyDate())); await _buildResultExactTable.ExecuteAsync(TableOperation.InsertOrReplace(result.CopyExact())); var failures = data.Failures; if (failures.Count > 0) { // Important to use InsertOrReplace here. It's possible for a populate job to be cut off in the // middle when the BuildFailure table is updated but not yet the BuildProcessed table. Hence // we'll up here again doing a batch insert. await AzureUtil.ExecuteBatchUnordered(_buildFailureExactTable, TableOperationType.InsertOrReplace, failures.Select(x => x.CopyExact())); await AzureUtil.ExecuteBatchUnordered(_buildFailureDateTable, TableOperationType.InsertOrReplace, failures.Select(x => x.CopyDate())); } await PopulateCounters(result); }
private static async Task Random() { /* * var boundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_corefx/job/master/job/fedora23_debug_tst/134/"); * var buildId = boundBuildId.BuildId; * var client = CreateClient(uri: boundBuildId.HostUri, auth: true); * var buildInfo = await client.GetBuildInfoAsync(buildId); * var buildResult = await client.GetBuildResultAsync(buildInfo); * var test = await client.GetFailedTestCasesAsync(buildId); * var prInfo = await client.GetPullRequestInfoAsync(buildId); */ var testboundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/release_1.0.0/job/x64_release_rhel7.2_pri1_flow/30/"); var testbuildId = testboundBuildId.BuildId; var client = CreateClient(uri: testboundBuildId.HostUri, auth: true); var elapsedTimeObj = client.GetBuildInfo(testbuildId).Duration; Console.WriteLine($"\tET: {elapsedTimeObj.TotalMilliseconds}"); var account = GetStorageAccount(); var dateKey = new DateKey(DateTimeOffset.UtcNow - TimeSpan.FromDays(1)); var table = account.CreateCloudTableClient().GetTableReference(AzureConstants.TableNames.BuildResultDate); var query = new TableQuery <BuildResultEntity>() .Where(FilterUtil .Column(ColumnNames.PartitionKey, dateKey, ColumnOperator.GreaterThanOrEqual) .And(FilterUtil.Column("MachineName", "Azure0602081822"))); var all = await AzureUtil.QueryAsync(table, query); foreach (var entity in all) { var boundBuildId = new BoundBuildId(SharedConstants.DotnetJenkinsUri.Host, entity.BuildId); Console.WriteLine(boundBuildId.Uri); } }
/// <summary> /// Function to help diagnose failures in processing. /// </summary> /// <returns></returns> private static async Task TestFailure() { /* * var name = "Private/Microsoft_vstest/master/Microsoft_vstest_Release_prtest"; * var number = 119; * var host = SharedConstants.DotnetJenkinsHostName; * var jobId = JobId.ParseName(name); * var boundBuildId = new BoundBuildId(host, new BuildId(number, jobId)); */ var url = "http://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/master/job/debug_windows_nt_bld/198"; var boundBuildId = BoundBuildId.Parse(url); var buildId = boundBuildId.BuildId; var account = GetStorageAccount(); var client = CreateClient(boundBuildId); var populator = new BuildTablePopulator(account.CreateCloudTableClient(), CounterUtilFactory, client, Console.Out); try { await populator.PopulateBuild(boundBuildId); } catch (Exception ex) { Console.WriteLine(ex); } }
/// <summary> /// Get or create the build state partition key for the build id. /// </summary> /// <remarks> /// This has to take into account that builds take place across days. Which day wins doesn't matter /// so long as we don't duplicate the data. /// </remarks> internal async Task<DateTimeKey> GetOrCreateBuildStateKey(BoundBuildId buildId) { // TODO: do this correctly var key = new DateTimeKey(DateTimeOffset.UtcNow, DateTimeKeyFlags.Date); var task = new Task<DateTimeKey>(() => key); task.Start(); return await task; }
public BuildStateEntity(DateTimeKey key, BoundBuildId buildId, bool isBuildFinished) { PartitionKey = key.Key; RowKey = GetRowKey(buildId); HostRaw = buildId.Host.ToString(); BuildNumber = buildId.Number; JobName = buildId.JobName; IsBuildFinished = isBuildFinished; }
/// <summary> /// Get or create the build state partition key for the build id. /// </summary> /// <remarks> /// This has to take into account that builds take place across days. Which day wins doesn't matter /// so long as we don't duplicate the data. /// </remarks> internal async Task <DateTimeKey> GetOrCreateBuildStateKey(BoundBuildId buildId) { // TODO: do this correctly var key = new DateTimeKey(DateTimeOffset.UtcNow, DateTimeKeyFlags.Date); var task = new Task <DateTimeKey>(() => key); task.Start(); return(await task); }
internal static BuildInfo ParseBuildInfo(Uri host, JobId jobId, JObject build) { var id = build.Value<int>("id"); var duration = TimeSpan.FromMilliseconds(build.Value<int>("duration")); var state = ParseBuildInfoState(build); var date = JenkinsUtil.ConvertTimestampToDateTimeOffset(build.Value<long>("timestamp")); var buildId = new BoundBuildId(host, id, jobId); var machineName = build.Value<string>("builtOn"); return new BuildInfo(buildId, state, date, duration, machineName); }
internal static JenkinsClient CreateJenkinsClient(BoundBuildId buildId) { if (JobUtil.IsAuthNeededHeuristic(buildId.JobId)) { var githubConnectionString = CloudConfigurationManager.GetSetting(SharedConstants.GithubConnectionStringName); var host = buildId.GetHostUri(useHttps: true); return(new JenkinsClient(host, githubConnectionString)); } else { return(new JenkinsClient(buildId.Host)); } }
public UnprocessedBuildEntity(BoundBuildId boundBuildId) { var buildId = boundBuildId.BuildId; var entityKey = GetEntityKey(buildId); PartitionKey = entityKey.PartitionKey; RowKey = entityKey.RowKey; JobName = buildId.JobName; BuildNumber = buildId.Number; LastUpdate = DateTime.UtcNow; HostName = boundBuildId.HostName; UriScheme = boundBuildId.UriScheme; }
private async Task <PopulateData> GetPopulateData(BoundBuildId buildId) { try { var data = await GetPopulateDataCore(buildId); WriteLine(buildId, $"adding reason {data.Result.ClassificationKind}"); return(data); } catch (Exception ex) { WriteLine(buildId, $"error processing {ex.Message}"); throw; } }
private static async Task TestPopulator() { var account = GetStorageAccount(); var client = CreateClient(auth: false); var populator = new BuildTablePopulator(account.CreateCloudTableClient(), client, Console.Out); var boundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/master/job/jitstress/job/x64_checked_osx_jitstress1_flow/7/"); try { await populator.PopulateBuild(boundBuildId.BuildId); } catch (Exception ex) { Console.WriteLine(ex); } }
public async Task PopulateBuildMissing(BoundBuildId buildId) { // The bulid is now deemed to be missing. Finish it off. var result = new BuildResultEntity( buildId, DateTimeOffset.UtcNow, TimeSpan.MinValue, JobKind.Normal, "", BuildResultClassification.Infrastructure, prInfo: null); // Deliberately using Insert here vs. InsertOrReplace. This is the worst possible outcome for this // type so let anyone else win await _buildResultDateTable.ExecuteAsync(TableOperation.Insert(result.CopyDate())); await _buildResultDateTable.ExecuteAsync(TableOperation.Insert(result.CopyExact())); await PopulateCounters(result); }
public BuildFailureEntity(BoundBuildId buildId, string identifier, DateTimeOffset buildDate, BuildFailureKind kind, string jobKind, string machineName, PullRequestInfo prInfo) { JobName = buildId.JobName; JobKind = jobKind; ViewName = AzureUtil.GetViewName(buildId.JobId); BuildNumber = buildId.Number; HostRaw = buildId.Host.ToString(); Identifier = identifier; BuildFailureKindRaw = kind.ToString(); BuildDateTime = buildDate.UtcDateTime; IsPullRequest = JobUtil.IsPullRequestJobName(buildId.JobId); MachineName = machineName; if (prInfo != null) { PullRequestId = prInfo.Id; PullRequestAuthor = prInfo.Author; PullRequestAuthorEmail = prInfo.AuthorEmail; PullRequestUrl = prInfo.PullUrl; PullRequestSha1 = prInfo.Sha1; Debug.Assert(HasPullRequestInfo); Debug.Assert(PullRequestInfo != null); } }
public static EntityKey GetEntityKey(DateTimeKey key, BoundBuildId buildId) => new EntityKey(key.Key, GetRowKey(buildId));
public static BuildFailureEntity CreateTestCaseFailure(DateTimeOffset buildDateTime, BoundBuildId buildId, string testCaseName, string jobKind, string machineName, PullRequestInfo prInfo) { return(new BuildFailureEntity( buildId, testCaseName, buildDateTime, BuildFailureKind.TestCase, jobKind: jobKind, machineName: machineName, prInfo: prInfo)); }
public void SimpleWithPort() { var uri = new Uri("https://example.com:400"); Assert.Equal(uri, BoundBuildId.NormalizeHostUri(uri)); }
public static EntityKey GetEntityKey(DateTimeOffset dateTime, BoundBuildId buildId) => new EntityKey(GetPartitionKey(dateTime), GetRowKey(buildId));
private async Task<PopulateData> GetPopulateData(BoundBuildId buildId) { try { var data = await GetPopulateDataCore(buildId); WriteLine(buildId, $"adding reason {data.Result.ClassificationKind}"); return data; } catch (Exception ex) { WriteLine(buildId, $"error processing {ex.Message}"); throw; } }
private void WriteLine(BoundBuildId buildId, string message) { _textWriter.WriteLine($"{buildId.JobName} - {buildId.Number}: {message}"); }
public static BuildFailureEntity CreateTestCaseFailure(DateTimeOffset buildDateTime, BoundBuildId buildId, string testCaseName, string jobKind, string machineName, PullRequestInfo prInfo) { return new BuildFailureEntity( buildId, testCaseName, buildDateTime, BuildFailureKind.TestCase, jobKind: jobKind, machineName: machineName, prInfo: prInfo); }
private async Task EnqueueProcessBuild(DateTimeKey buildStateKey, BoundBuildId buildId, TimeSpan? delay, CancellationToken cancellationToken) { await EnqueueCore(_processBuildQueue, buildStateKey, buildId, delay, cancellationToken); }
public static string GetRowKey(BoundBuildId buildId) => BuildKey.GetKey(buildId.BuildId);
/// <summary> /// Update the table storage to contain the result of the specified build. /// </summary> private async Task <PopulateData> GetPopulateDataCore(BoundBuildId id) { var buildInfo = await _client.GetBuildInfoAsync(id.BuildId); var jobKind = await _client.GetJobKindAsync(id.JobId); PullRequestInfo prInfo = null; if (JobUtil.IsPullRequestJobName(id.JobId.Name)) { try { prInfo = await _client.GetPullRequestInfoAsync(id.BuildId); } catch (Exception ex) { // TODO: Flow builds don't have the PR directly in the triggered jobs. Have to walk // back up to the parent job. For now swallow this error so we don't trigger false // positives in the error detection. _textWriter.WriteLine($"Error pulling PR info for {id}: {ex.Message}"); } } BuildResultClassification classification; switch (buildInfo.State) { case BuildState.Succeeded: classification = BuildResultClassification.Succeeded; break; case BuildState.Aborted: classification = BuildResultClassification.Aborted; break; case BuildState.Failed: classification = await PopulateFailedBuildResult(buildInfo, jobKind, prInfo); break; case BuildState.Running: classification = BuildResultClassification.Unknown; break; default: throw new Exception($"Invalid enum: {buildInfo.State} for {id.JobName} - {id.Number}"); } var resultEntity = new BuildResultEntity( id, buildInfo.Date, buildInfo.Duration, jobKind: jobKind, machineName: buildInfo.MachineName, classification: classification, prInfo: prInfo); var failures = classification.Kind == ClassificationKind.TestFailure ? await GetUnitTestFailures(buildInfo, jobKind, prInfo) : new List <BuildFailureEntity>(); return(new PopulateData(resultEntity, failures)); }
private async Task EnqueueEmailBuild(DateTimeKey buildStateKey, BoundBuildId buildId, CancellationToken cancellationToken) { await EnqueueCore(_emailBuildQueue, buildStateKey, buildId, null, cancellationToken); }
public void PathRemoval() { var uri = new Uri("https://example.com/again"); Assert.Equal(new Uri("https://example.com"), BoundBuildId.NormalizeHostUri(uri)); }
public void CaseFixup() { var uri = new Uri("https://example.com"); Assert.Equal(new Uri("https://example.com"), BoundBuildId.NormalizeHostUri(uri)); }
private static async Task EnqueueCore(CloudQueue queue, DateTimeKey buildStateKey, BoundBuildId buildId, TimeSpan? delay, CancellationToken cancellationToken) { // Enqueue a message to process the build. Insert a delay if the build isn't finished yet so that // we don't unnecessarily ask Jenkins for information. var buildMessage = new BuildStateMessage() { BuildStateKeyRaw = buildStateKey.Key, BuildNumber = buildId.Number, HostRaw = buildId.Host.ToString(), JobName = buildId.JobName }; var queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(buildMessage)); await queue.AddMessageAsync( queueMessage, timeToLive: null, initialVisibilityDelay: delay, options: null, operationContext: null, cancellationToken: cancellationToken); }
private async Task EnqueueProcessBuild(DateTimeKey buildStateKey, BoundBuildId buildId, TimeSpan?delay, CancellationToken cancellationToken) { await EnqueueCore(_processBuildQueue, buildStateKey, buildId, delay, cancellationToken); }
private static async Task EnqueueCore(CloudQueue queue, DateTimeKey buildStateKey, BoundBuildId buildId, TimeSpan?delay, CancellationToken cancellationToken) { // Enqueue a message to process the build. Insert a delay if the build isn't finished yet so that // we don't unnecessarily ask Jenkins for information. var buildMessage = new BuildStateMessage() { BuildStateKeyRaw = buildStateKey.Key, BuildNumber = buildId.Number, HostRaw = buildId.Host.ToString(), JobName = buildId.JobName }; var queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(buildMessage)); await queue.AddMessageAsync( queueMessage, timeToLive : null, initialVisibilityDelay : delay, options : null, operationContext : null, cancellationToken : cancellationToken); }
/// <summary> /// Update the table storage to contain the result of the specified build. /// </summary> private async Task<PopulateData> GetPopulateDataCore(BoundBuildId id) { var buildInfo = await _client.GetBuildInfoAsync(id.BuildId); var jobKind = await _client.GetJobKindAsync(id.JobId); PullRequestInfo prInfo = null; if (JobUtil.IsPullRequestJobName(id.JobId.Name)) { try { prInfo = await _client.GetPullRequestInfoAsync(id.BuildId); } catch (Exception ex) { // TODO: Flow builds don't have the PR directly in the triggered jobs. Have to walk // back up to the parent job. For now swallow this error so we don't trigger false // positives in the error detection. _textWriter.WriteLine($"Error pulling PR info for {id}: {ex.Message}"); } } BuildResultClassification classification; switch (buildInfo.State) { case BuildState.Succeeded: classification = BuildResultClassification.Succeeded; break; case BuildState.Aborted: classification = BuildResultClassification.Aborted; break; case BuildState.Failed: classification = await PopulateFailedBuildResult(buildInfo, jobKind, prInfo); break; case BuildState.Running: classification = BuildResultClassification.Unknown; break; default: throw new Exception($"Invalid enum: {buildInfo.State} for {id.JobName} - {id.Number}"); } var resultEntity = new BuildResultEntity( id, buildInfo.Date, buildInfo.Duration, jobKind: jobKind, machineName: buildInfo.MachineName, classification: classification, prInfo: prInfo); var failures = classification.Kind == ClassificationKind.TestFailure ? await GetUnitTestFailures(buildInfo, jobKind, prInfo) : new List<BuildFailureEntity>(); return new PopulateData(resultEntity, failures); }
private static JenkinsClient CreateClient(BoundBuildId buildId, bool?auth = null) { return(CreateClient(buildId.Host, buildId.JobId, auth)); }
internal static JenkinsClient CreateJenkinsClient(BoundBuildId buildId) { if (JobUtil.IsAuthNeededHeuristic(buildId.JobId)) { var githubConnectionString = CloudConfigurationManager.GetSetting(SharedConstants.GithubConnectionStringName); var host = buildId.GetHostUri(useHttps: true); return new JenkinsClient(host, githubConnectionString); } else { return new JenkinsClient(buildId.Host); } }