public async Task ProcessBuildAsync(BuildKey buildKey)
{
var modelBuildAttempts = await TriageContextUtil
.GetModelBuildAttemptsQuery(buildKey)
.Include(x => x.ModelBuild)
.ToListAsync()
.ConfigureAwait(false);
var modelBuild = modelBuildAttempts.FirstOrDefault()?.ModelBuild;
if (modelBuild is null)
{
// This happens when we have no data on the build at all
Logger.LogWarning($"No model for the build {buildKey}");
return;
}
var failed = modelBuild.BuildResult == BuildResult.Failed || modelBuild.BuildResult == BuildResult.Canceled;
if (!failed)
{
Logger.LogWarning($"Build did not fail so no retry is needed");
return;
}
await RetryOsxDeprovisionAsync(modelBuild, modelBuildAttempts);
}
/// <summary>
/// Get the report for builds that are associated with the given GitHub Issue Key
/// </summary>
public async Task <string> GetAssociatedIssueReportAsync(GitHubIssueKey issueKey)
{
var query = TriageContextUtil
.GetModelGitHubIssuesQuery(issueKey)
.Select(x => new
{
x.ModelBuild.AzureOrganization,
x.ModelBuild.AzureProject,
x.ModelBuild.BuildNumber,
x.ModelBuild.QueueTime,
x.ModelBuild.PullRequestNumber,
x.ModelBuild.GitHubOrganization,
x.ModelBuild.GitHubRepository,
x.ModelBuild.GitHubTargetBranch,
});
var builds = await query.ToListAsync().ConfigureAwait(false);
var results = builds
.Select(x => (new BuildInfo(
x.AzureOrganization,
x.AzureProject,
x.BuildNumber,
new GitHubBuildInfo(x.GitHubOrganization, x.GitHubRepository, x.PullRequestNumber, x.GitHubTargetBranch)),
x.QueueTime));
var report = ReportBuilder.BuildManual(results);
return(WrapInStartEndMarkers(report));
}
public StatusPageUtil(
IGitHubClientFactory gitHubClientFactory,
TriageContext context,
ILogger logger)
{
GitHubClientFactory = gitHubClientFactory;
TriageContextUtil = new TriageContextUtil(context);
Logger = logger;
}
public BuildRetryUtil(
DevOpsServer server,
TriageContext context,
ILogger logger)
{
Server = server;
TriageContextUtil = new TriageContextUtil(context);
Logger = logger;
}
public ModelDataUtil(
DotNetQueryUtil queryUtil,
TriageContextUtil triageContextUtil,
ILogger logger)
{
Server = queryUtil.Server;
QueryUtil = queryUtil;
TriageContextUtil = triageContextUtil;
Logger = logger;
}
public TrackingGitHubUtil(
IGitHubClientFactory gitHubClientFactory,
TriageContext context,
SiteLinkUtil siteLinkUtil,
ILogger logger)
{
GitHubClientFactory = gitHubClientFactory;
TriageContextUtil = new TriageContextUtil(context);
SiteLinkUtil = siteLinkUtil;
Logger = logger;
}
public TrackingIssueUtil(
HelixServer helixServer,
DotNetQueryUtil queryUtil,
TriageContextUtil triageContextUtil,
ILogger logger)
{
HelixServer = helixServer;
QueryUtil = queryUtil;
TriageContextUtil = triageContextUtil;
Logger = logger;
}
public async Task TriageAsync(BuildKey buildKey, int modelTrackingIssueId)
{
var attempts = await TriageContextUtil
.GetModelBuildAttemptsQuery(buildKey)
.Include(x => x.ModelBuild)
.ToListAsync()
.ConfigureAwait(false);
foreach (var attempt in attempts)
{
await TriageAsync(attempt.GetBuildAttemptKey(), modelTrackingIssueId).ConfigureAwait(false);
}
}
public async Task <BuildAttemptKey> EnsureModelInfoAsync(Build build, bool includeTests = true)
{
var buildInfo = build.GetBuildResultInfo();
var modelBuild = await TriageContextUtil.EnsureBuildAsync(buildInfo).ConfigureAwait(false);
await TriageContextUtil.EnsureResultAsync(modelBuild, build).ConfigureAwait(false);
var modelBuildAttempt = await EnsureTimeline().ConfigureAwait(false);
if (includeTests)
{
await EnsureTestRuns().ConfigureAwait(false);
}
return(new BuildAttemptKey(new BuildKey(build), modelBuildAttempt.Attempt));
async Task <ModelBuildAttempt> EnsureTimeline()
{
try
{
var timeline = await Server.GetTimelineAsync(buildInfo.Project, buildInfo.Number).ConfigureAwait(false);
if (timeline is null)
{
Logger.LogWarning("No timeline");
}
else
{
return(await TriageContextUtil.EnsureBuildAttemptAsync(buildInfo, timeline).ConfigureAwait(false));
}
}
catch (Exception ex)
{
Logger.LogWarning($"Error getting timeline: {ex.Message}");
}
return(await TriageContextUtil.EnsureBuildAttemptWithoutTimelineAsync(modelBuild, build).ConfigureAwait(false));
}
async Task EnsureTestRuns()
{
TestRun[] testRuns;
try
{
testRuns = await Server.ListTestRunsAsync(buildInfo.Project, buildInfo.Number).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning($"Error getting test runs: {ex.Message}");
return;
}
var attempt = modelBuildAttempt?.Attempt ?? 1;
foreach (var testRun in testRuns)
{
await EnsureTestRun(testRun, attempt).ConfigureAwait(false);
}
}
async Task EnsureTestRun(TestRun testRun, int attempt)
{
try
{
var modelTestRun = await TriageContextUtil.FindModelTestRunAsync(modelBuild.GetBuildKey(), testRun.Id).ConfigureAwait(false);
if (modelTestRun is object)
{
return;
}
// TODO: Need to record when the maximum test results are exceeded. The limit here is to
// protect us from a catastrophic run that has say several million failures (this is a real
// possibility
const int maxTestCaseResultCount = 200;
var dotNetTestRun = await QueryUtil.GetDotNetTestRunAsync(
build,
testRun,
DotNetUtil.FailedTestOutcomes,
includeSubResults : true,
onError : ex => Logger.LogWarning($"Error fetching test data {ex.Message}")).ConfigureAwait(false);
if (dotNetTestRun.TestCaseResults.Count > maxTestCaseResultCount)
{
dotNetTestRun = new DotNetTestRun(
dotNetTestRun.TestRunInfo,
dotNetTestRun.TestCaseResults.Take(maxTestCaseResultCount).ToReadOnlyCollection());
}
var helixMap = await Server.GetHelixMapAsync(dotNetTestRun).ConfigureAwait(false);
await TriageContextUtil.EnsureTestRunAsync(modelBuild, attempt, dotNetTestRun, helixMap).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.LogWarning($"Error uploading test run: {ex.Message}");
return;
}
}
}
public async Task <string> GetStatusIssueTextAsync(IGitHubClient gitHubClient)
{
var header = new StringBuilder();
var body = new StringBuilder();
var footer = new StringBuilder();
header.AppendLine("## Overview");
header.AppendLine("Please use these queries to discover issues");
await BuildOne("Blocking CI", "blocking-clean-ci", DotNetUtil.GetDefinitionKeyFromFriendlyName("runtime")).ConfigureAwait(false);
await BuildOne("Blocking Official Build", "blocking-official-build", DotNetUtil.GetDefinitionKeyFromFriendlyName("runtime-official")).ConfigureAwait(false);
await BuildOne("Blocking CI Optional", "blocking-clean-ci-optional", DotNetUtil.GetDefinitionKeyFromFriendlyName("runtime"));
await BuildOne("Blocking Outerloop", "blocking-outerloop", null);
// Blank line to move past the table
header.AppendLine("");
BuildFooter();
return(header.ToString() + body.ToString() + footer.ToString());
void BuildFooter()
{
footer.AppendLine(@"## Goals
1. A minimum 95% passing rate for the `runtime` pipeline
## Resources
1. [runtime pipeline analytics](https://dnceng.visualstudio.com/public/_build?definitionId=686&view=ms.vss-pipelineanalytics-web.new-build-definition-pipeline-analytics-view-cardmetrics)");
}
async Task BuildOne(string title, string label, DefinitionKey?definitionKey)
{
header.AppendLine($"- [{title}](https://github.com/dotnet/runtime/issues?q=is%3Aopen+is%3Aissue+label%3A{label})");
body.AppendLine($"## {title}");
body.AppendLine("|Status|Issue|Build Count|");
body.AppendLine("|---|---|---|");
var query = (await DoSearchAsync(gitHubClient, label).ConfigureAwait(false))
.OrderByDescending(x => x.Count);
foreach (var(issue, count) in query)
{
var emoji = issue.Labels.Any(x => x.Name == "intermittent")
? ":warning:"
: ":fire:";
var titleLimit = 75;
var issueText = issue.Title.Length >= titleLimit
? issue.Title.Substring(0, titleLimit - 5) + " ..."
: issue.Title;
var issueEntry = $"[{issueText}]({issue.HtmlUrl})";
var countStr = count.HasValue ? count.ToString() : "N/A";
body.AppendLine($"|{emoji}|{issueEntry}|{countStr}|");
}
}
async Task <List <(Octokit.Issue Issue, int?Count)> > DoSearchAsync(IGitHubClient gitHubClient, string label)
{
var request = new SearchIssuesRequest()
{
Labels = new [] { label },
State = ItemState.Open,
Type = IssueTypeQualifier.Issue,
Repos = { { "dotnet", "runtime" } },
};
var result = await gitHubClient.Search.SearchIssues(request).ConfigureAwait(false);
var list = new List <(Octokit.Issue Issue, int?Count)>();
foreach (var issue in result.Items)
{
var count = await GetImpactedBuildsCountAsync(issue.GetIssueKey()).ConfigureAwait(false);
list.Add((issue, count));
}
return(list);
}
async Task <int?> GetImpactedBuildsCountAsync(GitHubIssueKey issueKey)
{
var modelTrackingIssue = await TriageContextUtil
.GetModelTrackingIssuesQuery(issueKey)
.FirstOrDefaultAsync()
.ConfigureAwait(false);
if (modelTrackingIssue is null)
{
return(null);
}
var count = await Context
.ModelTrackingIssueResults
.Where(x => x.IsPresent && x.ModelTrackingIssueId == modelTrackingIssue.Id)
.CountAsync()
.ConfigureAwait(false);
return(count);;
}
}
public static ModelBuildKind GetModelBuildKind(this ModelBuild modelBuild) => TriageContextUtil.GetModelBuildKind(modelBuild.IsMergedPullRequest, modelBuild.PullRequestNumber);
private Task <ModelBuildAttempt> GetModelBuildAttemptAsync(BuildAttemptKey attemptKey) => TriageContextUtil .GetModelBuildAttemptQuery(attemptKey) .Include(x => x.ModelBuild) .SingleAsync();
public async Task TriageAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
if (modelBuildAttempt.ModelBuild is null)
{
throw new Exception("The attempt must include the build");
}
if (modelTrackingIssue.ModelBuildDefinitionId is { } definitionId&&
definitionId != modelBuildAttempt.ModelBuild.ModelBuildDefinitionId)
{
return;
}
// Quick spot check to avoid doing extra work if we've already triaged this attempt against this
// issue
if (await WasTriaged().ConfigureAwait(false))
{
return;
}
bool isPresent;
switch (modelTrackingIssue.TrackingKind)
{
case TrackingKind.Test:
isPresent = await TriageTestAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
case TrackingKind.Timeline:
isPresent = await TriageTimelineAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
case TrackingKind.HelixLogs:
isPresent = await TriageHelixLogsAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
#pragma warning disable 618
case TrackingKind.HelixConsole:
case TrackingKind.HelixRunClient:
// TODO: delete this once the DB is cleaned up
// These are old data types that we ignore.
isPresent = false;
break;
#pragma warning restore 618
default:
throw new Exception($"Unknown value {modelTrackingIssue.TrackingKind}");
}
var result = new ModelTrackingIssueResult()
{
ModelBuildAttempt = modelBuildAttempt,
ModelTrackingIssue = modelTrackingIssue,
IsPresent = isPresent
};
Context.ModelTrackingIssueResults.Add(result);
// This can race with other attempts to associate issues here. That is okay though because triage attempts are
// retried because they assume races with other operations can happen.
if (isPresent && modelTrackingIssue.GetGitHubIssueKey() is { } issueKey)
{
await TriageContextUtil.EnsureGitHubIssueAsync(modelBuildAttempt.ModelBuild, issueKey, saveChanges : false).ConfigureAwait(false);
}
await Context.SaveChangesAsync().ConfigureAwait(false);
async Task <bool> WasTriaged()
{
var query = Context
.ModelTrackingIssueResults
.Where(x => x.ModelBuildAttemptId == modelBuildAttempt.Id && x.ModelTrackingIssueId == modelTrackingIssue.Id);
return(await query.AnyAsync().ConfigureAwait(false));
}
}