public static BuildResultInfo GetBuildResultInfo(this ModelBuild modelBuild) =>
new BuildResultInfo(
GetBuildAndDefinitionInfo(modelBuild),
modelBuild.QueueTime,
modelBuild.StartTime,
modelBuild.FinishTime,
modelBuild.BuildResult ?? BuildResult.None);
public async Task <ModelGitHubIssue> EnsureGitHubIssueAsync(ModelBuild modelBuild, GitHubIssueKey issueKey, bool saveChanges)
{
var query = GetModelBuildQuery(modelBuild.GetBuildKey())
.SelectMany(x => x.ModelGitHubIssues)
.Where(x =>
x.Number == issueKey.Number &&
x.Organization == issueKey.Organization &&
x.Repository == issueKey.Repository);
var modelGitHubIssue = await query.SingleOrDefaultAsync().ConfigureAwait(false);
if (modelGitHubIssue is object)
{
return(modelGitHubIssue);
}
modelGitHubIssue = new ModelGitHubIssue()
{
Organization = issueKey.Organization,
Repository = issueKey.Repository,
Number = issueKey.Number,
ModelBuild = modelBuild,
};
Context.ModelGitHubIssues.Add(modelGitHubIssue);
if (saveChanges)
{
await Context.SaveChangesAsync().ConfigureAwait(false);
}
return(modelGitHubIssue);
}
/// <summary>
/// There are times when the AzDO API will not provide a Timeline for a Bulid. In those cases we still need to create
/// a <see cref="ModelBuildAttempt"/> entry. The assumption is this is for the first attempt and all of the values will
/// be
/// </summary>
/// <param name="modelBuild"></param>
/// <param name="build"></param>
/// <returns></returns>
public async Task <ModelBuildAttempt> EnsureBuildAttemptWithoutTimelineAsync(ModelBuild modelBuild, Build build)
{
const int attempt = 1;
var modelBuildAttempt = await Context.ModelBuildAttempts
.Where(x => x.ModelBuildId == modelBuild.Id && x.Attempt == attempt)
.FirstOrDefaultAsync().ConfigureAwait(false);
if (modelBuildAttempt is object)
{
return(modelBuildAttempt);
}
modelBuildAttempt = new ModelBuildAttempt()
{
Attempt = attempt,
BuildResult = build.Result,
StartTime = modelBuild.StartTime,
FinishTime = modelBuild.FinishTime,
ModelBuild = modelBuild,
IsTimelineMissing = false,
};
Context.ModelBuildAttempts.Add(modelBuildAttempt);
await Context.SaveChangesAsync().ConfigureAwait(false);
return(modelBuildAttempt);
}
public static BuildAndDefinitionInfo GetBuildAndDefinitionInfo(this ModelBuild modelBuild) =>
new BuildAndDefinitionInfo(
modelBuild.AzureOrganization,
modelBuild.AzureProject,
modelBuild.BuildNumber,
modelBuild.DefinitionId,
modelBuild.DefinitionName,
GetGitHubBuildInfo(modelBuild));
public async Task EnsureResultAsync(ModelBuild modelBuild, Build build)
{
if (modelBuild.BuildResult != build.Result)
{
var buildInfo = build.GetBuildResultInfo();
modelBuild.BuildResult = build.Result;
modelBuild.StartTime = buildInfo.StartTime;
modelBuild.FinishTime = buildInfo.FinishTime;
await Context.SaveChangesAsync().ConfigureAwait(false);
}
}
public async Task <ModelBuild> EnsureBuildAsync(BuildResultInfo buildInfo)
{
var modelBuildId = GetModelBuildId(buildInfo.BuildKey);
var modelBuild = Context.ModelBuilds
.Where(x => x.Id == modelBuildId)
.FirstOrDefault();
if (modelBuild is object)
{
// This code accounts for the fact that we will see multiple attempts of a build and that will
// change the result. When those happens we should update all of the following values. It may
// seem strange to update start and finish time here but that is how the AzDO APIs work and it's
// best to model them in that way.
if (modelBuild.BuildResult != buildInfo.BuildResult)
{
modelBuild.StartTime = buildInfo.StartTime;
modelBuild.FinishTime = buildInfo.FinishTime;
modelBuild.BuildResult = buildInfo.BuildResult;
await Context.SaveChangesAsync().ConfigureAwait(false);
}
return(modelBuild);
}
var prKey = buildInfo.PullRequestKey;
var modelBuildDefinition = await EnsureBuildDefinitionAsync(buildInfo.DefinitionInfo).ConfigureAwait(false);
modelBuild = new ModelBuild()
{
Id = modelBuildId,
ModelBuildDefinitionId = modelBuildDefinition.Id,
AzureOrganization = modelBuildDefinition.AzureOrganization,
AzureProject = modelBuildDefinition.AzureProject,
GitHubOrganization = buildInfo.GitHubBuildInfo?.Organization,
GitHubRepository = buildInfo.GitHubBuildInfo?.Repository,
GitHubTargetBranch = buildInfo.GitHubBuildInfo?.TargetBranch,
PullRequestNumber = prKey?.Number,
StartTime = buildInfo.StartTime,
FinishTime = buildInfo.FinishTime,
QueueTime = buildInfo.QueueTime,
BuildNumber = buildInfo.Number,
BuildResult = buildInfo.BuildResult,
DefinitionName = buildInfo.DefinitionName,
DefinitionId = buildInfo.DefinitionInfo.Id,
};
Context.ModelBuilds.Add(modelBuild);
Context.SaveChanges();
return(modelBuild);
}
public static DefinitionKey GetDefinitionKey(this ModelBuild modelBuild) =>
new DefinitionKey(
modelBuild.AzureOrganization,
modelBuild.AzureProject,
modelBuild.DefinitionId);
public static ModelBuildKind GetModelBuildKind(this ModelBuild modelBuild) => TriageContextUtil.GetModelBuildKind(modelBuild.IsMergedPullRequest, modelBuild.PullRequestNumber);
public static GitHubBuildInfo GetGitHubBuildInfo(this ModelBuild modelBuild) =>
new GitHubBuildInfo(
modelBuild.GitHubOrganization,
modelBuild.GitHubRepository,
modelBuild.PullRequestNumber,
modelBuild.GitHubTargetBranch);
public static BuildInfo GetBuildInfo(this ModelBuild modelBuild) =>
new BuildInfo(
modelBuild.AzureOrganization,
modelBuild.AzureProject,
modelBuild.BuildNumber,
GetGitHubBuildInfo(modelBuild));
public static BuildKey GetBuildKey(this ModelBuild modelBuild) =>
new BuildKey(
modelBuild.AzureOrganization,
modelBuild.AzureProject,
modelBuild.BuildNumber);
public static GitHubPullRequestKey?GetGitHubPullRequestKey(ModelBuild build) =>
build.PullRequestNumber.HasValue
? (GitHubPullRequestKey?)new GitHubPullRequestKey(build.GitHubOrganization, build.GitHubRepository, build.PullRequestNumber.Value)
: null;
public async Task <ModelTestRun> EnsureTestRunAsync(ModelBuild modelBuild, int attempt, DotNetTestRun testRun, Dictionary <HelixInfo, HelixLogInfo> helixMap)
{
var modelTestRun = await FindModelTestRunAsync(modelBuild.GetBuildKey(), testRun.TestRun.Id).ConfigureAwait(false);
if (modelTestRun is object)
{
return(modelTestRun);
}
var buildInfo = testRun.Build.GetBuildResultInfo();
modelTestRun = new ModelTestRun()
{
AzureOrganization = buildInfo.Organization,
AzureProject = buildInfo.Project,
ModelBuild = modelBuild,
TestRunId = testRun.TestRun.Id,
Name = testRun.TestRun.Name,
Attempt = attempt,
};
Context.ModelTestRuns.Add(modelTestRun);
foreach (var dotnetTestCaseResult in testRun.TestCaseResults)
{
var testCaseResult = dotnetTestCaseResult.TestCaseResult;
var testResult = new ModelTestResult()
{
TestFullName = testCaseResult.TestCaseTitle,
Outcome = testCaseResult.Outcome,
ModelTestRun = modelTestRun,
ModelBuild = modelBuild,
ErrorMessage = testCaseResult.ErrorMessage,
IsSubResultContainer = testCaseResult.SubResults?.Length > 0,
IsSubResult = false,
};
AddHelixInfo(testResult);
Context.ModelTestResults.Add(testResult);
if (testCaseResult.SubResults is { } subResults)
{
foreach (var subResult in subResults)
{
var iterationTestResult = new ModelTestResult()
{
TestFullName = testCaseResult.TestCaseTitle,
Outcome = subResult.Outcome,
ModelTestRun = modelTestRun,
ModelBuild = modelBuild,
ErrorMessage = subResult.ErrorMessage,
IsSubResultContainer = false,
IsSubResult = true
};
AddHelixInfo(iterationTestResult);
Context.ModelTestResults.Add(iterationTestResult);
}
}
void AddHelixInfo(ModelTestResult testResult)
{
if (dotnetTestCaseResult.HelixInfo is { } helixInfo&&
helixMap.TryGetValue(helixInfo, out var helixLogInfo))
{
testResult.IsHelixTestResult = true;
testResult.HelixConsoleUri = helixLogInfo.ConsoleUri;
testResult.HelixCoreDumpUri = helixLogInfo.CoreDumpUri;
testResult.HelixRunClientUri = helixLogInfo.RunClientUri;
testResult.HelixTestResultsUri = helixLogInfo.TestResultsUri;
}
}
}
await Context.SaveChangesAsync().ConfigureAwait(false);
return(modelTestRun);
}
public async Task <ModelBuildAttempt> EnsureBuildAttemptAsync(ModelBuild modelBuild, BuildResult buildResult, Timeline timeline)
{
var attempt = timeline.GetAttempt();
var modelBuildAttempt = await Context.ModelBuildAttempts
.Where(x => x.ModelBuildId == modelBuild.Id && x.Attempt == attempt)
.FirstOrDefaultAsync().ConfigureAwait(false);
if (modelBuildAttempt is object && !modelBuildAttempt.IsTimelineMissing)
{
return(modelBuildAttempt);
}
var startTimeQuery = timeline
.Records
.Where(x => x.Attempt == attempt)
.Select(x => DevOpsUtil.ConvertFromRestTime(x.StartTime))
.SelectNullableValue()
.Select(x => (DateTime?)x.DateTime);
var startTime = startTimeQuery.Any()
? startTimeQuery.Min()
: modelBuild.StartTime;
var finishTimeQuery = timeline
.Records
.Where(x => x.Attempt == attempt)
.Select(x => DevOpsUtil.ConvertFromRestTime(x.FinishTime))
.SelectNullableValue()
.Select(x => (DateTime?)x.DateTime);
var finishTime = finishTimeQuery.Any()
? finishTimeQuery.Max()
: modelBuild.FinishTime;
if (modelBuildAttempt is object)
{
modelBuildAttempt.BuildResult = buildResult;
modelBuildAttempt.StartTime = startTime;
modelBuildAttempt.FinishTime = finishTime;
modelBuildAttempt.IsTimelineMissing = false;
}
else
{
modelBuildAttempt = new ModelBuildAttempt()
{
Attempt = attempt,
BuildResult = buildResult,
StartTime = startTime,
FinishTime = finishTime,
ModelBuild = modelBuild,
IsTimelineMissing = false,
};
Context.ModelBuildAttempts.Add(modelBuildAttempt);
}
var timelineTree = TimelineTree.Create(timeline);
foreach (var record in timeline.Records)
{
if (record.Issues is null ||
!timelineTree.TryGetJob(record, out var job))
{
continue;
}
foreach (var issue in record.Issues)
{
var timelineIssue = new ModelTimelineIssue()
{
Attempt = attempt,
JobName = job.Name,
RecordName = record.Name,
TaskName = record.Task?.Name ?? "",
RecordId = record.Id,
Message = issue.Message,
ModelBuild = modelBuild,
IssueType = issue.Type,
ModelBuildAttempt = modelBuildAttempt,
};
Context.ModelTimelineIssues.Add(timelineIssue);
}
}
await Context.SaveChangesAsync().ConfigureAwait(false);
return(modelBuildAttempt);
}
private async Task RetryOsxDeprovisionAsync(ModelBuild modelBuild, List <ModelBuildAttempt> modelBuildAttempts)
{
Logger.LogInformation("Considering OSX deprovision retry");
if (modelBuildAttempts.Count > 1)
{
Logger.LogInformation("Build already has multiple attempts");
return;
}
var issues = await Context
.ModelTimelineIssues
.Where(x => x.ModelBuildId == modelBuild.Id)
.Select(x => new
{
x.Message,
x.IssueType,
x.JobName
})
.ToListAsync()
.ConfigureAwait(false);
var count = 0;
foreach (var issue in issues)
{
if (issue.Message.Contains("Received request to deprovision: The request was cancelled by the remote provider", StringComparison.OrdinalIgnoreCase))
{
count++;
}
}
if (count == 0)
{
Logger.LogInformation("No OSX failures");
return;
}
var jobFailedCount = issues
.Where(x => x.IssueType == IssueType.Error)
.GroupBy(x => x.JobName)
.Count();
if (jobFailedCount - count >= 4)
{
Logger.LogInformation("Too many non-OSX failures");
return;
}
Logger.LogInformation("Retrying");
await Server.RetryBuildAsync(modelBuild.AzureProject, modelBuild.BuildNumber).ConfigureAwait(false);
var model = new ModelOsxDeprovisionRetry()
{
OsxJobFailedCount = count,
JobFailedCount = jobFailedCount,
ModelBuild = modelBuild,
};
Context.ModelOsxDeprovisionRetry.Add(model);
await Context.SaveChangesAsync().ConfigureAwait(false);
}
