private async Task <bool> TriageTimelineAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
Debug.Assert(modelBuildAttempt.ModelBuild is object);
Debug.Assert(modelTrackingIssue.IsActive);
Debug.Assert(modelTrackingIssue.TrackingKind == TrackingKind.Timeline);
Debug.Assert(modelTrackingIssue.SearchQuery is object);
var request = new SearchTimelinesRequest();
request.ParseQueryString(modelTrackingIssue.SearchQuery);
var timelineQuery = request.Filter(Context.ModelTimelineIssues)
.Where(x =>
x.ModelBuildId == modelBuildAttempt.ModelBuild.Id &&
x.Attempt == modelBuildAttempt.Attempt);
var any = false;
foreach (var modelTimelineIssue in await timelineQuery.ToListAsync().ConfigureAwait(false))
{
var modelMatch = new ModelTrackingIssueMatch()
{
ModelTrackingIssue = modelTrackingIssue,
ModelBuildAttempt = modelBuildAttempt,
ModelTimelineIssue = modelTimelineIssue,
JobName = modelTimelineIssue.JobName,
};
Context.ModelTrackingIssueMatches.Add(modelMatch);
any = true;
}
return(any);
}
/// <summary>
/// There are times when the AzDO API will not provide a Timeline for a Bulid. In those cases we still need to create
/// a <see cref="ModelBuildAttempt"/> entry. The assumption is this is for the first attempt and all of the values will
/// be
/// </summary>
/// <param name="modelBuild"></param>
/// <param name="build"></param>
/// <returns></returns>
public async Task <ModelBuildAttempt> EnsureBuildAttemptWithoutTimelineAsync(ModelBuild modelBuild, Build build)
{
const int attempt = 1;
var modelBuildAttempt = await Context.ModelBuildAttempts
.Where(x => x.ModelBuildId == modelBuild.Id && x.Attempt == attempt)
.FirstOrDefaultAsync().ConfigureAwait(false);
if (modelBuildAttempt is object)
{
return(modelBuildAttempt);
}
modelBuildAttempt = new ModelBuildAttempt()
{
Attempt = attempt,
BuildResult = build.Result,
StartTime = modelBuild.StartTime,
FinishTime = modelBuild.FinishTime,
ModelBuild = modelBuild,
IsTimelineMissing = false,
};
Context.ModelBuildAttempts.Add(modelBuildAttempt);
await Context.SaveChangesAsync().ConfigureAwait(false);
return(modelBuildAttempt);
}
private async Task <bool> TriageTestAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
Debug.Assert(modelBuildAttempt.ModelBuild is object);
Debug.Assert(modelTrackingIssue.IsActive);
Debug.Assert(modelTrackingIssue.TrackingKind == TrackingKind.Test);
Debug.Assert(modelTrackingIssue.SearchQuery is object);
var request = new SearchTestsRequest();
request.ParseQueryString(modelTrackingIssue.SearchQuery);
IQueryable <ModelTestResult> testQuery = request
.Filter(Context.ModelTestResults)
.Where(x =>
x.ModelBuildId == modelBuildAttempt.ModelBuildId &&
x.ModelTestRun.Attempt == modelBuildAttempt.Attempt)
.Include(x => x.ModelTestRun);
var any = false;
foreach (var testResult in await testQuery.ToListAsync().ConfigureAwait(false))
{
var modelMatch = new ModelTrackingIssueMatch()
{
ModelTrackingIssue = modelTrackingIssue,
ModelBuildAttempt = modelBuildAttempt,
ModelTestResult = testResult,
JobName = testResult.ModelTestRun.Name,
};
Context.ModelTrackingIssueMatches.Add(modelMatch);
any = true;
}
return(any);
}
private async Task <bool> TriageHelixLogsAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
Debug.Assert(modelBuildAttempt.ModelBuild is object);
Debug.Assert(modelTrackingIssue.IsActive);
Debug.Assert(modelTrackingIssue.SearchQuery is object);
var request = new SearchHelixLogsRequest()
{
Limit = 100,
};
request.ParseQueryString(modelTrackingIssue.SearchQuery);
var query = request.Filter(Context.ModelTestResults)
.Where(x => x.ModelBuild.Id == modelBuildAttempt.ModelBuild.Id && x.ModelTestRun.Attempt == modelBuildAttempt.Attempt);
// TODO: selecting a lot of info here. Can improve perf by selecting only the needed
// columns. The search helix logs page already optimizes this. Consider factoring out
// the shared code.
var testResultList = await query.ToListAsync().ConfigureAwait(false);
var buildInfo = modelBuildAttempt.ModelBuild.GetBuildInfo();
var helixLogInfos = testResultList
.Select(x => x.GetHelixLogInfo())
.SelectNotNull()
.Select(x => (buildInfo, x));
var results = await HelixServer.SearchHelixLogsAsync(
helixLogInfos,
request,
onError : x => Logger.LogWarning(x.Message)).ConfigureAwait(false);
var any = false;
foreach (var result in results)
{
any = true;
var modelMatch = new ModelTrackingIssueMatch()
{
ModelBuildAttempt = modelBuildAttempt,
ModelTrackingIssue = modelTrackingIssue,
HelixLogKind = result.HelixLogKind,
HelixLogUri = result.HelixLogUri,
};
Context.ModelTrackingIssueMatches.Add(modelMatch);
}
return(any);
}
public async Task TriageAsync(ModelBuildAttempt modelBuildAttempt)
{
if (modelBuildAttempt.ModelBuild is null)
{
throw new Exception("The attempt must include the build");
}
Logger.LogInformation($"Triaging {modelBuildAttempt.ModelBuild.GetBuildResultInfo().BuildUri}");
var trackingIssues = await(Context
.ModelTrackingIssues
.Where(x => x.IsActive && (x.ModelBuildDefinition == null || x.ModelBuildDefinition.Id == modelBuildAttempt.ModelBuild.ModelBuildDefinitionId))
.ToListAsync()).ConfigureAwait(false);
foreach (var trackingIssue in trackingIssues)
{
await TriageAsync(modelBuildAttempt, trackingIssue).ConfigureAwait(false);
}
}
public static BuildAttemptKey GetBuildAttemptKey(this ModelBuildAttempt modelBuildAttempt) =>
new BuildAttemptKey(
modelBuildAttempt.ModelBuild.GetBuildKey(),
modelBuildAttempt.Attempt);
public async Task <ModelBuildAttempt> EnsureBuildAttemptAsync(ModelBuild modelBuild, BuildResult buildResult, Timeline timeline)
{
var attempt = timeline.GetAttempt();
var modelBuildAttempt = await Context.ModelBuildAttempts
.Where(x => x.ModelBuildId == modelBuild.Id && x.Attempt == attempt)
.FirstOrDefaultAsync().ConfigureAwait(false);
if (modelBuildAttempt is object && !modelBuildAttempt.IsTimelineMissing)
{
return(modelBuildAttempt);
}
var startTimeQuery = timeline
.Records
.Where(x => x.Attempt == attempt)
.Select(x => DevOpsUtil.ConvertFromRestTime(x.StartTime))
.SelectNullableValue()
.Select(x => (DateTime?)x.DateTime);
var startTime = startTimeQuery.Any()
? startTimeQuery.Min()
: modelBuild.StartTime;
var finishTimeQuery = timeline
.Records
.Where(x => x.Attempt == attempt)
.Select(x => DevOpsUtil.ConvertFromRestTime(x.FinishTime))
.SelectNullableValue()
.Select(x => (DateTime?)x.DateTime);
var finishTime = finishTimeQuery.Any()
? finishTimeQuery.Max()
: modelBuild.FinishTime;
if (modelBuildAttempt is object)
{
modelBuildAttempt.BuildResult = buildResult;
modelBuildAttempt.StartTime = startTime;
modelBuildAttempt.FinishTime = finishTime;
modelBuildAttempt.IsTimelineMissing = false;
}
else
{
modelBuildAttempt = new ModelBuildAttempt()
{
Attempt = attempt,
BuildResult = buildResult,
StartTime = startTime,
FinishTime = finishTime,
ModelBuild = modelBuild,
IsTimelineMissing = false,
};
Context.ModelBuildAttempts.Add(modelBuildAttempt);
}
var timelineTree = TimelineTree.Create(timeline);
foreach (var record in timeline.Records)
{
if (record.Issues is null ||
!timelineTree.TryGetJob(record, out var job))
{
continue;
}
foreach (var issue in record.Issues)
{
var timelineIssue = new ModelTimelineIssue()
{
Attempt = attempt,
JobName = job.Name,
RecordName = record.Name,
TaskName = record.Task?.Name ?? "",
RecordId = record.Id,
Message = issue.Message,
ModelBuild = modelBuild,
IssueType = issue.Type,
ModelBuildAttempt = modelBuildAttempt,
};
Context.ModelTimelineIssues.Add(timelineIssue);
}
}
await Context.SaveChangesAsync().ConfigureAwait(false);
return(modelBuildAttempt);
}
public async Task TriageAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
if (modelBuildAttempt.ModelBuild is null)
{
throw new Exception("The attempt must include the build");
}
if (modelTrackingIssue.ModelBuildDefinitionId is { } definitionId&&
definitionId != modelBuildAttempt.ModelBuild.ModelBuildDefinitionId)
{
return;
}
// Quick spot check to avoid doing extra work if we've already triaged this attempt against this
// issue
if (await WasTriaged().ConfigureAwait(false))
{
return;
}
bool isPresent;
switch (modelTrackingIssue.TrackingKind)
{
case TrackingKind.Test:
isPresent = await TriageTestAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
case TrackingKind.Timeline:
isPresent = await TriageTimelineAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
case TrackingKind.HelixLogs:
isPresent = await TriageHelixLogsAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
#pragma warning disable 618
case TrackingKind.HelixConsole:
case TrackingKind.HelixRunClient:
// TODO: delete this once the DB is cleaned up
// These are old data types that we ignore.
isPresent = false;
break;
#pragma warning restore 618
default:
throw new Exception($"Unknown value {modelTrackingIssue.TrackingKind}");
}
var result = new ModelTrackingIssueResult()
{
ModelBuildAttempt = modelBuildAttempt,
ModelTrackingIssue = modelTrackingIssue,
IsPresent = isPresent
};
Context.ModelTrackingIssueResults.Add(result);
// This can race with other attempts to associate issues here. That is okay though because triage attempts are
// retried because they assume races with other operations can happen.
if (isPresent && modelTrackingIssue.GetGitHubIssueKey() is { } issueKey)
{
await TriageContextUtil.EnsureGitHubIssueAsync(modelBuildAttempt.ModelBuild, issueKey, saveChanges : false).ConfigureAwait(false);
}
await Context.SaveChangesAsync().ConfigureAwait(false);
async Task <bool> WasTriaged()
{
var query = Context
.ModelTrackingIssueResults
.Where(x => x.ModelBuildAttemptId == modelBuildAttempt.Id && x.ModelTrackingIssueId == modelTrackingIssue.Id);
return(await query.AnyAsync().ConfigureAwait(false));
}
}
