public async Task <string> GetTrackingIssueReport(
ModelTrackingIssue modelTrackingIssue,
int limit = DefaultReportLimit,
bool includeMarkers = true,
DateTime?baseTime = null)
{
var time = baseTime ?? DateTime.UtcNow;
var reportTextTask = modelTrackingIssue.TrackingKind switch
{
TrackingKind.Test => GetReportForTestAsync(modelTrackingIssue, limit, time),
TrackingKind.Timeline => GetReportForTimelineAsync(modelTrackingIssue, limit, time),
TrackingKind.HelixLogs => GetReportForHelixAsync(modelTrackingIssue, limit),
#pragma warning disable 618
// TODO: delete once these types are removed from the DB
TrackingKind.HelixConsole => throw null !,
TrackingKind.HelixRunClient => throw null !,
#pragma warning restore 618
_ => throw new Exception($"Invalid value {modelTrackingIssue.TrackingKind}"),
};
var reportText = await reportTextTask.ConfigureAwait(false);
return(includeMarkers
? WrapInStartEndMarkers(reportText)
: reportText);
}
public async Task UpdateTrackingGitHubIssueAsync(ModelTrackingIssue modelTrackingIssue)
{
if (modelTrackingIssue.GetGitHubIssueKey() is { } issueKey)
{
await UpdateTrackingGitHubIssueAsync(modelTrackingIssue, issueKey).ConfigureAwait(false);
}
}
private async Task <string> GetReportForHelixAsync(ModelTrackingIssue modelTrackingIssue, int limit)
{
Debug.Assert(modelTrackingIssue.TrackingKind == TrackingKind.HelixLogs);
var matches = await Context
.ModelTrackingIssueMatches
.Where(x => x.ModelTrackingIssueId == modelTrackingIssue.Id)
.OrderByDescending(x => x.ModelBuildAttempt.ModelBuild.BuildNumber)
.Take(limit)
.Select(x => new
{
AzureOrganization = x.ModelBuildAttempt.ModelBuild.AzureOrganization,
AzureProject = x.ModelBuildAttempt.ModelBuild.AzureProject,
DefinitionId = x.ModelBuildAttempt.ModelBuild.DefinitionId,
DefinitionName = x.ModelBuildAttempt.ModelBuild.DefinitionName,
GitHubOrganization = x.ModelBuildAttempt.ModelBuild.GitHubOrganization,
GitHubRepository = x.ModelBuildAttempt.ModelBuild.GitHubRepository,
GitHubPullRequestNumber = x.ModelBuildAttempt.ModelBuild.PullRequestNumber,
GitHubTargetBranch = x.ModelBuildAttempt.ModelBuild.GitHubTargetBranch,
BuildNumber = x.ModelBuildAttempt.ModelBuild.BuildNumber,
HelixLogKind = x.HelixLogKind,
HelixLogUri = x.HelixLogUri,
})
.ToListAsync().ConfigureAwait(false);
var map = new Dictionary <BuildKey, (BuildInfo BuildInfo, HelixLogInfo?HelixLogInfo)>();
var set = new HashSet <HelixLogKind>();
foreach (var item in matches)
{
var key = new BuildKey(item.AzureOrganization, item.AzureProject, item.BuildNumber);
if (!map.TryGetValue(key, out var tuple))
{
var buildInfo = new BuildInfo(
key.Organization,
key.Project,
key.Number,
new GitHubBuildInfo(item.GitHubOrganization, item.GitHubRepository, item.GitHubPullRequestNumber, item.GitHubTargetBranch));
tuple = (buildInfo, null);
}
set.Add(item.HelixLogKind);
tuple.HelixLogInfo = tuple.HelixLogInfo is { } log
? log.SetUri(item.HelixLogKind, item.HelixLogUri)
: new HelixLogInfo(item.HelixLogKind, item.HelixLogUri);
map[key] = tuple;
}
var reportBuilder = new ReportBuilder();
return(reportBuilder.BuildSearchHelix(
map.Values.OrderByDescending(x => x.BuildInfo.Number),
set.ToArray(),
markdown: true));
}
private async Task <string> GetReportForTestAsync(ModelTrackingIssue modelTrackingIssue, int limit, DateTime baseTime)
{
Debug.Assert(modelTrackingIssue.TrackingKind == TrackingKind.Test);
var matches = await Context
.ModelTrackingIssueMatches
.Where(x => x.ModelTrackingIssueId == modelTrackingIssue.Id)
.Select(x => new
{
AzureOrganization = x.ModelBuildAttempt.ModelBuild.AzureOrganization,
AzureProject = x.ModelBuildAttempt.ModelBuild.AzureProject,
DefinitionId = x.ModelBuildAttempt.ModelBuild.DefinitionId,
DefinitionName = x.ModelBuildAttempt.ModelBuild.DefinitionName,
GitHubOrganization = x.ModelBuildAttempt.ModelBuild.GitHubOrganization,
GitHubRepository = x.ModelBuildAttempt.ModelBuild.GitHubRepository,
GitHubPullRequestNumber = x.ModelBuildAttempt.ModelBuild.PullRequestNumber,
GitHubTargetBranch = x.ModelBuildAttempt.ModelBuild.GitHubTargetBranch,
BuildNumber = x.ModelBuildAttempt.ModelBuild.BuildNumber,
QueueTime = x.ModelBuildAttempt.ModelBuild.QueueTime,
TestRunName = x.ModelTestResult.ModelTestRun.Name,
TestResult = x.ModelTestResult,
})
.ToListAsync().ConfigureAwait(false);
var reportBuilder = new ReportBuilder();
var reportBody = reportBuilder.BuildSearchTests(matches
.OrderByDescending(x => x.BuildNumber)
.Take(limit)
.Select(x => (
new BuildAndDefinitionInfo(
x.AzureOrganization,
x.AzureProject,
x.BuildNumber,
x.DefinitionId,
x.DefinitionName,
new GitHubBuildInfo(x.GitHubOrganization, x.GitHubRepository, x.GitHubPullRequestNumber, x.GitHubTargetBranch)),
(string?)x.TestRunName,
x.TestResult.GetHelixLogInfo())),
includeDefinition: true,
includeHelix: matches.Any(x => x.TestResult.IsHelixTestResult));
var builder = new StringBuilder();
AppendHeader(builder, modelTrackingIssue);
builder.AppendLine(reportBody);
if (matches.Count > limit)
{
builder.AppendLine($"Displaying {limit} of {matches.Count} results");
}
AppendFooter(builder, matches.Select(x => (x.BuildNumber, x.QueueTime)), baseTime);
return(builder.ToString());
}
private async Task <bool> UpdateTrackingGitHubIssueAsync(ModelTrackingIssue modelTrackingIssue, GitHubIssueKey issueKey)
{
IGitHubClient?gitHubClient = await TryCreateForIssue(issueKey).ConfigureAwait(false);
if (gitHubClient is null)
{
return(false);
}
var report = await GetTrackingIssueReport(modelTrackingIssue).ConfigureAwait(false);
var succeeded = await UpdateGitHubIssueReport(gitHubClient, issueKey, report).ConfigureAwait(false);
Logger.LogInformation($"Updated {issueKey.IssueUri}");
return(succeeded);
}
private async Task <string> GetReportForTimelineAsync(ModelTrackingIssue modelTrackingIssue, int limit, DateTime baseTime)
{
Debug.Assert(modelTrackingIssue.TrackingKind == TrackingKind.Timeline);
var matches = await Context
.ModelTrackingIssueMatches
.Where(x => x.ModelTrackingIssueId == modelTrackingIssue.Id)
.OrderByDescending(x => x.ModelBuildAttempt.ModelBuild.BuildNumber)
.Take(limit)
.Select(x => new
{
AzureOrganization = x.ModelBuildAttempt.ModelBuild.AzureOrganization,
AzureProject = x.ModelBuildAttempt.ModelBuild.AzureProject,
DefinitionId = x.ModelBuildAttempt.ModelBuild.DefinitionId,
DefinitionName = x.ModelBuildAttempt.ModelBuild.DefinitionName,
GitHubOrganization = x.ModelBuildAttempt.ModelBuild.GitHubOrganization,
GitHubRepository = x.ModelBuildAttempt.ModelBuild.GitHubRepository,
GitHubPullRequestNumber = x.ModelBuildAttempt.ModelBuild.PullRequestNumber,
GitHubTargetBranch = x.ModelBuildAttempt.ModelBuild.GitHubTargetBranch,
BuildNumber = x.ModelBuildAttempt.ModelBuild.BuildNumber,
QueueTime = x.ModelBuildAttempt.ModelBuild.QueueTime,
TimelineIssue = x.ModelTimelineIssue
})
.ToListAsync().ConfigureAwait(false);
var reportBuilder = new ReportBuilder();
var reportBody = reportBuilder.BuildSearchTimeline(
matches.Select(x => (
new BuildAndDefinitionInfo(
x.AzureOrganization,
x.AzureProject,
x.BuildNumber,
x.DefinitionId,
x.DefinitionName,
new GitHubBuildInfo(x.GitHubOrganization, x.GitHubRepository, x.GitHubPullRequestNumber, x.GitHubTargetBranch)),
x.TimelineIssue?.JobName)),
markdown: true,
includeDefinition: true);
var builder = new StringBuilder();
AppendHeader(builder, modelTrackingIssue);
builder.AppendLine(reportBody);
AppendFooter(builder, matches.Select(x => (x.BuildNumber, x.QueueTime)), baseTime);
return(builder.ToString());
}
private void AppendHeader(StringBuilder builder, ModelTrackingIssue modelTrackingIssue)
{
builder.AppendLine($"Runfo Tracking Issue: [{modelTrackingIssue.IssueTitle}]({SiteLinkUtil.GetTrackingIssueUri(modelTrackingIssue.Id)})");
}
private async Task <bool> TriageHelixLogsAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
Debug.Assert(modelBuildAttempt.ModelBuild is object);
Debug.Assert(modelTrackingIssue.IsActive);
Debug.Assert(modelTrackingIssue.SearchQuery is object);
var request = new SearchHelixLogsRequest()
{
Limit = 100,
};
request.ParseQueryString(modelTrackingIssue.SearchQuery);
var query = request.Filter(Context.ModelTestResults)
.Where(x => x.ModelBuild.Id == modelBuildAttempt.ModelBuild.Id && x.ModelTestRun.Attempt == modelBuildAttempt.Attempt);
// TODO: selecting a lot of info here. Can improve perf by selecting only the needed
// columns. The search helix logs page already optimizes this. Consider factoring out
// the shared code.
var testResultList = await query.ToListAsync().ConfigureAwait(false);
var buildInfo = modelBuildAttempt.ModelBuild.GetBuildInfo();
var helixLogInfos = testResultList
.Select(x => x.GetHelixLogInfo())
.SelectNotNull()
.Select(x => (buildInfo, x));
var results = await HelixServer.SearchHelixLogsAsync(
helixLogInfos,
request,
onError : x => Logger.LogWarning(x.Message)).ConfigureAwait(false);
var any = false;
foreach (var result in results)
{
any = true;
var modelMatch = new ModelTrackingIssueMatch()
{
ModelBuildAttempt = modelBuildAttempt,
ModelTrackingIssue = modelTrackingIssue,
HelixLogKind = result.HelixLogKind,
HelixLogUri = result.HelixLogUri,
};
Context.ModelTrackingIssueMatches.Add(modelMatch);
}
return(any);
}
private async Task <bool> TriageTimelineAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
Debug.Assert(modelBuildAttempt.ModelBuild is object);
Debug.Assert(modelTrackingIssue.IsActive);
Debug.Assert(modelTrackingIssue.TrackingKind == TrackingKind.Timeline);
Debug.Assert(modelTrackingIssue.SearchQuery is object);
var request = new SearchTimelinesRequest();
request.ParseQueryString(modelTrackingIssue.SearchQuery);
var timelineQuery = request.Filter(Context.ModelTimelineIssues)
.Where(x =>
x.ModelBuildId == modelBuildAttempt.ModelBuild.Id &&
x.Attempt == modelBuildAttempt.Attempt);
var any = false;
foreach (var modelTimelineIssue in await timelineQuery.ToListAsync().ConfigureAwait(false))
{
var modelMatch = new ModelTrackingIssueMatch()
{
ModelTrackingIssue = modelTrackingIssue,
ModelBuildAttempt = modelBuildAttempt,
ModelTimelineIssue = modelTimelineIssue,
JobName = modelTimelineIssue.JobName,
};
Context.ModelTrackingIssueMatches.Add(modelMatch);
any = true;
}
return(any);
}
private async Task <bool> TriageTestAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
Debug.Assert(modelBuildAttempt.ModelBuild is object);
Debug.Assert(modelTrackingIssue.IsActive);
Debug.Assert(modelTrackingIssue.TrackingKind == TrackingKind.Test);
Debug.Assert(modelTrackingIssue.SearchQuery is object);
var request = new SearchTestsRequest();
request.ParseQueryString(modelTrackingIssue.SearchQuery);
IQueryable <ModelTestResult> testQuery = request
.Filter(Context.ModelTestResults)
.Where(x =>
x.ModelBuildId == modelBuildAttempt.ModelBuildId &&
x.ModelTestRun.Attempt == modelBuildAttempt.Attempt)
.Include(x => x.ModelTestRun);
var any = false;
foreach (var testResult in await testQuery.ToListAsync().ConfigureAwait(false))
{
var modelMatch = new ModelTrackingIssueMatch()
{
ModelTrackingIssue = modelTrackingIssue,
ModelBuildAttempt = modelBuildAttempt,
ModelTestResult = testResult,
JobName = testResult.ModelTestRun.Name,
};
Context.ModelTrackingIssueMatches.Add(modelMatch);
any = true;
}
return(any);
}
public async Task TriageAsync(ModelBuildAttempt modelBuildAttempt, ModelTrackingIssue modelTrackingIssue)
{
if (modelBuildAttempt.ModelBuild is null)
{
throw new Exception("The attempt must include the build");
}
if (modelTrackingIssue.ModelBuildDefinitionId is { } definitionId&&
definitionId != modelBuildAttempt.ModelBuild.ModelBuildDefinitionId)
{
return;
}
// Quick spot check to avoid doing extra work if we've already triaged this attempt against this
// issue
if (await WasTriaged().ConfigureAwait(false))
{
return;
}
bool isPresent;
switch (modelTrackingIssue.TrackingKind)
{
case TrackingKind.Test:
isPresent = await TriageTestAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
case TrackingKind.Timeline:
isPresent = await TriageTimelineAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
case TrackingKind.HelixLogs:
isPresent = await TriageHelixLogsAsync(modelBuildAttempt, modelTrackingIssue).ConfigureAwait(false);
break;
#pragma warning disable 618
case TrackingKind.HelixConsole:
case TrackingKind.HelixRunClient:
// TODO: delete this once the DB is cleaned up
// These are old data types that we ignore.
isPresent = false;
break;
#pragma warning restore 618
default:
throw new Exception($"Unknown value {modelTrackingIssue.TrackingKind}");
}
var result = new ModelTrackingIssueResult()
{
ModelBuildAttempt = modelBuildAttempt,
ModelTrackingIssue = modelTrackingIssue,
IsPresent = isPresent
};
Context.ModelTrackingIssueResults.Add(result);
// This can race with other attempts to associate issues here. That is okay though because triage attempts are
// retried because they assume races with other operations can happen.
if (isPresent && modelTrackingIssue.GetGitHubIssueKey() is { } issueKey)
{
await TriageContextUtil.EnsureGitHubIssueAsync(modelBuildAttempt.ModelBuild, issueKey, saveChanges : false).ConfigureAwait(false);
}
await Context.SaveChangesAsync().ConfigureAwait(false);
async Task <bool> WasTriaged()
{
var query = Context
.ModelTrackingIssueResults
.Where(x => x.ModelBuildAttemptId == modelBuildAttempt.Id && x.ModelTrackingIssueId == modelTrackingIssue.Id);
return(await query.AnyAsync().ConfigureAwait(false));
}
}
