public Functions(DevOpsServer server, GitHubClient gitHubClient, TriageContext context)
{
Server = server;
GitHubClient = gitHubClient;
Context = context;
TriageContextUtil = new TriageContextUtil(context);
}
public async Task OnPullRequestMergedAsync(
DevOpsServer server,
TriageContextUtil triageContextUtil,
GitHubPullRequestKey prKey,
string project,
CancellationToken cancellationToken = default)
{
// Pull requests can trigger builds in multiple definitions. Need to calculate the merged PR build
// for each of them
var allBuilds = await server.ListPullRequestBuildsAsync(prKey, project).ConfigureAwait(false);
foreach (var group in allBuilds.GroupBy(x => x.Definition.Id))
{
var mergedBuild = group
.Where(x => x.Status == BuildStatus.Completed && x.Result != BuildResult.Canceled)
.OrderByDescending(x => x.Id)
.FirstOrDefault();
if (mergedBuild is object)
{
var modelBuild = await triageContextUtil.EnsureBuildAsync(mergedBuild.GetBuildResultInfo()).ConfigureAwait(false);
modelBuild.IsMergedPullRequest = true;
await triageContextUtil.Context.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
}
}
}
public ModelBuild AddBuild(string data, ModelBuildDefinition def)
{
var parts = data.Split("|");
var number = GetPartOrNull(parts, 0) is { } part?int.Parse(part) : BuildCount++;
var dt = GetPartOrNull(parts, 3);
var br = GetPartOrNull(parts, 4);
var build = new ModelBuild()
{
Id = TriageContextUtil.GetModelBuildId(new BuildKey(def.AzureOrganization, def.AzureProject, number)),
BuildNumber = number,
GitHubOrganization = GetPartOrNull(parts, 1),
GitHubRepository = GetPartOrNull(parts, 2),
AzureOrganization = def.AzureOrganization,
AzureProject = def.AzureProject,
QueueTime = dt is object?DateTime.ParseExact(dt, "yyyy-MM-dd", null) : (DateTime?)null,
BuildResult = br is object?Enum.Parse <BuildResult>(br) : (BuildResult?)null,
ModelBuildDefinition = def,
DefinitionId = def.DefinitionId,
DefinitionName = def.DefinitionName,
};
Context.ModelBuilds.Add(build);
return(build);
}
public NewTrackingIssueModel(TriageContext triageContext, FunctionQueueUtil functionQueueUtil, IGitHubClientFactory gitHubClientFactory, ILogger <NewTrackingIssueModel> logger)
{
TriageContext = triageContext;
TriageContextUtil = new TriageContextUtil(triageContext);
GitHubClientFactory = gitHubClientFactory;
FunctionQueueUtil = functionQueueUtil;
Logger = logger;
}
public Functions(DevOpsServer server, TriageContext context, GitHubClientFactory gitHubClientFactory)
{
Server = server;
Context = context;
TriageContextUtil = new TriageContextUtil(context);
GitHubClientFactory = gitHubClientFactory;
SiteLinkUtil = SiteLinkUtil.Published;
HelixServer = new HelixServer();
}
public async Task <IActionResult> OnPost(string gitHubIssueUri, string formAction)
{
if (!GitHubIssueKey.TryCreateFromUri(gitHubIssueUri, out var issueKey))
{
return(await OnError("Not a valid GitHub Issue Url"));
}
var buildKey = GetBuildKey(Number !.Value);
if (formAction == "addIssue")
{
try
{
var modelBuild = await TriageContextUtil.GetModelBuildAsync(buildKey);
await TriageContextUtil.EnsureGitHubIssueAsync(modelBuild, issueKey, saveChanges : true);
}
catch (DbUpdateException ex)
{
if (!ex.IsUniqueKeyViolation())
{
throw;
}
}
}
else
{
var query = TriageContextUtil
.GetModelBuildQuery(buildKey)
.SelectMany(x => x.ModelGitHubIssues)
.Where(x =>
x.Organization == issueKey.Organization &&
x.Repository == issueKey.Repository &&
x.Number == issueKey.Number);
var modelGitHubIssue = await query.FirstOrDefaultAsync();
if (modelGitHubIssue is object)
{
TriageContextUtil.Context.ModelGitHubIssues.Remove(modelGitHubIssue);
await TriageContextUtil.Context.SaveChangesAsync();
}
}
var util = new TrackingGitHubUtil(GitHubClientFactory, TriageContextUtil.Context, SiteLinkUtil.Published, Logger);
await util.UpdateAssociatedGitHubIssueAsync(issueKey);
return(await OnGet());
async Task <IActionResult> OnError(string message)
{
GitHubIssueAddErrorMessage = message;
return(await OnGet());
}
}
public StandardTestBase(ITestOutputHelper testOutputHelper)
{
Connection = CreateInMemoryDatabase();
var options = new DbContextOptionsBuilder <TriageContext>()
.UseSqlite(Connection)
.Options;
Context = new TriageContext(options);
TriageContextUtil = new TriageContextUtil(Context);
TestableHttpMessageHandler = new TestableHttpMessageHandler();
TestableLogger = new TestableLogger(testOutputHelper);
TestableGitHubClientFactory = new TestableGitHubClientFactory();
var httpClient = new HttpClient(TestableHttpMessageHandler);
Server = new DevOpsServer("random", httpClient: httpClient);
HelixServer = new HelixServer(httpClient: httpClient);
QueryUtil = new DotNetQueryUtil(Server);
Context.Database.EnsureDeleted();
Context.Database.EnsureCreated();
}
public async Task <IActionResult> OnPost()
{
if (TrackingKind == TrackingKind.Unknown)
{
ErrorMessage = "Invalid Tracking Kind";
return(Page());
}
if (string.IsNullOrEmpty(IssueTitle))
{
ErrorMessage = "Need an issue title";
return(Page());
}
if (IssueTitle.Length >= ModelTrackingIssue.IssueTitleLengthLimit)
{
ErrorMessage = $"Please limit issue title to {ModelTrackingIssue.IssueTitleLengthLimit} characters";
return(Page());
}
if (string.IsNullOrEmpty(SearchText))
{
ErrorMessage = "Must provide search text";
return(Page());
}
ModelBuildDefinition?modelBuildDefinition = null;
if (!string.IsNullOrEmpty(Definition))
{
modelBuildDefinition = await TriageContextUtil.FindModelBuildDefinitionAsync(Definition);
if (modelBuildDefinition is null)
{
ErrorMessage = $"Cannot find build definition with name or ID: {Definition}";
return(Page());
}
}
switch (TrackingKind)
{
#pragma warning disable 618
case TrackingKind.HelixConsole:
case TrackingKind.HelixRunClient:
ErrorMessage = $"'{TrackingKind}' is deprecated. Please use {TrackingKind.HelixLogs}";
return(Page());
case TrackingKind.HelixLogs:
{
if (TryParseQueryString <SearchHelixLogsRequest>(out var request))
{
if (request.HelixLogKinds.Count == 0)
{
ErrorMessage = "Need to specify at least one log kind to search";
return(Page());
}
}
else
{
return(Page());
}
}
break;
case TrackingKind.Test:
if (!TryParseQueryString <SearchTestsRequest>(out _))
{
return(Page());
}
break;
case TrackingKind.Timeline:
if (!TryParseQueryString <SearchTimelinesRequest>(out _))
{
return(Page());
}
break;
}
GitHubIssueKey?issueKey = null;
if (!string.IsNullOrEmpty(GitHubIssueUri))
{
if (GitHubIssueKey.TryCreateFromUri(GitHubIssueUri, out var key))
{
issueKey = key;
GitHubOrganization = key.Organization;
GitHubRepository = key.Repository;
}
else
{
ErrorMessage = $"Invalid GitHub issue link: {GitHubIssueUri}";
return(Page());
}
}
else if (string.IsNullOrEmpty(GitHubRepository) || string.IsNullOrEmpty(GitHubOrganization))
{
ErrorMessage = "Must provide GitHub organization and repository";
return(Page());
}
IGitHubClient?gitHubClient = null;
try
{
gitHubClient = await GitHubClientFactory.CreateForAppAsync(GitHubOrganization, GitHubRepository);
}
catch (Exception ex)
{
ErrorMessage = $"Cannot create GitHub client for that repository: {ex.Message}";
return(Page());
}
var modelTrackingIssue = await CreateTrackingIssue(gitHubClient);
return(RedirectToPage(
"./Issue",
new { id = modelTrackingIssue.Id }));
async Task <ModelTrackingIssue> CreateTrackingIssue(IGitHubClient gitHubClient)
{
var issueKey = await GetOrCreateGitHubIssueAsync(gitHubClient);
var modelTrackingIssue = new ModelTrackingIssue()
{
IsActive = true,
IssueTitle = IssueTitle,
TrackingKind = TrackingKind,
SearchQuery = SearchText,
ModelBuildDefinition = modelBuildDefinition,
GitHubOrganization = issueKey.Organization,
GitHubRepository = issueKey.Repository,
GitHubIssueNumber = issueKey.Number,
};
TriageContext.ModelTrackingIssues.Add(modelTrackingIssue);
await TriageContext.SaveChangesAsync();
await InitialTriageAsync(modelTrackingIssue);
return(modelTrackingIssue);
}
async Task InitialTriageAsync(ModelTrackingIssue modelTrackingIssue)
{
if (modelBuildDefinition is object)
{
// The initial triage is only done for tracking issues that have definitions
// associated with. Lacking a definition we end up querying all builds and that
// can produce a *lot* of data. Hard to find a good formula that is performant
// there hence we limit to only builds with definitions.
var request = new SearchBuildsRequest()
{
Definition = modelBuildDefinition?.DefinitionId.ToString() ?? null,
Queued = new DateRequestValue(7, RelationalKind.GreaterThan),
Result = new BuildResultRequestValue(BuildResult.Succeeded, EqualsKind.NotEquals),
};
await FunctionQueueUtil.QueueTriageBuildAttempts(TriageContextUtil, modelTrackingIssue, request);
// Issues are bulk updated on a 15 minute cycle. This is a new issue though so want to make sure that
// the user sees progress soon. Schedule two manual updates in the near future on this so the issue
// gets rolling then it will fall into the 15 minute bulk cycle.
await FunctionQueueUtil.QueueUpdateIssueAsync(modelTrackingIssue, TimeSpan.FromSeconds(30));
await FunctionQueueUtil.QueueUpdateIssueAsync(modelTrackingIssue, TimeSpan.FromMinutes(2));
}
else
{
await FunctionQueueUtil.QueueUpdateIssueAsync(modelTrackingIssue, TimeSpan.FromMinutes(0));
}
}
async Task <GitHubIssueKey> GetOrCreateGitHubIssueAsync(IGitHubClient gitHubClient)
{
if (issueKey is { } key)
{
await TrackingGitHubUtil.EnsureGitHubIssueHasMarkers(gitHubClient, key);
return(key);
}
var newIssue = new NewIssue(IssueTitle)
{
Body = TrackingGitHubUtil.WrapInStartEndMarkers("Runfo Creating Tracking Issue (data being generated)")
};
var issue = await gitHubClient.Issue.Create(GitHubOrganization, GitHubRepository, newIssue);
return(issue.GetIssueKey());
}
bool TryParseQueryString <T>(out T value)
where T : ISearchRequest, new()
{
value = new T();
try
{
value.ParseQueryString(SearchText ?? "");
return(true);
}
catch (Exception ex)
{
ErrorMessage = ex.ToString();
return(false);
}
}
}
/// <summary>
/// This function will queue up a number of <see cref="ModelBuildAttempt"/> instances to triage against the specified
/// <see cref="ModelTrackingIssue"/>. This is useful to essentially seed old builds against a given tracking
/// issue (aka populate the data set) while at the same time new builds will be showing up via normal completion.
/// It will return the number of attempts that were queued for processing,
///
/// One particular challenge we have to keep in mind is that this is going to be queueing up a lot of builds
/// into our Azure functions. Those will scale to whatever data we put into there. Need to be mindful to not
/// queue up say 100,000 builds as that will end up spiking all our resources. Have to put some throttling
/// in here.
/// </summary>
public static async Task <int> QueueTriageBuildAttempts(
this FunctionQueueUtil util,
TriageContextUtil triageContextUtil,
ModelTrackingIssue trackingIssue,
SearchBuildsRequest buildsRequest,
int limit = 200)
{
// Need to filter to a bulid definition other wise there is no reasonable way to filter the builds. Any
// triage is basically pointless.
if (trackingIssue.ModelBuildDefinition is null && !buildsRequest.HasDefinition)
{
throw new Exception("Must filter to a build definition");
}
// Ensure there is some level of filtering occuring here.
if (buildsRequest.Result is null)
{
buildsRequest.Result = new BuildResultRequestValue(BuildResult.Succeeded, EqualsKind.NotEquals);
}
if (buildsRequest.Queued is null && buildsRequest.Started is null && buildsRequest.Finished is null)
{
buildsRequest.Queued = new DateRequestValue(7, RelationalKind.GreaterThan);
}
var buildsQuery = buildsRequest
.Filter(triageContextUtil.Context.ModelBuilds)
.OrderByDescending(x => x.BuildNumber)
.SelectMany(x => x.ModelBuildAttempts)
.Select(x => new
{
x.ModelBuild.BuildNumber,
x.ModelBuild.AzureOrganization,
x.ModelBuild.AzureProject,
x.Attempt,
x.Id
});
var existingAttemptsQuery = triageContextUtil
.Context
.ModelTrackingIssueResults
.Where(x => x.ModelTrackingIssueId == trackingIssue.Id)
.Include(x => x.ModelBuildAttempt)
.ThenInclude(x => x.ModelBuild)
.Select(x => new
{
x.ModelBuildAttempt.ModelBuild.AzureOrganization,
x.ModelBuildAttempt.ModelBuild.AzureProject,
x.ModelBuildAttempt.ModelBuild.BuildNumber,
x.ModelBuildAttempt.Attempt,
});
var existingAttemptsResults = await existingAttemptsQuery.ToListAsync();
var existingAttemptsSet = new HashSet <BuildAttemptKey>(
existingAttemptsResults.Select(x => new BuildAttemptKey(x.AzureOrganization, x.AzureProject, x.BuildNumber, x.Attempt)));
var attempts = await buildsQuery.ToListAsync();
var attemptKeys = attempts
.Select(x => new BuildAttemptKey(x.AzureOrganization, x.AzureProject, x.BuildNumber, x.Attempt))
.Where(x => !existingAttemptsSet.Contains(x))
.Take(limit)
.ToList();
await util.QueueTriageBuildAttemptsAsync(trackingIssue, attemptKeys);
return(attemptKeys.Count);
}
public BuildLogsModel(TriageContextUtil triageContextUtil, DotNetQueryUtilFactory factory)
{
TriageContextUtil = triageContextUtil;
DotNetQueryUtilFactory = factory;
}
public HelixLogsModel(TriageContextUtil triageContextUtil)
{
TriageContextUtil = triageContextUtil;
}
public async Task OnGet()
{
ErrorMessage = null;
if (string.IsNullOrEmpty(Query))
{
Query = new SearchBuildsRequest()
{
Started = new DateRequestValue(dayQuery: 5),
BuildType = new BuildTypeRequestValue(ModelBuildKind.PullRequest, EqualsKind.NotEquals, "pr"),
}.GetQueryString();
}
if (string.IsNullOrEmpty(Definition))
{
Definition = "roslyn-ci";
return;
}
var modelBuildDefinition = await FindDefinitionAsync(Definition);
if (modelBuildDefinition is null)
{
ErrorMessage = $"Could not find definition for {Definition}";
return;
}
var searchBuildsRequest = new SearchBuildsRequest();
searchBuildsRequest.ParseQueryString(Query);
searchBuildsRequest.Definition = null;
await PopulateBuildInfoAsync();
await PopulateJobData();
HasResults = true;
async Task PopulateBuildInfoAsync()
{
var buildQuery = TriageContextUtil.Context.ModelBuilds
.Where(x => x.ModelBuildDefinitionId == modelBuildDefinition.Id && x.BuildResult.HasValue);
var builds = await searchBuildsRequest.Filter(buildQuery)
.Select(x => new
{
x.BuildResult,
x.IsMergedPullRequest,
x.PullRequestNumber,
})
.ToListAsync();
BuildCount = builds.Count;
RollingPassRate = GetPassRate(ModelBuildKind.Rolling);
MergedPullRequestPassRate = GetPassRate(ModelBuildKind.MergedPullRequest);
var totalPassRate = (double)(builds.Count(x => IsSuccess(x.BuildResult !.Value))) / builds.Count;
TotalPassRate = totalPassRate.ToString("P2");
string?GetPassRate(ModelBuildKind kind)
{
var filtered = builds
.Where(x => TriageContextUtil.GetModelBuildKind(x.IsMergedPullRequest, x.PullRequestNumber) == kind)
.ToList();
if (filtered.Count == 0)
{
return(null);
}
var count = filtered.Count(x => IsSuccess(x.BuildResult !.Value));
var rate = (double)count / filtered.Count;
return(rate.ToString("P2"));
}
}
async Task PopulateJobData()
{
IQueryable <ModelTimelineIssue> query = TriageContextUtil.Context
.ModelTimelineIssues
.Where(x => x.IssueType == IssueType.Error && x.ModelBuild.ModelBuildDefinitionId == modelBuildDefinition.Id && x.ModelBuild.BuildResult.HasValue);
query = searchBuildsRequest.Filter(query);
const int limit = 1_000;
var issues = await query
.Select(x => new
{
x.JobName,
x.ModelBuild.BuildResult,
x.ModelBuild.IsMergedPullRequest,
x.ModelBuild.PullRequestNumber
})
.Take(limit)
.ToListAsync();
if (issues.Count == limit)
{
JobLimitHit = limit;
}
searchBuildsRequest.Definition = Definition;
foreach (var group in issues.GroupBy(x => x.JobName))
{
var request = new SearchTimelinesRequest()
{
JobName = group.Key,
Type = IssueType.Error
};
var data = new JobData()
{
JobName = group.Key,
MergedPullRequestFailureCount = GetFailureCount(ModelBuildKind.MergedPullRequest),
RollingFailureCount = GetFailureCount(ModelBuildKind.Rolling),
TotalFailureCount = GetFailureCount(null),
JobPageBuildQuery = searchBuildsRequest.GetQueryString(),
JobPageTimelineQuery = request.GetQueryString(),
};
JobDataList.Add(data);
int GetFailureCount(ModelBuildKind?kind) =>
kind is
{
}
k
? group.Where(x => TriageContextUtil.GetModelBuildKind(x.IsMergedPullRequest, x.PullRequestNumber) == k).Count()
: group.Count();
}
searchBuildsRequest.Definition = null;
JobDataList.Sort((x, y) => y.TotalFailureCount - x.TotalFailureCount);
}
public DefinitionModel(TriageContextUtil triageContextUtil)
{
TriageContextUtil = triageContextUtil;
}
public MergedPullRequestsModel(TriageContextUtil triageContextUtil)
{
TriageContextUtil = triageContextUtil;
}
public TrackingIssueModel(TriageContextUtil triageContextUtil, FunctionQueueUtil functionQueueUtil, IGitHubClientFactory gitHubClientFactory)
{
TriageContextUtil = triageContextUtil;
FunctionQueueUtil = functionQueueUtil;
GitHubClientFactory = gitHubClientFactory;
}
public BuildModel(TriageContextUtil triageContextUtil, IGitHubClientFactory gitHubClientFactory, ILogger <BuildModel> logger)
{
TriageContextUtil = triageContextUtil;
GitHubClientFactory = gitHubClientFactory;
Logger = logger;
}
public async Task <IActionResult> OnGet()
{
if (!(Number is { } number))
{
return(Page());
}
var buildKey = GetBuildKey(number);
var project = buildKey.Project;
var organization = buildKey.Organization;
var buildId = TriageContextUtil.GetModelBuildId(buildKey);
var modelBuild = await PopulateBuildInfo();
await PopulateTimeline();
await PopulateTests();
return(Page());
async Task <ModelBuild?> PopulateBuildInfo()
{
var modelBuild = await TriageContextUtil
.GetModelBuildQuery(buildKey)
.Include(x => x.ModelGitHubIssues)
.FirstOrDefaultAsync();
if (modelBuild is null)
{
return(null);
}
BuildUri = DevOpsUtil.GetBuildUri(organization, project, number);
BuildResult = modelBuild.BuildResult ?? BuildResult.None;
Repository = $"{modelBuild.GitHubOrganization}/{modelBuild.GitHubRepository}";
RepositoryUri = $"https://{modelBuild.GitHubOrganization}/{modelBuild.GitHubRepository}";
DefinitionName = modelBuild.DefinitionName;
TargetBranch = modelBuild.GitHubTargetBranch;
GitHubIssues.Clear();
GitHubIssues.AddRange(modelBuild.ModelGitHubIssues.Select(x => x.GetGitHubIssueKey()));
if (modelBuild.PullRequestNumber is { } prNumber)
{
PullRequestKey = new GitHubPullRequestKey(
modelBuild.GitHubOrganization,
modelBuild.GitHubRepository,
prNumber);
}
return(modelBuild);
}
async Task PopulateTimeline()
{
var query = TriageContextUtil
.Context
.ModelTimelineIssues
.Where(x => x.ModelBuildId == buildId)
.Include(x => x.ModelBuild);
TimelineIssuesDisplay = await TimelineIssuesDisplay.Create(
query,
includeBuildColumn : false,
includeIssueTypeColumn : true,
includeAttemptColumn : true);
Attempts = TimelineIssuesDisplay.Issues.Count > 0
? TimelineIssuesDisplay.Issues.Max(x => x.Attempt)
: 1;
}
async Task PopulateTests()
{
var query = TriageContextUtil
.Context
.ModelTestResults
.Where(x => x.ModelBuildId == buildId)
.Include(x => x.ModelTestRun)
.Include(x => x.ModelBuild);
var modelTestResults = await query.ToListAsync();
TestResultsDisplay = new TestResultsDisplay(modelTestResults)
{
IncludeBuildColumn = false,
IncludeBuildKindColumn = false,
IncludeTestFullNameColumn = true,
IncludeTestFullNameLinks = true,
IncludeErrorMessageColumn = true,
};
if (modelBuild is object)
{
TestResultsDisplay.BuildsRequest = new SearchBuildsRequest()
{
Definition = modelBuild.DefinitionName,
Started = new DateRequestValue(dayQuery: 7)
};
}
}
}
public RunfoController(TriageContextUtil triageContextUtil)
{
TriageContextUtil = triageContextUtil;
}
public TestsModel(TriageContextUtil triageContextUtil, IGitHubClientFactory gitHubClientFactory)
{
TriageContextUtil = triageContextUtil;
GitHubClientFactory = gitHubClientFactory;
}
public TimelinesModel(TriageContextUtil triageContextUtil)
{
TriageContextUtil = triageContextUtil;
}
