public NewTrackingIssueModel(TriageContext triageContext, FunctionQueueUtil functionQueueUtil, IGitHubClientFactory gitHubClientFactory, ILogger <NewTrackingIssueModel> logger) { TriageContext = triageContext; TriageContextUtil = new TriageContextUtil(triageContext); GitHubClientFactory = gitHubClientFactory; FunctionQueueUtil = functionQueueUtil; Logger = logger; }
public async Task <IActionResult> OnPost() { if (TrackingKind == TrackingKind.Unknown) { ErrorMessage = "Invalid Tracking Kind"; return(Page()); } if (string.IsNullOrEmpty(IssueTitle)) { ErrorMessage = "Need an issue title"; return(Page()); } if (IssueTitle.Length >= ModelTrackingIssue.IssueTitleLengthLimit) { ErrorMessage = $"Please limit issue title to {ModelTrackingIssue.IssueTitleLengthLimit} characters"; return(Page()); } if (string.IsNullOrEmpty(SearchText)) { ErrorMessage = "Must provide search text"; return(Page()); } ModelBuildDefinition?modelBuildDefinition = null; if (!string.IsNullOrEmpty(Definition)) { modelBuildDefinition = await TriageContextUtil.FindModelBuildDefinitionAsync(Definition); if (modelBuildDefinition is null) { ErrorMessage = $"Cannot find build definition with name or ID: {Definition}"; return(Page()); } } switch (TrackingKind) { #pragma warning disable 618 case TrackingKind.HelixConsole: case TrackingKind.HelixRunClient: ErrorMessage = $"'{TrackingKind}' is deprecated. Please use {TrackingKind.HelixLogs}"; return(Page()); case TrackingKind.HelixLogs: { if (TryParseQueryString <SearchHelixLogsRequest>(out var request)) { if (request.HelixLogKinds.Count == 0) { ErrorMessage = "Need to specify at least one log kind to search"; return(Page()); } } else { return(Page()); } } break; case TrackingKind.Test: if (!TryParseQueryString <SearchTestsRequest>(out _)) { return(Page()); } break; case TrackingKind.Timeline: if (!TryParseQueryString <SearchTimelinesRequest>(out _)) { return(Page()); } break; } GitHubIssueKey?issueKey = null; if (!string.IsNullOrEmpty(GitHubIssueUri)) { if (GitHubIssueKey.TryCreateFromUri(GitHubIssueUri, out var key)) { issueKey = key; GitHubOrganization = key.Organization; GitHubRepository = key.Repository; } else { ErrorMessage = $"Invalid GitHub issue link: {GitHubIssueUri}"; return(Page()); } } else if (string.IsNullOrEmpty(GitHubRepository) || string.IsNullOrEmpty(GitHubOrganization)) { ErrorMessage = "Must provide GitHub organization and repository"; return(Page()); } IGitHubClient?gitHubClient = null; try { gitHubClient = await GitHubClientFactory.CreateForAppAsync(GitHubOrganization, GitHubRepository); } catch (Exception ex) { ErrorMessage = $"Cannot create GitHub client for that repository: {ex.Message}"; return(Page()); } var modelTrackingIssue = await CreateTrackingIssue(gitHubClient); return(RedirectToPage( "./Issue", new { id = modelTrackingIssue.Id })); async Task <ModelTrackingIssue> CreateTrackingIssue(IGitHubClient gitHubClient) { var issueKey = await GetOrCreateGitHubIssueAsync(gitHubClient); var modelTrackingIssue = new ModelTrackingIssue() { IsActive = true, IssueTitle = IssueTitle, TrackingKind = TrackingKind, SearchQuery = SearchText, ModelBuildDefinition = modelBuildDefinition, GitHubOrganization = issueKey.Organization, GitHubRepository = issueKey.Repository, GitHubIssueNumber = issueKey.Number, }; TriageContext.ModelTrackingIssues.Add(modelTrackingIssue); await TriageContext.SaveChangesAsync(); await InitialTriageAsync(modelTrackingIssue); return(modelTrackingIssue); } async Task InitialTriageAsync(ModelTrackingIssue modelTrackingIssue) { if (modelBuildDefinition is object) { // The initial triage is only done for tracking issues that have definitions // associated with. Lacking a definition we end up querying all builds and that // can produce a *lot* of data. Hard to find a good formula that is performant // there hence we limit to only builds with definitions. var request = new SearchBuildsRequest() { Definition = modelBuildDefinition?.DefinitionId.ToString() ?? null, Queued = new DateRequestValue(7, RelationalKind.GreaterThan), Result = new BuildResultRequestValue(BuildResult.Succeeded, EqualsKind.NotEquals), }; await FunctionQueueUtil.QueueTriageBuildAttempts(TriageContextUtil, modelTrackingIssue, request); // Issues are bulk updated on a 15 minute cycle. This is a new issue though so want to make sure that // the user sees progress soon. Schedule two manual updates in the near future on this so the issue // gets rolling then it will fall into the 15 minute bulk cycle. await FunctionQueueUtil.QueueUpdateIssueAsync(modelTrackingIssue, TimeSpan.FromSeconds(30)); await FunctionQueueUtil.QueueUpdateIssueAsync(modelTrackingIssue, TimeSpan.FromMinutes(2)); } else { await FunctionQueueUtil.QueueUpdateIssueAsync(modelTrackingIssue, TimeSpan.FromMinutes(0)); } } async Task <GitHubIssueKey> GetOrCreateGitHubIssueAsync(IGitHubClient gitHubClient) { if (issueKey is { } key) { await TrackingGitHubUtil.EnsureGitHubIssueHasMarkers(gitHubClient, key); return(key); } var newIssue = new NewIssue(IssueTitle) { Body = TrackingGitHubUtil.WrapInStartEndMarkers("Runfo Creating Tracking Issue (data being generated)") }; var issue = await gitHubClient.Issue.Create(GitHubOrganization, GitHubRepository, newIssue); return(issue.GetIssueKey()); } bool TryParseQueryString <T>(out T value) where T : ISearchRequest, new() { value = new T(); try { value.ParseQueryString(SearchText ?? ""); return(true); } catch (Exception ex) { ErrorMessage = ex.ToString(); return(false); } } }
/// <summary> /// This function will queue up a number of <see cref="ModelBuildAttempt"/> instances to triage against the specified /// <see cref="ModelTrackingIssue"/>. This is useful to essentially seed old builds against a given tracking /// issue (aka populate the data set) while at the same time new builds will be showing up via normal completion. /// It will return the number of attempts that were queued for processing, /// /// One particular challenge we have to keep in mind is that this is going to be queueing up a lot of builds /// into our Azure functions. Those will scale to whatever data we put into there. Need to be mindful to not /// queue up say 100,000 builds as that will end up spiking all our resources. Have to put some throttling /// in here. /// </summary> public static async Task <int> QueueTriageBuildAttempts( this FunctionQueueUtil util, TriageContextUtil triageContextUtil, ModelTrackingIssue trackingIssue, SearchBuildsRequest buildsRequest, int limit = 200) { // Need to filter to a bulid definition other wise there is no reasonable way to filter the builds. Any // triage is basically pointless. if (trackingIssue.ModelBuildDefinition is null && !buildsRequest.HasDefinition) { throw new Exception("Must filter to a build definition"); } // Ensure there is some level of filtering occuring here. if (buildsRequest.Result is null) { buildsRequest.Result = new BuildResultRequestValue(BuildResult.Succeeded, EqualsKind.NotEquals); } if (buildsRequest.Queued is null && buildsRequest.Started is null && buildsRequest.Finished is null) { buildsRequest.Queued = new DateRequestValue(7, RelationalKind.GreaterThan); } var buildsQuery = buildsRequest .Filter(triageContextUtil.Context.ModelBuilds) .OrderByDescending(x => x.BuildNumber) .SelectMany(x => x.ModelBuildAttempts) .Select(x => new { x.ModelBuild.BuildNumber, x.ModelBuild.AzureOrganization, x.ModelBuild.AzureProject, x.Attempt, x.Id }); var existingAttemptsQuery = triageContextUtil .Context .ModelTrackingIssueResults .Where(x => x.ModelTrackingIssueId == trackingIssue.Id) .Include(x => x.ModelBuildAttempt) .ThenInclude(x => x.ModelBuild) .Select(x => new { x.ModelBuildAttempt.ModelBuild.AzureOrganization, x.ModelBuildAttempt.ModelBuild.AzureProject, x.ModelBuildAttempt.ModelBuild.BuildNumber, x.ModelBuildAttempt.Attempt, }); var existingAttemptsResults = await existingAttemptsQuery.ToListAsync(); var existingAttemptsSet = new HashSet <BuildAttemptKey>( existingAttemptsResults.Select(x => new BuildAttemptKey(x.AzureOrganization, x.AzureProject, x.BuildNumber, x.Attempt))); var attempts = await buildsQuery.ToListAsync(); var attemptKeys = attempts .Select(x => new BuildAttemptKey(x.AzureOrganization, x.AzureProject, x.BuildNumber, x.Attempt)) .Where(x => !existingAttemptsSet.Contains(x)) .Take(limit) .ToList(); await util.QueueTriageBuildAttemptsAsync(trackingIssue, attemptKeys); return(attemptKeys.Count); }
public TrackingIssueModel(TriageContextUtil triageContextUtil, FunctionQueueUtil functionQueueUtil, IGitHubClientFactory gitHubClientFactory) { TriageContextUtil = triageContextUtil; FunctionQueueUtil = functionQueueUtil; GitHubClientFactory = gitHubClientFactory; }