public override async Task OnActivateAsync() { using (var context = _contextFactory.CreateInstance()) { var orgId = this.GetPrimaryKeyLong(); // Ensure this organization actually exists var org = await context.Organizations .Include(x => x.OrganizationAccounts) .SingleOrDefaultAsync(x => x.Id == orgId); if (org == null) { throw new InvalidOperationException($"Organization {orgId} does not exist and cannot be activated."); } Initialize(org.Id, org.Login); // MUST MATCH SAVE _metadata = org.Metadata; _adminMetadata = org.OrganizationMetadata; _projectMetadata = org.ProjectMetadata; } await base.OnActivateAsync(); }
private async Task UpdatePullRequestCommitStatuses(IGitHubActor ghc, DataUpdater updater) { var commitStatusesResponse = await ghc.CommitStatuses(_repoFullName, _prHeadSha, _prStatusMetadata, RequestPriority.Interactive); if (commitStatusesResponse.IsOk) { await updater.UpdateCommitStatuses(_repoId, _prHeadSha, commitStatusesResponse.Result); } _prStatusMetadata = GitHubMetadata.FromResponse(commitStatusesResponse); }
private async Task SyncTimerCallback(object state = null) { if (DateTimeOffset.UtcNow.Subtract(_lastSyncInterest) > SyncIdle) { DeactivateOnIdle(); return; } var metaDataMeaningfullyChanged = false; var updater = new DataUpdater(_contextFactory, _mapper); try { // NOTE: The following requests are (relatively) infrequent and important for access control (repos/orgs) // Give them high priority. // User if (_metadata.IsExpired()) { var user = await _github.User(_metadata, RequestPriority.Interactive); if (user.IsOk) { metaDataMeaningfullyChanged = true; await updater.UpdateAccounts(user.Date, new[] { user.Result }); // Unlike orgs, login renames are fine here. // Current user is implicit in all calls, not specified. } // Don't update until saved. _metadata = GitHubMetadata.FromResponse(user); } await _syncLimit.WaitAsync(); try { metaDataMeaningfullyChanged |= await InternalSync(updater); } finally { _syncLimit.Release(); } } catch (GitHubRateException) { // nothing to do } await updater.Changes.Submit(_queueClient, urgent : true); // Mentions _mentions.Sync().LogFailure(_userInfo); // Save changes if (metaDataMeaningfullyChanged) { await Save(); } }
private async Task UpdateIssueDetails(IGitHubActor ghc, DataUpdater updater) { var issueResponse = await ghc.Issue(_repoFullName, _issueNumber, _metadata, RequestPriority.Interactive); if (issueResponse.IsOk) { _isPullRequest = issueResponse.Result.PullRequest != null; // Issues can become PRs await updater.UpdateIssues(_repoId, issueResponse.Date, new[] { issueResponse.Result }); } _metadata = GitHubMetadata.FromResponse(issueResponse); }
private async Task UpdateProjects(DataUpdater updater, IGitHubPoolable github) { if (_projectMetadata.IsExpired()) { var projects = await github.OrganizationProjects(_login, _projectMetadata); if (projects.IsOk) { await updater.UpdateOrganizationProjects(_orgId, projects.Date, projects.Result); } _projectMetadata = GitHubMetadata.FromResponse(projects); } }
private async Task UpdateIssueReactions(IGitHubActor ghc, DataUpdater updater) { if (_reactionMetadata.IsExpired()) { var issueReactionsResponse = await ghc.IssueReactions(_repoFullName, _issueNumber, _reactionMetadata, RequestPriority.Interactive); if (issueReactionsResponse.IsOk) { await updater.UpdateIssueReactions(_repoId, issueReactionsResponse.Date, _issueId, issueReactionsResponse.Result); } _reactionMetadata = GitHubMetadata.FromResponse(issueReactionsResponse); } }
private async Task <ISet <long> > UpdatePullRequestComments(IGitHubActor ghc, DataUpdater updater) { ISet <long> prCommentIds = null; var prCommentsResponse = await ghc.PullRequestComments(_repoFullName, _issueNumber, _prCommentMetadata, RequestPriority.Interactive); if (prCommentsResponse.IsOk) { prCommentIds = prCommentsResponse.Result.Select(x => x.Id).ToHashSet(); await updater.UpdatePullRequestComments(_repoId, _issueId, prCommentsResponse.Date, prCommentsResponse.Result); } _prCommentMetadata = GitHubMetadata.FromResponse(prCommentsResponse); return(prCommentIds); }
private async Task SyncTimerCallback(object state = null) { if (DateTimeOffset.UtcNow.Subtract(_lastSyncInterest) > SyncIdle) { DeactivateOnIdle(); return; } var metaDataMeaningfullyChanged = false; var updater = new DataUpdater(_contextFactory, _mapper); try { // Issue Mentions if (_mentionMetadata.IsExpired()) { var mentions = await _github.IssueMentions(_mentionSince, MentionNibblePages, _mentionMetadata, RequestPriority.Background); if (mentions.IsOk && mentions.Result.Any()) { metaDataMeaningfullyChanged = true; await updater.UpdateIssueMentions(_userId, mentions.Result); var maxSince = mentions.Result.Max(x => x.UpdatedAt).AddSeconds(-5); if (maxSince != _mentionSince) { await updater.UpdateAccountMentionSince(_userId, maxSince); _mentionSince = maxSince; } } // Don't update until saved. _mentionMetadata = GitHubMetadata.FromResponse(mentions); } } catch (GitHubRateException) { // nothing to do } await updater.Changes.Submit(_queueClient); // Save changes if (metaDataMeaningfullyChanged) { await Save(); } }
private async Task UpdateAdmins(DataUpdater updater, IGitHubPoolable github) { if (_adminMetadata.IsExpired()) { var admins = await github.OrganizationMembers(_login, role : "admin", cacheOptions : _adminMetadata); if (admins.IsOk) { await updater.SetOrganizationAdmins(_orgId, admins.Date, admins.Result); } else if (!admins.Succeeded) { throw new Exception($"Unexpected response: [{admins?.Request?.Uri}] {admins?.Status}"); } _adminMetadata = GitHubMetadata.FromResponse(admins); } }
private async Task UpdateDetails(DataUpdater updater, IGitHubPoolable github) { if (_metadata.IsExpired()) { var org = await github.Organization(_orgId, _metadata); if (org.IsOk) { await updater.UpdateAccounts(org.Date, new[] { org.Result }); // Update login For the rest of sync. _login = org.Result.Login; // Safest to just start over. DeactivateOnIdle(); } _metadata = GitHubMetadata.FromResponse(org); } }
public override async Task OnActivateAsync() { _issueNumber = (int)this.GetPrimaryKeyLong(out _repoFullName); using (var context = _contextFactory.CreateInstance()) { var issue = await context.Issues .AsNoTracking() .SingleOrDefaultAsync(x => x.Repository.FullName == _repoFullName && x.Number == _issueNumber); if (issue == null) { throw new InvalidOperationException($"Issue {_repoFullName}#{_issueNumber} does not exist and cannot be activated."); } _issueId = issue.Id; _repoId = issue.RepositoryId; _isPullRequest = issue.PullRequest; _metadata = issue.Metadata; _reactionMetadata = issue.ReactionMetadata; if (_isPullRequest) { var pullRequest = await context.PullRequests .AsNoTracking() .SingleOrDefaultAsync(x => x.IssueId == _issueId); _prId = pullRequest?.Id; _prHeadSha = pullRequest?.Head?.Sha; _prBaseBranch = pullRequest?.Base?.Ref; _prMergeableStateBlocked = pullRequest?.MergeableState?.Equals("blocked", StringComparison.OrdinalIgnoreCase) == true; _prMetadata = pullRequest?.Metadata; _prCommentMetadata = pullRequest?.CommentMetadata; _prStatusMetadata = pullRequest?.StatusMetadata; _prMergeStatusMetadata = pullRequest?.MergeStatusMetadata; } } await base.OnActivateAsync(); }
private async Task UpdatePullRequestDetails(IGitHubActor ghc, long forUserId, DataUpdater updater) { // Sadly, the PR info doesn't contain labels 😠var prResponse = await ghc.PullRequest(_repoFullName, _issueNumber, _prMetadata, RequestPriority.Interactive); if (prResponse.IsOk) { var pr = prResponse.Result; _prId = pr.Id; // Issues can become PRs _prHeadSha = pr.Head.Sha; _prBaseBranch = pr.Base.Ref; _prMergeableStateBlocked = pr.MergeableState == "blocked"; await updater.UpdatePullRequests(_repoId, prResponse.Date, new[] { prResponse.Result }); } _prMetadata = GitHubMetadata.FromResponse(prResponse); // Branch Protection if (_prMergeableStateBlocked && _prBaseBranch != null) { _grainFactory.GetGrain <IRepositoryActor>(_repoId).SyncProtectedBranch(_prBaseBranch, forUserId).LogFailure(); } }
public override async Task OnActivateAsync() { // Set this first as subsequent calls require it. _userId = this.GetPrimaryKeyLong(); // Ensure this user actually exists, and lookup their token. User user = null; using (var context = _contextFactory.CreateInstance()) { user = await context.Users .AsNoTracking() .Include(x => x.Tokens) .SingleOrDefaultAsync(x => x.Id == _userId); } if (user == null) { throw new InvalidOperationException($"User {_userId} does not exist and cannot be activated."); } if (!user.Tokens.Any()) { throw new InvalidOperationException($"User {_userId} has an invalid token and cannot be activated."); } _github = _grainFactory.GetGrain <IGitHubActor>(user.Id); _mentionMetadata = user.MentionMetadata; _mentionSince = user.MentionSince ?? EpochUtility.EpochOffset; // Always sync while active _lastSyncInterest = DateTimeOffset.UtcNow; _syncTimer = RegisterTimer(SyncTimerCallback, null, TimeSpan.Zero, SyncDelay); await base.OnActivateAsync(); }
private async Task <(ISet <long> IssueCommentIds, ISet <long> CommitCommentIds)> UpdateIssueTimeline(IGitHubActor ghc, long forUserId, DataUpdater updater) { /////////////////////////////////////////// /* NOTE! * We can't sync the timeline incrementally, because the client wants commit and * reference data inlined. This means we always have to download all the * timeline events in case an old one now has updated data. Other options are to * just be wrong, or to simply reference the user by id and mark them referenced * by the repo. */ ////////////////////////////////////////// var issueCommentIds = new HashSet <long>(); var commitCommentIds = new HashSet <long>(); // TODO: cache per-user // TODO: If caching, are there things that should occur every time anyway? var timelineResponse = await ghc.Timeline(_repoFullName, _issueNumber, _issueId, priority : RequestPriority.Interactive); if (timelineResponse.IsOk) { var allEvents = timelineResponse.Result; var filteredEvents = allEvents.Where(x => !_IgnoreTimelineEvents.Contains(x.Event)).ToArray(); // For adding to the DB later // TODO: Technically this can pick stale accounts if an old and new version are both in the collection. // My batching branch fixes this by tracking versions // Set must allow nulls var accounts = new HashSet <gm.Account>(KeyEqualityComparer.FromKeySelector((gm.Account x) => x?.Id)); foreach (var timelineEvent in filteredEvents) { accounts.Add(timelineEvent.Actor); accounts.Add(timelineEvent.Assignee); } await LookupEventCommitDetails(ghc, accounts, filteredEvents); await LookupEventSourceDetails(ghc, accounts, filteredEvents); // Fixup and sanity checks foreach (var item in filteredEvents) { switch (item.Event) { case "cross-referenced": if (item.Source?.Actor != null) { accounts.Add(item.Source?.Actor); } if (item.Source?.Issue?.User != null) { accounts.Add(item.Source?.Issue?.User); } //if (item.Actor != null) { // It's a comment reference // accounts.Add(item.Source?.Actor); // item.Actor = item.Source?.Actor; //} else { // It's an issue reference // accounts.Add(item.Source?.Issue?.User); // item.Actor = item.Source?.Issue?.User; //} break; case "committed": item.CreatedAt = item.ExtensionDataDictionary["committer"]["date"].ToObject <DateTimeOffset>(); break; default: // Leave most things alone. break; } if (item.CreatedAt == DateTimeOffset.MinValue) { throw new Exception($"Unable to process event of type {item.Event} on {_repoFullName}/{_issueNumber} ({_issueId}). Invalid date."); } } await updater.UpdateTimelineEvents(_repoId, timelineResponse.Date, forUserId, accounts, filteredEvents); // Comments var commentEvents = allEvents.Where(x => x.Event == "commented").ToArray(); if (commentEvents.Any()) { // The events have all the info we need. var comments = commentEvents.Select(x => x.Roundtrip <gm.IssueComment>(serializerSettings: GitHubSerialization.JsonSerializerSettings)).ToArray(); // Update known ids issueCommentIds.UnionWith(comments.Select(x => x.Id)); await updater.UpdateIssueComments(_repoId, timelineResponse.Date, comments); } // Commit Comments // Comments in commit-commented events look complete. // Let's run with it. var commitCommentEvents = allEvents.Where(x => x.Event == "commit-commented").ToArray(); if (commitCommentEvents.Any()) { var commitComments = commitCommentEvents .SelectMany(x => x.ExtensionDataDictionary["comments"].ToObject <IEnumerable <gm.CommitComment> >(GitHubSerialization.JsonSerializer)) .ToArray(); // Update known ids commitCommentIds.UnionWith(commitComments.Select(x => x.Id)); await updater.UpdateCommitComments(_repoId, timelineResponse.Date, commitComments); } // Merged event commit statuses if (_isPullRequest) { var mergedEvent = allEvents .Where(x => x.Event == "merged") .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); var mergeCommitId = mergedEvent?.CommitId; if (mergeCommitId != null) { var mergeCommitStatusesResponse = await ghc.CommitStatuses(_repoFullName, mergeCommitId, _prMergeStatusMetadata, RequestPriority.Interactive); if (mergeCommitStatusesResponse.IsOk) { await updater.UpdateCommitStatuses(_repoId, mergeCommitId, mergeCommitStatusesResponse.Result); } _prMergeStatusMetadata = GitHubMetadata.FromResponse(mergeCommitStatusesResponse); } } } return(IssueCommentIds : issueCommentIds, CommitCommentIds : commitCommentIds); }
public override async Task OnActivateAsync() { // Set this first as subsequent calls require it. _userId = this.GetPrimaryKeyLong(); // Ensure this user actually exists, and lookup their token. User user = null; using (var context = _contextFactory.CreateInstance()) { user = await context.Users .AsNoTracking() .Include(x => x.Tokens) .Include(x => x.Settings) .Include(x => x.LinkedRepositories) .Include(x => x.SyncRepositories) .Include(x => x.AccountOrganizations) .SingleOrDefaultAsync(x => x.Id == _userId); } if (user == null) { throw new InvalidOperationException($"User {_userId} does not exist and cannot be activated."); } if (!user.Tokens.Any()) { throw new InvalidOperationException($"User {_userId} has an invalid token and cannot be activated."); } _userInfo = $"{user.Login} ({user.Id})"; _github = _grainFactory.GetGrain <IGitHubActor>(user.Id); _mentions = _grainFactory.GetGrain <IMentionsActor>(user.Id); _metadata = user.Metadata; _repoMetadata = user.RepositoryMetadata; _orgMetadata = user.OrganizationMetadata; _syncSettings = user.Settings?.SyncSettings; _linkedRepos = user.LinkedRepositories.Select(x => x.RepositoryId).ToHashSet(); _orgActors = user.AccountOrganizations .ToDictionary(x => x.OrganizationId, x => _grainFactory.GetGrain <IOrganizationActor>(x.OrganizationId)); _repoActors = user.SyncRepositories .ToDictionary(x => x.RepositoryId, x => _grainFactory.GetGrain <IRepositoryActor>(x.RepositoryId)); _includeRepoMetadata = user.SyncRepositories .Where(x => !_linkedRepos.Contains(x.RepositoryId)) .ToDictionary(x => x.RepositoryId, x => x.RepositoryMetadata); // Migration Step // No settings + linked repos + empty sync repos == MIGRATE! if (_linkedRepos.Any() && _syncSettings == null && !_repoActors.Any()) { Interlocked.Increment(ref _linkedReposDesired); Interlocked.Increment(ref _syncReposDesired); } // We always want to sync while the UserActor is loaded; _lastSyncInterest = DateTimeOffset.UtcNow; _syncTimer = RegisterTimer(SyncTimerCallback, null, TimeSpan.Zero, SyncDelay); await base.OnActivateAsync(); }
public async Task <bool> InternalSync(DataUpdater updater) { if (_syncLimit.CurrentCount != 0) { throw new InvalidOperationException($"{nameof(InternalSync)} requires the sync semaphore be held."); } var metaDataMeaningfullyChanged = false; try { // NOTE: The following requests are (relatively) infrequent and important for access control (repos/orgs) // Give them high priority. // Update this user's org memberships if (_orgMetadata.IsExpired()) { var orgs = await _github.OrganizationMemberships(cacheOptions : _orgMetadata, priority : RequestPriority.Interactive); if (orgs.IsOk) { metaDataMeaningfullyChanged = true; await updater.SetUserOrganizations(_userId, orgs.Date, orgs.Result); _orgActors = orgs.Result .ToDictionary(x => x.Organization.Id, x => _grainFactory.GetGrain <IOrganizationActor>(x.Organization.Id)); // Also re-evaluate their linked repos Interlocked.Increment(ref _linkedReposDesired); } // Don't update until saved. _orgMetadata = GitHubMetadata.FromResponse(orgs); } // Update this user's repo memberships var savedLinkCurrent = _linkedReposCurrent; var savedLinkDesired = _linkedReposDesired; if ((savedLinkCurrent < savedLinkDesired) || _repoMetadata.IsExpired()) { var repos = await _github.Repositories(_repoMetadata, RequestPriority.Interactive); if (repos.IsOk) { metaDataMeaningfullyChanged = true; Interlocked.Increment(ref _syncReposDesired); await updater.SetUserRepositories(_userId, repos.Date, repos.Result); _linkedRepos = repos.Result.Select(x => x.Id).ToHashSet(); // don't update _repoActors yet } // Don't update until saved. _repoMetadata = GitHubMetadata.FromResponse(repos); Interlocked.CompareExchange(ref _linkedReposCurrent, savedLinkDesired, savedLinkCurrent); } // Update this user's sync repos var savedReposCurrent = _syncReposCurrent; var savedReposDesired = _syncReposDesired; if (savedReposCurrent < savedReposDesired) { IEnumerable <g.Repository> updateRepos = null; IDictionary <long, GitHubMetadata> combinedRepoMetadata = null; var date = DateTimeOffset.UtcNow; // Not *technically* correct, but probably ok if (_syncSettings?.Include.Any() == true) { // Request all "included" repos to verify access var repoReqs = _syncSettings.Include .Where(x => !_linkedRepos.Contains(x)) // Exclude linked repos (access already known) .Select(x => (RepoId: x, Request: _github.Repository(x, _includeRepoMetadata.Val(x), RequestPriority.Interactive))) .ToArray(); await Task.WhenAll(repoReqs.Select(x => x.Request)); // Collect the "successful" responses // Check explicitly for 404, since 502s are so common :/ var successful = repoReqs .Where(x => x.Request.Status == TaskStatus.RanToCompletion) .Where(x => x.Request.Result.Status != HttpStatusCode.NotFound) .ToArray(); _includeRepoMetadata = successful.ToDictionary(x => x.RepoId, x => GitHubMetadata.FromResponse(x.Request.Result)); updateRepos = successful .Where(x => x.Request.Result.IsOk) .Select(x => x.Request.Result.Result) .ToArray(); // now union/intersect all the things combinedRepoMetadata = new Dictionary <long, GitHubMetadata>(); foreach (var repoId in _syncSettings.Include) { if (_linkedRepos.Contains(repoId)) { combinedRepoMetadata.Add(repoId, null); } else if (_includeRepoMetadata.ContainsKey(repoId)) { combinedRepoMetadata.Add(repoId, _includeRepoMetadata[repoId]); } // else drop it } } var syncRepoMetadata = await updater.UpdateAccountSyncRepositories( _userId, _syncSettings?.AutoTrack ?? true, date, updateRepos, combinedRepoMetadata, _syncSettings?.Exclude); _repoActors = syncRepoMetadata.Keys .ToDictionary(x => x, x => _grainFactory.GetGrain <IRepositoryActor>(x)); _includeRepoMetadata = syncRepoMetadata .Where(x => !_linkedRepos.Contains(x.Key)) .ToDictionary(x => x.Key, x => x.Value); // TODO: Actually detect changes. var cs = new ChangeSummary(); cs.Add(userId: _userId); updater.UnionWithExternalChanges(cs); Interlocked.CompareExchange(ref _syncReposCurrent, savedReposDesired, savedReposCurrent); } } catch (GitHubRateException) { // nothing to do } // We do this here so newly added repos and orgs sync immediately // Sync repos foreach (var repo in _repoActors.Values) { repo.Sync().LogFailure(_userInfo); } // Sync orgs foreach (var org in _orgActors.Values) { org.Sync().LogFailure(_userInfo); } return(metaDataMeaningfullyChanged); }
private async Task UpdatePullRequestReviews(IGitHubActor ghc, long forUserId, DataUpdater updater) { gm.Review myReview = null; // Reviews and Review comments need to use the current user's token. Don't track metadata (yet - per user ideally) var prReviewsResponse = await ghc.PullRequestReviews(_repoFullName, _issueNumber, priority : RequestPriority.Interactive); if (prReviewsResponse.IsOk) { await updater.UpdateReviews(_repoId, _issueId, prReviewsResponse.Date, prReviewsResponse.Result, userId : forUserId, complete : true); myReview = prReviewsResponse.Result .Where(x => x.State.Equals("PENDING", StringComparison.OrdinalIgnoreCase)) .FirstOrDefault(x => x.User.Id == forUserId); } using (var context = _contextFactory.CreateInstance()) { await context.UpdateMetadata("PullRequests", "IssueId", "ReviewMetadataJson", _issueId, GitHubMetadata.FromResponse(prReviewsResponse)); } // Only fetch if *this user* has a pending review // Since we make the request every time, it's ok not to look for pending reviews in the DB if (myReview != null) { var reviewCommentsResponse = await ghc.PullRequestReviewComments(_repoFullName, _issueNumber, myReview.Id, priority : RequestPriority.Interactive); if (reviewCommentsResponse.IsOk && reviewCommentsResponse.Result.Any()) { await updater.UpdatePullRequestComments(_repoId, _issueId, reviewCommentsResponse.Date, reviewCommentsResponse.Result, pendingReviewId : myReview.Id); } } }