public override async Task OnActivateAsync()
{
using (var context = _contextFactory.CreateInstance()) {
var orgId = this.GetPrimaryKeyLong();
// Ensure this organization actually exists
var org = await context.Organizations
.Include(x => x.OrganizationAccounts)
.SingleOrDefaultAsync(x => x.Id == orgId);
if (org == null)
{
throw new InvalidOperationException($"Organization {orgId} does not exist and cannot be activated.");
}
Initialize(org.Id, org.Login);
// MUST MATCH SAVE
_metadata = org.Metadata;
_adminMetadata = org.OrganizationMetadata;
_projectMetadata = org.ProjectMetadata;
}
await base.OnActivateAsync();
}
private async Task UpdatePullRequestCommitStatuses(IGitHubActor ghc, DataUpdater updater)
{
var commitStatusesResponse = await ghc.CommitStatuses(_repoFullName, _prHeadSha, _prStatusMetadata, RequestPriority.Interactive);
if (commitStatusesResponse.IsOk)
{
await updater.UpdateCommitStatuses(_repoId, _prHeadSha, commitStatusesResponse.Result);
}
_prStatusMetadata = GitHubMetadata.FromResponse(commitStatusesResponse);
}
private async Task SyncTimerCallback(object state = null)
{
if (DateTimeOffset.UtcNow.Subtract(_lastSyncInterest) > SyncIdle)
{
DeactivateOnIdle();
return;
}
var metaDataMeaningfullyChanged = false;
var updater = new DataUpdater(_contextFactory, _mapper);
try {
// NOTE: The following requests are (relatively) infrequent and important for access control (repos/orgs)
// Give them high priority.
// User
if (_metadata.IsExpired())
{
var user = await _github.User(_metadata, RequestPriority.Interactive);
if (user.IsOk)
{
metaDataMeaningfullyChanged = true;
await updater.UpdateAccounts(user.Date, new[] { user.Result });
// Unlike orgs, login renames are fine here.
// Current user is implicit in all calls, not specified.
}
// Don't update until saved.
_metadata = GitHubMetadata.FromResponse(user);
}
await _syncLimit.WaitAsync();
try {
metaDataMeaningfullyChanged |= await InternalSync(updater);
} finally {
_syncLimit.Release();
}
} catch (GitHubRateException) {
// nothing to do
}
await updater.Changes.Submit(_queueClient, urgent : true);
// Mentions
_mentions.Sync().LogFailure(_userInfo);
// Save changes
if (metaDataMeaningfullyChanged)
{
await Save();
}
}
private async Task UpdateIssueDetails(IGitHubActor ghc, DataUpdater updater)
{
var issueResponse = await ghc.Issue(_repoFullName, _issueNumber, _metadata, RequestPriority.Interactive);
if (issueResponse.IsOk)
{
_isPullRequest = issueResponse.Result.PullRequest != null; // Issues can become PRs
await updater.UpdateIssues(_repoId, issueResponse.Date, new[] { issueResponse.Result });
}
_metadata = GitHubMetadata.FromResponse(issueResponse);
}
private async Task UpdateProjects(DataUpdater updater, IGitHubPoolable github)
{
if (_projectMetadata.IsExpired())
{
var projects = await github.OrganizationProjects(_login, _projectMetadata);
if (projects.IsOk)
{
await updater.UpdateOrganizationProjects(_orgId, projects.Date, projects.Result);
}
_projectMetadata = GitHubMetadata.FromResponse(projects);
}
}
private async Task UpdateIssueReactions(IGitHubActor ghc, DataUpdater updater)
{
if (_reactionMetadata.IsExpired())
{
var issueReactionsResponse = await ghc.IssueReactions(_repoFullName, _issueNumber, _reactionMetadata, RequestPriority.Interactive);
if (issueReactionsResponse.IsOk)
{
await updater.UpdateIssueReactions(_repoId, issueReactionsResponse.Date, _issueId, issueReactionsResponse.Result);
}
_reactionMetadata = GitHubMetadata.FromResponse(issueReactionsResponse);
}
}
private async Task <ISet <long> > UpdatePullRequestComments(IGitHubActor ghc, DataUpdater updater)
{
ISet <long> prCommentIds = null;
var prCommentsResponse = await ghc.PullRequestComments(_repoFullName, _issueNumber, _prCommentMetadata, RequestPriority.Interactive);
if (prCommentsResponse.IsOk)
{
prCommentIds = prCommentsResponse.Result.Select(x => x.Id).ToHashSet();
await updater.UpdatePullRequestComments(_repoId, _issueId, prCommentsResponse.Date, prCommentsResponse.Result);
}
_prCommentMetadata = GitHubMetadata.FromResponse(prCommentsResponse);
return(prCommentIds);
}
private async Task SyncTimerCallback(object state = null)
{
if (DateTimeOffset.UtcNow.Subtract(_lastSyncInterest) > SyncIdle)
{
DeactivateOnIdle();
return;
}
var metaDataMeaningfullyChanged = false;
var updater = new DataUpdater(_contextFactory, _mapper);
try {
// Issue Mentions
if (_mentionMetadata.IsExpired())
{
var mentions = await _github.IssueMentions(_mentionSince, MentionNibblePages, _mentionMetadata, RequestPriority.Background);
if (mentions.IsOk && mentions.Result.Any())
{
metaDataMeaningfullyChanged = true;
await updater.UpdateIssueMentions(_userId, mentions.Result);
var maxSince = mentions.Result.Max(x => x.UpdatedAt).AddSeconds(-5);
if (maxSince != _mentionSince)
{
await updater.UpdateAccountMentionSince(_userId, maxSince);
_mentionSince = maxSince;
}
}
// Don't update until saved.
_mentionMetadata = GitHubMetadata.FromResponse(mentions);
}
} catch (GitHubRateException) {
// nothing to do
}
await updater.Changes.Submit(_queueClient);
// Save changes
if (metaDataMeaningfullyChanged)
{
await Save();
}
}
private async Task UpdateAdmins(DataUpdater updater, IGitHubPoolable github)
{
if (_adminMetadata.IsExpired())
{
var admins = await github.OrganizationMembers(_login, role : "admin", cacheOptions : _adminMetadata);
if (admins.IsOk)
{
await updater.SetOrganizationAdmins(_orgId, admins.Date, admins.Result);
}
else if (!admins.Succeeded)
{
throw new Exception($"Unexpected response: [{admins?.Request?.Uri}] {admins?.Status}");
}
_adminMetadata = GitHubMetadata.FromResponse(admins);
}
}
private async Task UpdateDetails(DataUpdater updater, IGitHubPoolable github)
{
if (_metadata.IsExpired())
{
var org = await github.Organization(_orgId, _metadata);
if (org.IsOk)
{
await updater.UpdateAccounts(org.Date, new[] { org.Result });
// Update login For the rest of sync.
_login = org.Result.Login;
// Safest to just start over.
DeactivateOnIdle();
}
_metadata = GitHubMetadata.FromResponse(org);
}
}
public override async Task OnActivateAsync()
{
_issueNumber = (int)this.GetPrimaryKeyLong(out _repoFullName);
using (var context = _contextFactory.CreateInstance()) {
var issue = await context.Issues
.AsNoTracking()
.SingleOrDefaultAsync(x => x.Repository.FullName == _repoFullName && x.Number == _issueNumber);
if (issue == null)
{
throw new InvalidOperationException($"Issue {_repoFullName}#{_issueNumber} does not exist and cannot be activated.");
}
_issueId = issue.Id;
_repoId = issue.RepositoryId;
_isPullRequest = issue.PullRequest;
_metadata = issue.Metadata;
_reactionMetadata = issue.ReactionMetadata;
if (_isPullRequest)
{
var pullRequest = await context.PullRequests
.AsNoTracking()
.SingleOrDefaultAsync(x => x.IssueId == _issueId);
_prId = pullRequest?.Id;
_prHeadSha = pullRequest?.Head?.Sha;
_prBaseBranch = pullRequest?.Base?.Ref;
_prMergeableStateBlocked = pullRequest?.MergeableState?.Equals("blocked", StringComparison.OrdinalIgnoreCase) == true;
_prMetadata = pullRequest?.Metadata;
_prCommentMetadata = pullRequest?.CommentMetadata;
_prStatusMetadata = pullRequest?.StatusMetadata;
_prMergeStatusMetadata = pullRequest?.MergeStatusMetadata;
}
}
await base.OnActivateAsync();
}
private async Task UpdatePullRequestDetails(IGitHubActor ghc, long forUserId, DataUpdater updater)
{
// Sadly, the PR info doesn't contain labels 😭
var prResponse = await ghc.PullRequest(_repoFullName, _issueNumber, _prMetadata, RequestPriority.Interactive);
if (prResponse.IsOk)
{
var pr = prResponse.Result;
_prId = pr.Id; // Issues can become PRs
_prHeadSha = pr.Head.Sha;
_prBaseBranch = pr.Base.Ref;
_prMergeableStateBlocked = pr.MergeableState == "blocked";
await updater.UpdatePullRequests(_repoId, prResponse.Date, new[] { prResponse.Result });
}
_prMetadata = GitHubMetadata.FromResponse(prResponse);
// Branch Protection
if (_prMergeableStateBlocked && _prBaseBranch != null)
{
_grainFactory.GetGrain <IRepositoryActor>(_repoId).SyncProtectedBranch(_prBaseBranch, forUserId).LogFailure();
}
}
public override async Task OnActivateAsync()
{
// Set this first as subsequent calls require it.
_userId = this.GetPrimaryKeyLong();
// Ensure this user actually exists, and lookup their token.
User user = null;
using (var context = _contextFactory.CreateInstance()) {
user = await context.Users
.AsNoTracking()
.Include(x => x.Tokens)
.SingleOrDefaultAsync(x => x.Id == _userId);
}
if (user == null)
{
throw new InvalidOperationException($"User {_userId} does not exist and cannot be activated.");
}
if (!user.Tokens.Any())
{
throw new InvalidOperationException($"User {_userId} has an invalid token and cannot be activated.");
}
_github = _grainFactory.GetGrain <IGitHubActor>(user.Id);
_mentionMetadata = user.MentionMetadata;
_mentionSince = user.MentionSince ?? EpochUtility.EpochOffset;
// Always sync while active
_lastSyncInterest = DateTimeOffset.UtcNow;
_syncTimer = RegisterTimer(SyncTimerCallback, null, TimeSpan.Zero, SyncDelay);
await base.OnActivateAsync();
}
private async Task <(ISet <long> IssueCommentIds, ISet <long> CommitCommentIds)> UpdateIssueTimeline(IGitHubActor ghc, long forUserId, DataUpdater updater)
{
///////////////////////////////////////////
/* NOTE!
* We can't sync the timeline incrementally, because the client wants commit and
* reference data inlined. This means we always have to download all the
* timeline events in case an old one now has updated data. Other options are to
* just be wrong, or to simply reference the user by id and mark them referenced
* by the repo.
*/
//////////////////////////////////////////
var issueCommentIds = new HashSet <long>();
var commitCommentIds = new HashSet <long>();
// TODO: cache per-user
// TODO: If caching, are there things that should occur every time anyway?
var timelineResponse = await ghc.Timeline(_repoFullName, _issueNumber, _issueId, priority : RequestPriority.Interactive);
if (timelineResponse.IsOk)
{
var allEvents = timelineResponse.Result;
var filteredEvents = allEvents.Where(x => !_IgnoreTimelineEvents.Contains(x.Event)).ToArray();
// For adding to the DB later
// TODO: Technically this can pick stale accounts if an old and new version are both in the collection.
// My batching branch fixes this by tracking versions
// Set must allow nulls
var accounts = new HashSet <gm.Account>(KeyEqualityComparer.FromKeySelector((gm.Account x) => x?.Id));
foreach (var timelineEvent in filteredEvents)
{
accounts.Add(timelineEvent.Actor);
accounts.Add(timelineEvent.Assignee);
}
await LookupEventCommitDetails(ghc, accounts, filteredEvents);
await LookupEventSourceDetails(ghc, accounts, filteredEvents);
// Fixup and sanity checks
foreach (var item in filteredEvents)
{
switch (item.Event)
{
case "cross-referenced":
if (item.Source?.Actor != null)
{
accounts.Add(item.Source?.Actor);
}
if (item.Source?.Issue?.User != null)
{
accounts.Add(item.Source?.Issue?.User);
}
//if (item.Actor != null) { // It's a comment reference
// accounts.Add(item.Source?.Actor);
// item.Actor = item.Source?.Actor;
//} else { // It's an issue reference
// accounts.Add(item.Source?.Issue?.User);
// item.Actor = item.Source?.Issue?.User;
//}
break;
case "committed":
item.CreatedAt = item.ExtensionDataDictionary["committer"]["date"].ToObject <DateTimeOffset>();
break;
default:
// Leave most things alone.
break;
}
if (item.CreatedAt == DateTimeOffset.MinValue)
{
throw new Exception($"Unable to process event of type {item.Event} on {_repoFullName}/{_issueNumber} ({_issueId}). Invalid date.");
}
}
await updater.UpdateTimelineEvents(_repoId, timelineResponse.Date, forUserId, accounts, filteredEvents);
// Comments
var commentEvents = allEvents.Where(x => x.Event == "commented").ToArray();
if (commentEvents.Any())
{
// The events have all the info we need.
var comments = commentEvents.Select(x => x.Roundtrip <gm.IssueComment>(serializerSettings: GitHubSerialization.JsonSerializerSettings)).ToArray();
// Update known ids
issueCommentIds.UnionWith(comments.Select(x => x.Id));
await updater.UpdateIssueComments(_repoId, timelineResponse.Date, comments);
}
// Commit Comments
// Comments in commit-commented events look complete.
// Let's run with it.
var commitCommentEvents = allEvents.Where(x => x.Event == "commit-commented").ToArray();
if (commitCommentEvents.Any())
{
var commitComments = commitCommentEvents
.SelectMany(x => x.ExtensionDataDictionary["comments"].ToObject <IEnumerable <gm.CommitComment> >(GitHubSerialization.JsonSerializer))
.ToArray();
// Update known ids
commitCommentIds.UnionWith(commitComments.Select(x => x.Id));
await updater.UpdateCommitComments(_repoId, timelineResponse.Date, commitComments);
}
// Merged event commit statuses
if (_isPullRequest)
{
var mergedEvent = allEvents
.Where(x => x.Event == "merged")
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
var mergeCommitId = mergedEvent?.CommitId;
if (mergeCommitId != null)
{
var mergeCommitStatusesResponse = await ghc.CommitStatuses(_repoFullName, mergeCommitId, _prMergeStatusMetadata, RequestPriority.Interactive);
if (mergeCommitStatusesResponse.IsOk)
{
await updater.UpdateCommitStatuses(_repoId, mergeCommitId, mergeCommitStatusesResponse.Result);
}
_prMergeStatusMetadata = GitHubMetadata.FromResponse(mergeCommitStatusesResponse);
}
}
}
return(IssueCommentIds : issueCommentIds, CommitCommentIds : commitCommentIds);
}
public override async Task OnActivateAsync()
{
// Set this first as subsequent calls require it.
_userId = this.GetPrimaryKeyLong();
// Ensure this user actually exists, and lookup their token.
User user = null;
using (var context = _contextFactory.CreateInstance()) {
user = await context.Users
.AsNoTracking()
.Include(x => x.Tokens)
.Include(x => x.Settings)
.Include(x => x.LinkedRepositories)
.Include(x => x.SyncRepositories)
.Include(x => x.AccountOrganizations)
.SingleOrDefaultAsync(x => x.Id == _userId);
}
if (user == null)
{
throw new InvalidOperationException($"User {_userId} does not exist and cannot be activated.");
}
if (!user.Tokens.Any())
{
throw new InvalidOperationException($"User {_userId} has an invalid token and cannot be activated.");
}
_userInfo = $"{user.Login} ({user.Id})";
_github = _grainFactory.GetGrain <IGitHubActor>(user.Id);
_mentions = _grainFactory.GetGrain <IMentionsActor>(user.Id);
_metadata = user.Metadata;
_repoMetadata = user.RepositoryMetadata;
_orgMetadata = user.OrganizationMetadata;
_syncSettings = user.Settings?.SyncSettings;
_linkedRepos = user.LinkedRepositories.Select(x => x.RepositoryId).ToHashSet();
_orgActors = user.AccountOrganizations
.ToDictionary(x => x.OrganizationId, x => _grainFactory.GetGrain <IOrganizationActor>(x.OrganizationId));
_repoActors = user.SyncRepositories
.ToDictionary(x => x.RepositoryId, x => _grainFactory.GetGrain <IRepositoryActor>(x.RepositoryId));
_includeRepoMetadata = user.SyncRepositories
.Where(x => !_linkedRepos.Contains(x.RepositoryId))
.ToDictionary(x => x.RepositoryId, x => x.RepositoryMetadata);
// Migration Step
// No settings + linked repos + empty sync repos == MIGRATE!
if (_linkedRepos.Any() && _syncSettings == null && !_repoActors.Any())
{
Interlocked.Increment(ref _linkedReposDesired);
Interlocked.Increment(ref _syncReposDesired);
}
// We always want to sync while the UserActor is loaded;
_lastSyncInterest = DateTimeOffset.UtcNow;
_syncTimer = RegisterTimer(SyncTimerCallback, null, TimeSpan.Zero, SyncDelay);
await base.OnActivateAsync();
}
public async Task <bool> InternalSync(DataUpdater updater)
{
if (_syncLimit.CurrentCount != 0)
{
throw new InvalidOperationException($"{nameof(InternalSync)} requires the sync semaphore be held.");
}
var metaDataMeaningfullyChanged = false;
try {
// NOTE: The following requests are (relatively) infrequent and important for access control (repos/orgs)
// Give them high priority.
// Update this user's org memberships
if (_orgMetadata.IsExpired())
{
var orgs = await _github.OrganizationMemberships(cacheOptions : _orgMetadata, priority : RequestPriority.Interactive);
if (orgs.IsOk)
{
metaDataMeaningfullyChanged = true;
await updater.SetUserOrganizations(_userId, orgs.Date, orgs.Result);
_orgActors = orgs.Result
.ToDictionary(x => x.Organization.Id, x => _grainFactory.GetGrain <IOrganizationActor>(x.Organization.Id));
// Also re-evaluate their linked repos
Interlocked.Increment(ref _linkedReposDesired);
}
// Don't update until saved.
_orgMetadata = GitHubMetadata.FromResponse(orgs);
}
// Update this user's repo memberships
var savedLinkCurrent = _linkedReposCurrent;
var savedLinkDesired = _linkedReposDesired;
if ((savedLinkCurrent < savedLinkDesired) || _repoMetadata.IsExpired())
{
var repos = await _github.Repositories(_repoMetadata, RequestPriority.Interactive);
if (repos.IsOk)
{
metaDataMeaningfullyChanged = true;
Interlocked.Increment(ref _syncReposDesired);
await updater.SetUserRepositories(_userId, repos.Date, repos.Result);
_linkedRepos = repos.Result.Select(x => x.Id).ToHashSet();
// don't update _repoActors yet
}
// Don't update until saved.
_repoMetadata = GitHubMetadata.FromResponse(repos);
Interlocked.CompareExchange(ref _linkedReposCurrent, savedLinkDesired, savedLinkCurrent);
}
// Update this user's sync repos
var savedReposCurrent = _syncReposCurrent;
var savedReposDesired = _syncReposDesired;
if (savedReposCurrent < savedReposDesired)
{
IEnumerable <g.Repository> updateRepos = null;
IDictionary <long, GitHubMetadata> combinedRepoMetadata = null;
var date = DateTimeOffset.UtcNow; // Not *technically* correct, but probably ok
if (_syncSettings?.Include.Any() == true)
{
// Request all "included" repos to verify access
var repoReqs = _syncSettings.Include
.Where(x => !_linkedRepos.Contains(x)) // Exclude linked repos (access already known)
.Select(x => (RepoId: x, Request: _github.Repository(x, _includeRepoMetadata.Val(x), RequestPriority.Interactive)))
.ToArray();
await Task.WhenAll(repoReqs.Select(x => x.Request));
// Collect the "successful" responses
// Check explicitly for 404, since 502s are so common :/
var successful = repoReqs
.Where(x => x.Request.Status == TaskStatus.RanToCompletion)
.Where(x => x.Request.Result.Status != HttpStatusCode.NotFound)
.ToArray();
_includeRepoMetadata = successful.ToDictionary(x => x.RepoId, x => GitHubMetadata.FromResponse(x.Request.Result));
updateRepos = successful
.Where(x => x.Request.Result.IsOk)
.Select(x => x.Request.Result.Result)
.ToArray();
// now union/intersect all the things
combinedRepoMetadata = new Dictionary <long, GitHubMetadata>();
foreach (var repoId in _syncSettings.Include)
{
if (_linkedRepos.Contains(repoId))
{
combinedRepoMetadata.Add(repoId, null);
}
else if (_includeRepoMetadata.ContainsKey(repoId))
{
combinedRepoMetadata.Add(repoId, _includeRepoMetadata[repoId]);
}
// else drop it
}
}
var syncRepoMetadata = await updater.UpdateAccountSyncRepositories(
_userId,
_syncSettings?.AutoTrack ?? true,
date,
updateRepos,
combinedRepoMetadata,
_syncSettings?.Exclude);
_repoActors = syncRepoMetadata.Keys
.ToDictionary(x => x, x => _grainFactory.GetGrain <IRepositoryActor>(x));
_includeRepoMetadata = syncRepoMetadata
.Where(x => !_linkedRepos.Contains(x.Key))
.ToDictionary(x => x.Key, x => x.Value);
// TODO: Actually detect changes.
var cs = new ChangeSummary();
cs.Add(userId: _userId);
updater.UnionWithExternalChanges(cs);
Interlocked.CompareExchange(ref _syncReposCurrent, savedReposDesired, savedReposCurrent);
}
} catch (GitHubRateException) {
// nothing to do
}
// We do this here so newly added repos and orgs sync immediately
// Sync repos
foreach (var repo in _repoActors.Values)
{
repo.Sync().LogFailure(_userInfo);
}
// Sync orgs
foreach (var org in _orgActors.Values)
{
org.Sync().LogFailure(_userInfo);
}
return(metaDataMeaningfullyChanged);
}
private async Task UpdatePullRequestReviews(IGitHubActor ghc, long forUserId, DataUpdater updater)
{
gm.Review myReview = null;
// Reviews and Review comments need to use the current user's token. Don't track metadata (yet - per user ideally)
var prReviewsResponse = await ghc.PullRequestReviews(_repoFullName, _issueNumber, priority : RequestPriority.Interactive);
if (prReviewsResponse.IsOk)
{
await updater.UpdateReviews(_repoId, _issueId, prReviewsResponse.Date, prReviewsResponse.Result, userId : forUserId, complete : true);
myReview = prReviewsResponse.Result
.Where(x => x.State.Equals("PENDING", StringComparison.OrdinalIgnoreCase))
.FirstOrDefault(x => x.User.Id == forUserId);
}
using (var context = _contextFactory.CreateInstance()) {
await context.UpdateMetadata("PullRequests", "IssueId", "ReviewMetadataJson", _issueId, GitHubMetadata.FromResponse(prReviewsResponse));
}
// Only fetch if *this user* has a pending review
// Since we make the request every time, it's ok not to look for pending reviews in the DB
if (myReview != null)
{
var reviewCommentsResponse = await ghc.PullRequestReviewComments(_repoFullName, _issueNumber, myReview.Id, priority : RequestPriority.Interactive);
if (reviewCommentsResponse.IsOk && reviewCommentsResponse.Result.Any())
{
await updater.UpdatePullRequestComments(_repoId, _issueId, reviewCommentsResponse.Date, reviewCommentsResponse.Result, pendingReviewId : myReview.Id);
}
}
}
