public async static Task PerformTaskAsync( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IUserRepoSearchActor userRepoSearchActor, IReliableConcurrentQueue <ScrapingTask> firstTaskQueue, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue) { switch (scrapingTask.Type) { case ScrapingTaskType.Repository: await ScrapeRepositoryAsync( scrapingTask : scrapingTask, gitHubClient : gitHubClient, tx : tx, userRepoSearchActor : userRepoSearchActor, secondTaskQueue : secondTaskQueue); break; case ScrapingTaskType.UserInfo: await ScrapeUserInfoAsync( scrapingTask : scrapingTask, gitHubClient : gitHubClient, tx : tx, userRepoSearchActor : userRepoSearchActor, secondTaskQueue : secondTaskQueue); break; case ScrapingTaskType.FollowingRepositories: await ScrapeFollowingRepositoriesAsync( scrapingTask : scrapingTask, gitHubClient : gitHubClient, tx : tx, userRepoSearchActor : userRepoSearchActor, secondTaskQueue : secondTaskQueue); break; } }
private async static Task ScrapeRepositoryAsync( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IUserRepoSearchActor userRepoSearchActor, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue) { ScheduledRepository scheduledRepository = scrapingTask.ScheduledRepositories[0]; Result <Repository> result = await gitHubClient .ScrapeRepositoryAsync( authToken : scrapingTask.AuthToken, owner : scheduledRepository.Owner, name : scheduledRepository.Name); if (scrapingTask.ScheduledRepositories.Count > 1) { ScrapingTask nextScrapingTask = Serializer.DeepCopy(scrapingTask); nextScrapingTask.ScheduledRepositories = nextScrapingTask .ScheduledRepositories.Skip(1).ToList(); await secondTaskQueue.EnqueueAsync(tx, nextScrapingTask); } if (result.Success) { result.Data.IsNew = false; await userRepoSearchActor.SetRepositoryAsync(result.Data); } }
private async static Task ProcessScrapingTask( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IReliableConcurrentQueue <ScrapingTask> firstTaskQueue, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue, IUserRepoSearchActorProvider userRepoSearchActorProvider) { IUserRepoSearchActor userRepoSearchActor = userRepoSearchActorProvider.Provide(scrapingTask.UserLogin); await Scraper.PerformTaskAsync( scrapingTask : scrapingTask, gitHubClient : gitHubClient, tx : tx, userRepoSearchActor : userRepoSearchActor, firstTaskQueue : firstTaskQueue, secondTaskQueue : secondTaskQueue); }
private async static Task ScrapeUserInfoAsync( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IUserRepoSearchActor userRepoSearchActor, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue) { Result <UserInfo> result = await gitHubClient .GetUserInfoAsync(scrapingTask.AuthToken); if (result.Success) { await userRepoSearchActor.SetUserInfoAsync(result.Data); await secondTaskQueue.EnqueueAsync(tx, new ScrapingTask { Type = ScrapingTaskType.FollowingRepositories, AuthToken = scrapingTask.AuthToken, UserLogin = scrapingTask.UserLogin }); } }
private async static Task ScrapeFollowingRepositoriesAsync( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IUserRepoSearchActor userRepoSearchActor, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue) { Result <UserInfo> userInfoResult = await gitHubClient .GetUserInfoAsync(scrapingTask.AuthToken); if (!userInfoResult.Success) { return; } Result <IEnumerable <Repository> > repositoriesResult = await gitHubClient .ScrapeFollowingRepositoriesAsync(scrapingTask.AuthToken); if (!repositoriesResult.Success) { return; } List <ScheduledRepository> scheduledRepositories = repositoriesResult.Data.Select( repository => new ScheduledRepository { Name = repository.Name, Owner = repository.Owner }).ToList(); await secondTaskQueue.EnqueueAsync(tx, new ScrapingTask { Type = ScrapingTaskType.Repository, AuthToken = scrapingTask.AuthToken, UserLogin = scrapingTask.UserLogin, ScheduledRepositories = scheduledRepositories }); }