public async Task <CursorPagedResults <UserDTO> > GetUsers(ReadUsers readUsers)
{
var result = new CursorPagedResults <UserDTO>();
if (readUsers.Pagination.FetchNext > 0)
{
result.Result = await context.Users.AsNoTracking().Where(x => x.UserId > readUsers.Pagination.Cursor).Take(readUsers.Pagination.FetchNext).ProjectTo <UserDTO>(mapperConfiguration).ToListAsync();
result.Cursor = readUsers.Pagination.Cursor;
result.NextCursor = result.Result.LastOrDefault()?.UserId;
}
else
{
var users = await context.Users.AsNoTracking().Where(x => x.UserId <= readUsers.Pagination.Cursor).TakeLast(readUsers.Pagination.FetchNext - 1).ProjectTo <UserDTO>(mapperConfiguration).ToListAsync();
if (users.Count() != Math.Abs(readUsers.Pagination.FetchNext))
{
result.Result = users;
result.Cursor = 0;
}
else
{
result.Result = users.Skip(1).ToList();
result.Cursor = result.Result.FirstOrDefault()?.UserId;
}
result.NextCursor = readUsers.Pagination.Cursor;
}
return(result);
}
public async Task <CursorPagedResults <RepositorySummary> > ReadRepositorySummariesAsync(string owner, int take, string endCursor)
{
var respositorySummaries = new CursorPagedResults <RepositorySummary>();
var ownerType = await ReadOwnerType(owner).ConfigureAwait(false);
if (ownerType == OwnerType.Organization)
{
respositorySummaries = await repositorySourceRepository.ReadRepositorySummariesAsync(owner, null, take, endCursor).ConfigureAwait(false);
}
else if (ownerType == OwnerType.User)
{
respositorySummaries = await repositorySourceRepository.ReadRepositorySummariesAsync(null, owner, take, endCursor).ConfigureAwait(false);
}
return(respositorySummaries);
}
public async Task OrchestrateAsync(RunTimeConfiguration config)
{
bool moreRepostoriesToRead = false;
var sourceRepositoriesRead = 0;
var sourceRepositoriesAnalyzed = 0;
var repositoryAnalysisErrors = new List <(string repoName, string errorMessage, string errorStackTrace)>();
try
{
var userOrOranizationNameQueryStringKey = string.Empty;
var userOrOganziationNameQueryStringValue = string.Empty;
if (!string.IsNullOrWhiteSpace(config.User))
{
userOrOranizationNameQueryStringKey = "user";
userOrOganziationNameQueryStringValue = config.User;
}
else
{
userOrOranizationNameQueryStringKey = "organization";
userOrOganziationNameQueryStringValue = config.Organization;
}
restClient.BaseUrl = new Uri(config.Url);
string endCursor = null;
var stopWatch = Stopwatch.StartNew();
do
{
logger.Information($"Reading next batch of {config.BatchSize} repositories for login {config.User ?? config.Organization}");
CursorPagedResults <RepositorySummary> results = null;
if (endCursor != null)
{
var request = new RestRequest("/api/repositorysource/repositories");
request.AddQueryParameter("owner", userOrOganziationNameQueryStringValue);
request.AddQueryParameter("take", config.BatchSize.ToString());
request.AddQueryParameter("endCursor", endCursor);
var response = await restClient.ExecuteTaskAsync <CursorPagedResults <RepositorySummary> >(request);
if (!response.IsSuccessful)
{
throw new ArgumentException($"{response.StatusDescription} - {response.ErrorMessage}");
}
results = response.Data;
}
else
{
var request = new RestRequest("/api/repositorysource/repositories");
request.AddQueryParameter("owner", userOrOganziationNameQueryStringValue);
request.AddQueryParameter("take", config.BatchSize.ToString());
var response = await restClient.ExecuteTaskAsync <CursorPagedResults <RepositorySummary> >(request);
if (!response.IsSuccessful)
{
throw new ArgumentException($"{response.StatusDescription} - {response.ErrorMessage}");
}
results = response.Data;
}
sourceRepositoriesRead += results.Results.Count();
endCursor = results.EndCursor;
moreRepostoriesToRead = results.MoreToRead;
var repositoryAnalysisTasks = new List <Task>();
using (var semaphore = new SemaphoreSlim(config.Concurrency))
{
foreach (var result in results.Results)
{
// await here until there is a room for this task
await semaphore.WaitAsync();
repositoryAnalysisTasks.Add(SendAnalysisRequest(semaphore, result));
}
await Task.WhenAll(repositoryAnalysisTasks);
}
logger.Information($"Finished analyizing batch of {config.BatchSize} repositories. {sourceRepositoriesAnalyzed} respositories analyzed thus far");
} while (moreRepostoriesToRead);
stopWatch.Stop();
logger.Information($"\nAnalyized {sourceRepositoriesAnalyzed} out of {sourceRepositoriesRead} repositories in {stopWatch.Elapsed.TotalMinutes} minutes");
logger.Information($"\nThere were {repositoryAnalysisErrors.Count} analyisis errors");
foreach (var repositoryAnalysisError in repositoryAnalysisErrors)
{
logger.Error($"{repositoryAnalysisError.repoName} - {repositoryAnalysisError.errorMessage}");
}
Console.WriteLine("\nExecution complete!");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
}
catch (Exception ex)
{
logger.Error($"FATAL EXCEPTION OCCURRED! {ex.Message}");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
}
async Task SendAnalysisRequest(SemaphoreSlim semaphore, RepositorySummary repositorySummary)
{
try
{
if (config.AsOf.HasValue)
{
logger.Information($"Starting analysis of {repositorySummary.Url} as of {config.AsOf.Value.ToString("F")}");
}
else
{
logger.Information($"Starting analysis of {repositorySummary.Url}");
}
var repositoryAnalysis = new RepositoryAnalysis
{
ForceCompleteRefresh = config.RefreshAll,
RepositoryLastUpdatedOn = repositorySummary.UpdatedAt,
RepositoryId = repositorySummary.Url,
AsOf = config.AsOf
};
var request = new RestRequest("/api/repositoryanalysis/", Method.POST);
request.AddJsonBody(repositoryAnalysis);
if (sourceRepositoriesAnalyzed == 0)
{
// If no requests have been completed then set the timeout to be higher as if an organization
// is being targeted then the reading of all the team information can take a few minutes.
request.Timeout = config.FirstApiCallTimeout;
}
var response = await restClient.ExecuteTaskAsync(request);
if (!response.IsSuccessful)
{
if (response.ErrorMessage != null && response.ErrorMessage.StartsWith("No connection could be made because the target machine actively refused it"))
{
logger.Error($"UNABLE TO REACH API!!");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
return;
}
else
{
logger.Error(response?.ErrorMessage);
repositoryAnalysisErrors.Add((repositorySummary.Url, response.StatusDescription, null));
}
}
}
catch (Exception ex)
{
repositoryAnalysisErrors.Add((repositorySummary.Url, ex.Message, ex.StackTrace));
}
finally
{
semaphore.Release();
Interlocked.Increment(ref sourceRepositoriesAnalyzed);
}
}
}
public async Task <CursorPagedResults <RepositorySummary> > ReadRepositorySummariesAsync(string organization, string user, int take, string endCursor)
{
string loginType = null;
string login = null;
var query = @"
query ($login: String!, $take: Int, $after: String) {
#LOGIN_TYPE#(login: $login) {
repositories(first: $take, after: $after, orderBy: {field: PUSHED_AT, direction: DESC}) {
nodes {
url
createdAt
pushedAt
}
pageInfo {
hasNextPage
endCursor
}
}
}
}
";
if (!string.IsNullOrWhiteSpace(user))
{
loginType = "user";
login = user;
}
else
{
loginType = "organization";
login = organization;
}
query = query.Replace("#LOGIN_TYPE#", loginType);
var variables = new { login = login, take = take, after = endCursor };
GraphQlNodesParent <Model.Github.GraphQL.Repository> graphQLRepositories = null;
if (loginType == "user")
{
var graphQLUser = await graphQLClient.QueryAsync <Model.Github.GraphQL.User>(query, variables).ConfigureAwait(false);
graphQLRepositories = graphQLUser.Repositories;
}
else
{
var graphQLOrganization = await graphQLClient.QueryAsync <Model.Github.GraphQL.Organization>(query, variables).ConfigureAwait(false);
graphQLRepositories = graphQLOrganization.Repositories;
}
var cursorPagedResults = new CursorPagedResults <RepositorySummary>();
cursorPagedResults.EndCursor = graphQLRepositories.PageInfo.EndCursor;
cursorPagedResults.MoreToRead = graphQLRepositories.PageInfo.HasNextPage;
var results = new List <RepositorySummary>();
foreach (var graphQLRepository in graphQLRepositories.Nodes)
{
var repositorySummary = new RepositorySummary
{
CreatedAt = graphQLRepository.CreatedAt,
UpdatedAt = graphQLRepository.PushedAt.Value,
Url = graphQLRepository.Url
};
results.Add(repositorySummary);
}
cursorPagedResults.Results = results;
return(cursorPagedResults);
}
