/// <summary>
/// Gets random repository according to search parameters.
/// </summary>
/// <param name="parameters">Search parameters.</param>
/// <returns>Random repository.</returns>
public async Task <Repository> GetRandomRepositoryAsync(RepositoryParameters parameters)
{
Ensure.ArgumentNotNull(parameters, nameof(parameters));
// cache total count value to avoid excessive API requests
int repositoryCount = await _repositoryApiService.GetRepositoriesCount(parameters);
if (repositoryCount == 0)
{
return(null);
}
var index = Random.Next(repositoryCount - 1);
var queries = await GetQueries(parameters, index, index, MAX_REPOSITORIES_COUNT);
var queryParams = queries.FirstOrDefault();
if (queryParams == null)
{
throw new Exception("Unexpected exception occured. Try different parameters");
}
var repositories = await _repositoryApiService.GetRepositoriesAsync(queryParams.Parameters,
queryParams.From, 1);
return(repositories.FirstOrDefault());
}
public async Task <int> GetRepositoryStarsCount(RepositoryParameters parameters, int index)
{
string query = RepositorySearchToQuery(parameters);
// cache stars count value to avoid excessive API requests
var cacheKey = $"StarsCount-{query}";
var cachedValue = _cacheService.GetValue <int?>(cacheKey);
int starsCount;
if (cachedValue.HasValue)
{
starsCount = cachedValue.Value;
}
else
{
var searchResult = await this._client.Query <QueryData>(RepositoryQueries.REPOSITORY_STARS_QUERY,
new
{
query = query,
first = 1,
after = GraphQLHelper.GetCursor(index - 1)
});
if (searchResult.Search.Edges.Count == 0)
{
throw new Exception("Repository is not found");
}
starsCount = searchResult.Search.Edges[0].Node.StargazerCount;
_cacheService.SetValue(cacheKey, starsCount, TimeSpan.FromMinutes(STARS_COUNT_CACHE_EXPIRATION_TIME_IN_MINUTES));
}
return(starsCount);
}
public async Task RepositoryService_GetQueries_SelectAllWithMaxCreated()
{
var parameters = new RepositoryParameters();
parameters.MaxCreated = DateTime.Now.AddMonths(-3);
var expectedResult = this._repositories.Where(r => r.CreatedDate <= parameters.MaxCreated).ToList();
int from = 1;
int to = expectedResult.Count;
int max = 3;
var queries = await this._repositoryService.GetQueries(parameters, from, to, max);
var result = new List <Repository>();
foreach (var query in queries)
{
var repositories = await this._repositoryApiService.GetRepositoriesAsync(query.Parameters,
query.From, (query.To - query.From + 1));
result.AddRange(repositories);
}
var repos = _repositories.OrderByDescending(r => r.StarsCount).ThenByDescending(r => r.UpdatedDate);
Assert.AreEqual(to - from + 1, result.Count);
CollectionAssert.AreEquivalent(expectedResult, result);
}
public Task <int> GetRepositoryStarsCount(RepositoryParameters parameters, int index)
{
return(Task.FromResult(this._repositories
.Where(r => CheckRepository(r, parameters))
.OrderByDescending(r => r.StarsCount)
.ThenByDescending(r => r.UpdatedDate)
.ToList()[index - 1].StarsCount));
}
public ucHome()
{
InitializeComponent();
rp = new RepositoryParameters();
RefreshRepository();
}
public Task <List <Repository> > GetRepositoriesAsync(RepositoryParameters parameters, int from, int count)
{
return(Task.FromResult(this._repositories
.Where(r => CheckRepository(r, parameters))
.OrderByDescending(r => r.StarsCount)
.ThenByDescending(r => r.UpdatedDate)
.Skip(from - 1)
.Take(count)
.ToList()));
}
private string RepositorySearchToQuery(RepositoryParameters search)
{
string query = string.Empty;
if (!string.IsNullOrEmpty(search.Text))
{
query += search.Text;
}
if (!string.IsNullOrEmpty(search.Language))
{
query += $" language:{search.Language}";
}
if (search.LastUpdateAfter.HasValue)
{
query += $" pushed:>={search.LastUpdateAfter.Value.ToString("yyyy-MM-dd")}";
}
if (search.HasGoodFirstIssues.HasValue && search.HasGoodFirstIssues.Value)
{
query += $" good-first-issues:>0";
}
if (search.HasHelpWantedIssues.HasValue && search.HasHelpWantedIssues.Value)
{
query += $" help-wanted-issues:>0";
}
if (search.MinNumberOfStars.HasValue && search.MaxNumberOfStars.HasValue)
{
query += $" stars:{search.MinNumberOfStars.Value}..{search.MaxNumberOfStars.Value}";
}
else if (search.MinNumberOfStars.HasValue)
{
query += $" stars:>={search.MinNumberOfStars.Value}";
}
else if (search.MaxNumberOfStars.HasValue)
{
query += $" stars:<={search.MaxNumberOfStars.Value}";
}
if (search.MinCreated.HasValue && search.MaxCreated.HasValue)
{
query += $" created:{search.MinCreated.Value.ToString("yyyy-MM-ddTHH:mm:ss")}..{search.MaxCreated.Value.ToString("yyyy-MM-ddTHH:mm:ss")}";
}
else if (search.MinCreated.HasValue)
{
query += $" created:>={search.MinCreated.Value.ToString("yyyy-MM-ddTHH:mm:ss")}";
}
else if (search.MaxCreated.HasValue)
{
query += $" created:<={search.MaxCreated.Value.ToString("yyyy-MM-ddTHH:mm:ss")}";
}
// TODO: setting up sorting in UI
query += " sort:stars-desc sort:updated-desc";
return(query);
}
/// <summary>
/// Получить посуточную статистику по переданным дням
/// </summary>
internal IEnumerable <PeriodStatisticsDto> GetStatisticsByDates(RepositoryParameters pars, IEnumerable <DateTime> dates)
{
_logger.LogInformation($"Getting statistics started: \"{pars.RepoPath}\"");
using var repo = new Repository(pars.RepoPath);
var commits = GetCommits(repo, dates, false);
var result = dates.Select(date => GetDayStatistics(repo, pars.Name, pars.WebUI, commits, date)).ToList();
_logger.LogInformation($"Getting statistics ended: \"{pars.RepoPath}\"");
return(result);
}
public async Task <List <Repository> > GetRepositoriesAsync(RepositoryParameters parameters,
int from, int count)
{
string query = RepositorySearchToQuery(parameters);
var searchResult = await this._client.Query <QueryData>(RepositoryQueries.REPOSITORY_SEARCH_QUERY,
new
{
query = query,
first = count,
after = GraphQLHelper.GetCursor(from - 1),
languageCount = MAX_LANGUAGES_COUNT
});
return(searchResult.Search.Edges.Select(e => ConvertEdgeToRepository(e)).ToList());
}
public async Task <IActionResult> Random([FromQuery] RepositoryParameters parameters)
{
if (parameters == null ||
(parameters.HasGoodFirstIssues == null || parameters.HasGoodFirstIssues == false) &&
(parameters.HasHelpWantedIssues == null || parameters.HasHelpWantedIssues == false) &&
parameters.Language == null &&
parameters.LastUpdateAfter == null &&
parameters.MinNumberOfStars == null &&
string.IsNullOrEmpty(parameters.Text))
{
return(BadRequest("Request parameters should be set"));
}
var repository = await this._repositoryService.GetRandomRepositoryAsync(parameters);
return(Ok(new { Repository = repository }));
}
public async Task RepositoryService_GetQueries_LessThanMax()
{
var parameters = new RepositoryParameters();
int from = 0;
int to = 3;
int max = 3;
var queries = await this._repositoryService.GetQueries(parameters, from, to, max);
Assert.AreEqual(1, queries.Count);
Assert.AreEqual(from, queries[0].From);
Assert.AreEqual(to, queries[0].To);
Assert.AreEqual(parameters.MinNumberOfStars, queries[0].Parameters.MinNumberOfStars);
Assert.AreEqual(parameters.MaxNumberOfStars, queries[0].Parameters.MaxNumberOfStars);
Assert.AreEqual(parameters.MinCreated, queries[0].Parameters.MinCreated);
Assert.AreEqual(parameters.MaxCreated, queries[0].Parameters.MaxCreated);
}
public async Task RepositoryService_GetQueries_SimpleStarsCountSkip()
{
var parameters = new RepositoryParameters();
int max = 3;
int from = 8;
int to = from + max - 1;
var queries = await this._repositoryService.GetQueries(parameters, from, to, max);
Assert.AreEqual(1, queries.Count);
Assert.AreEqual(1, queries[0].From);
Assert.AreEqual(max, queries[0].To);
Assert.IsNull(parameters.MinNumberOfStars);
Assert.AreEqual(parameters.MaxNumberOfStars, 2);
Assert.IsNull(parameters.MinCreated);
Assert.IsNull(parameters.MaxCreated);
}
public async Task <IActionResult> Search([FromQuery] RepositoryParameters parameters)
{
if (parameters == null ||
(parameters.HasGoodFirstIssues == null || parameters.HasGoodFirstIssues == false) &&
(parameters.HasHelpWantedIssues == null || parameters.HasHelpWantedIssues == false) &&
parameters.Language == null &&
parameters.LastUpdateAfter == null &&
parameters.MinNumberOfStars == null &&
string.IsNullOrEmpty(parameters.Text))
{
return(BadRequest("Request parameters should be set"));
}
if (!parameters.PageNumber.HasValue || !parameters.PageSize.HasValue)
{
return(BadRequest("Pagination parameters should be set"));
}
var repositoryPage = await this._repositoryService.GetRepositoriesAsync(parameters);
return(Ok(repositoryPage));
}
/// <summary>
/// Получить оценку рабочего
/// </summary>
/// <param name="repositoryParameters"></param>
/// <param name="dates"></param>
/// <returns></returns>
private RepositoryWorkEstimateDto GetRepositoryWorkEstimate(RepositoryParameters repositoryParameters, IEnumerable <DateTime> dates)
{
_logger.LogInformation($"Getting work estimates started: \"{repositoryParameters.RepoPath}\"");
using var repo = new Repository(repositoryParameters.RepoPath);
var commits = GetCommits(repo, dates, true);
var estimates = commits.Select(c => new
{
Email = c.Author.Email.ToLower(),
Commit = c
})
.GroupBy(key => key.Email)
.Select(g => new
{
Email = g.Key,
//Commits = g.Select(v => v.Commit).ToList(),
Hours = EstimateHours(g.Select(v => v.Commit.Author.When))
})
.Select(i => new PersonWorkEstimateDto
{
Email = i.Email,
//i.Commits,
Hours = Math.Round(i.Hours, 2),
Days = Math.Round(i.Hours / 8, 2)
})
.ToList();
var result = new RepositoryWorkEstimateDto
{
RepositoryName = repositoryParameters.Name,
Estimates = estimates
};
_logger.LogInformation($"Getting work estimates ended: \"{repositoryParameters.RepoPath}\"");
return(result);
}
public async Task <int> GetRepositoriesCount(RepositoryParameters parameters)
{
string query = RepositorySearchToQuery(parameters);
// cache total count value to avoid excessive API requests
var cacheKey = $"TotalCount-{query}";
var cachedValue = _cacheService.GetValue <int?>(cacheKey);
int repositoryCount;
if (cachedValue.HasValue)
{
repositoryCount = cachedValue.Value;
}
else
{
var totalCountResult = await this._client.Query <QueryData>(RepositoryQueries.REPOSITORIES_COUNT_QUERY,
new { query = query });
repositoryCount = totalCountResult.Search.RepositoryCount;
_cacheService.SetValue(cacheKey, repositoryCount, TimeSpan.FromMinutes(TOTAL_COUNT_CACHE_EXPIRATION_TIME_IN_MINUTES));
}
return(repositoryCount);
}
public async Task RepositoryService_GetQueries_SelectAll()
{
var parameters = new RepositoryParameters();
int from = 1;
int to = this._repositories.Count;
int max = 3;
var queries = await this._repositoryService.GetQueries(parameters, from, to, max);
var result = new List <Repository>();
foreach (var query in queries)
{
var repositories = await this._repositoryApiService.GetRepositoriesAsync(query.Parameters,
query.From, (query.To - query.From + 1));
result.AddRange(repositories);
}
var repos = _repositories.OrderByDescending(r => r.StarsCount).ThenByDescending(r => r.UpdatedDate);
Assert.AreEqual(to - from + 1, result.Count);
CollectionAssert.AreEquivalent(this._repositories, result);
}
/// <summary>
/// Gets list of repositories according to search parameters.
/// </summary>
/// <param name="parameters">Search and pagination parameters.</param>
/// <returns>Page containing list of repositories.</returns>
public async Task <RepositoryPage> GetRepositoriesAsync(RepositoryParameters parameters)
{
Ensure.ArgumentNotNull(parameters, nameof(parameters));
var result = new RepositoryPage();
result.TotalCount = await this._repositoryApiService.GetRepositoriesCount(parameters);
result.Repositories = new List <Repository>();
var from = parameters.PageSize.Value * (parameters.PageNumber.Value - 1) + 1;
var to = Math.Min(result.TotalCount, from + parameters.PageSize.Value - 1);
var queries = await GetQueries(parameters, from, to, MAX_REPOSITORIES_COUNT);
foreach (var query in queries)
{
var repositories = await this._repositoryApiService.GetRepositoriesAsync(query.Parameters,
query.From, (query.To - query.From + 1));
result.Repositories.AddRange(repositories);
}
return(result);
}
public async Task RepositoryService_GetQueries_TwoBatches()
{
var parameters = new RepositoryParameters();
int from = 2;
int to = 4;
int max = 3;
int starsCount = 3;
parameters.MinNumberOfStars = starsCount;
var queries = await this._repositoryService.GetQueries(parameters, from, to, max);
var result = new List <Repository>();
foreach (var query in queries)
{
var repositories = await this._repositoryApiService.GetRepositoriesAsync(query.Parameters,
query.From, (query.To - query.From + 1));
result.AddRange(repositories);
}
Assert.AreEqual(to - from + 1, result.Count);
Assert.AreEqual(result.Count, result.Count(r => r.StarsCount == starsCount));
}
public QueryParams(RepositoryParameters parameters, int from, int to)
{
this.Parameters = parameters;
this.From = from;
this.To = to;
}
/// <summary>
/// Workaround for GitHub repositories count limit.
/// Splits requested data into batches by combination of stars count and creation date.
/// Returns list of query parameters to request data from those batches.
/// Sorting by stars count is applied, sorting by update/creation dates is not guaranteed.
/// </summary>
/// <param name="parameters">Parameters for a query without limitation.</param>
/// <param name="from">Index of the first requested repository.</param>
/// <param name="to">Index of the last requested repository.</param>
/// <param name="max">Defined limit (1000; parametrized for unit testing).</param>
/// <returns>List of query parameters considering limitation.</returns>
internal async Task <List <QueryParams> > GetQueries(RepositoryParameters parameters,
int from, int to, int max)
{
var queries = new List <QueryParams>();
// if search period is inside limitation then just execute search as it is
if (from <= max && to <= max)
{
queries.Add(new QueryParams(parameters.Clone(), from, Math.Min(to, max)));
return(queries);
}
while (to > max)
{
// find the number of stars for the last repository in the next batch of [max] repositories;
// calculate the number of repositories in this batch with the number of stars greater than the last
// to avoid calculating those repositories twice
var starsCount = await _repositoryApiService.GetRepositoryStarsCount(parameters, max);
var correctedParameters = parameters.Clone();
correctedParameters.MinNumberOfStars = starsCount + 1;
var repositoryCountShift = await _repositoryApiService.GetRepositoriesCount(correctedParameters);
from = Math.Max(from - repositoryCountShift, 1);
to -= repositoryCountShift;
// calculate the number of repositories with the number of stars equal to the last one
correctedParameters.MinNumberOfStars = starsCount;
correctedParameters.MaxNumberOfStars = starsCount;
var repositoryCount = await _repositoryApiService.GetRepositoriesCount(correctedParameters);
// if the page we're looking for is inside current batch then start to compose correct search queries
// that would return less than [max] repositories
if (from <= repositoryCount)
{
DateTime endDate = DateTime.Today.AddDays(1);
if (parameters.MaxCreated.HasValue && endDate > parameters.MaxCreated)
{
endDate = parameters.MaxCreated.Value;
}
DateTime startDate = endDate.AddHours(-DEFAULT_HOURS_SEARCH_PERIOD);
if (parameters.MinCreated.HasValue && startDate < parameters.MinCreated)
{
startDate = parameters.MinCreated.Value;
}
correctedParameters.MaxCreated = endDate;
// if batch size (repositoryCount) is greater than [max], divide a batch with equal
// stars count by created date (constant value, won't change during execution);
while (repositoryCount >= max && to >= max)
{
correctedParameters.MinCreated = parameters.MinCreated;
correctedParameters.MaxCreated = endDate;
// by default use one year period; if there are more than [max] repositories inside
// this period, then divide it by two and try again
var addCount = await _repositoryApiService.GetRepositoriesCount(correctedParameters);
var hours = DEFAULT_HOURS_SEARCH_PERIOD;
while (addCount > max)
{
startDate = endDate.AddHours(-hours);
correctedParameters.MinCreated = startDate;
addCount = await _repositoryApiService.GetRepositoriesCount(correctedParameters);
hours = (int)Math.Ceiling(hours / ((double)addCount / (double)max));
}
// if start of the page is inside current batch and the end is not,
// add query for the first part of the page to the result
if (from <= addCount)
{
queries.Add(new QueryParams(correctedParameters.Clone(), from, addCount));
from = addCount + 1;
}
// subtract processed amount of repositories
repositoryCount -= addCount;
from = Math.Max(1, from - addCount);
to -= addCount;
endDate = startDate.AddSeconds(-1);
startDate = endDate.AddHours(-DEFAULT_HOURS_SEARCH_PERIOD);
if (parameters.MinCreated.HasValue && startDate < parameters.MinCreated)
{
startDate = parameters.MinCreated.Value;
}
}
correctedParameters.MinCreated = parameters.MinCreated;
correctedParameters.MaxCreated = endDate;
queries.Add(new QueryParams(correctedParameters.Clone(), from,
Math.Min(to, repositoryCount)));
from = Math.Min(to, repositoryCount) + 1;
correctedParameters.MaxCreated = parameters.MaxCreated;
}
from = Math.Max(1, from - repositoryCount);
to -= repositoryCount;
// change query to go to the next batch
parameters.MaxNumberOfStars = starsCount - 1;
}
if (to > 0)
{
queries.Add(new QueryParams(parameters.Clone(), from, to));
}
return(queries);
}
public Task <int> GetRepositoriesCount(RepositoryParameters parameters)
{
return(Task.FromResult(this._repositories
.Where(r => CheckRepository(r, parameters))
.Count()));
}
private bool CheckRepository(Repository repository, RepositoryParameters parameters)
{
if (parameters.HasGoodFirstIssues.HasValue && parameters.HasGoodFirstIssues.Value)
{
if (repository.GoodFirstIssuesCount == 0)
{
return(false);
}
}
if (parameters.HasHelpWantedIssues.HasValue && parameters.HasHelpWantedIssues.Value)
{
if (repository.HelpWantedIssuesCount == 0)
{
return(false);
}
}
if (!string.IsNullOrEmpty(parameters.Language))
{
if (!repository.PrimaryLanguage.Equals(parameters.Language, StringComparison.CurrentCultureIgnoreCase))
{
return(false);
}
}
if (!string.IsNullOrEmpty(parameters.Text))
{
if (!repository.Description.Contains(parameters.Text, StringComparison.CurrentCultureIgnoreCase) &&
!repository.Name.Contains(parameters.Text, StringComparison.CurrentCultureIgnoreCase) &&
!repository.Owner.Contains(parameters.Text, StringComparison.CurrentCultureIgnoreCase))
{
return(false);
}
}
if (parameters.LastUpdateAfter.HasValue)
{
if (repository.UpdatedDate < parameters.LastUpdateAfter.Value)
{
return(false);
}
}
if (parameters.MinCreated.HasValue)
{
if (repository.CreatedDate < parameters.MinCreated.Value)
{
return(false);
}
}
if (parameters.MaxCreated.HasValue)
{
if (repository.CreatedDate > parameters.MaxCreated.Value)
{
return(false);
}
}
if (parameters.MinNumberOfStars.HasValue)
{
if (repository.StarsCount < parameters.MinNumberOfStars.Value)
{
return(false);
}
}
if (parameters.MaxNumberOfStars.HasValue)
{
if (repository.StarsCount > parameters.MaxNumberOfStars.Value)
{
return(false);
}
}
return(true);
}