/// <summary>
/// Accumulates all the elements in a paginated response, then deserializes each element as <c>TElement</c>.
/// </summary>
/// <param name="response">The first response received after initiating the request.</param>
/// <typeparam name="TElement">The model type of the elements returned in the response.</typeparam>
/// <returns>The list of deserialized elements.</returns>
private async Task <List <TElement> > AccumulateDeserializePages <TElement>(HttpResponseMessage response)
{
response.AssertSuccess();
var pages = new List <HttpContent> {
response.Content
};
while (response.Headers.TryGetValues("Link", out IEnumerable <string> linkValues))
{
var links = LinkHeader.LinksFromHeader(linkValues.First());
if (links?.NextLink == null)
{
break;
}
response = await _client.GetAsync(links.NextLink);
response.AssertSuccess();
pages.Add(response.Content);
}
var accumulated = new List <TElement>();
foreach (var content in pages)
{
var responseStr = await content.ReadAsStringAsync();
accumulated.AddRange(JsonConvert.DeserializeObject <List <TElement> >(responseStr));
}
return(accumulated);
}
private async IAsyncEnumerable <TElement> StreamDeserializePages <TElement>(HttpResponseMessage response)
{
var firstPage = JsonConvert.DeserializeObject <List <TElement> >(await response.AssertSuccess()
.Content
.ReadAsStringAsync());
foreach (var e in firstPage)
{
yield return(e);
}
while (response.Headers.TryGetValues("Link", out IEnumerable <string> linkValues))
{
var links = LinkHeader.LinksFromHeader(linkValues.First());
if (links?.NextLink == null)
{
break;
}
response = await _client.GetAsync(links.NextLink);
var content = response.AssertSuccess().Content;
var list = JsonConvert.DeserializeObject <List <TElement> >(await content.ReadAsStringAsync());
foreach (var e in list)
{
yield return(e);
}
}
}
/// <summary>
/// Accumulates all the elements in a paginated response.
/// </summary>
/// <param name="response">The first response received after initiating the request.</param>
/// <returns>The list of JSON elements.</returns>
private async Task <List <JToken> > AccumulatePages(HttpResponseMessage response)
{
response.AssertSuccess();
var pages = new List <HttpContent> {
response.Content
};
while (response.Headers.TryGetValues("Link", out var linkValues))
{
var links = LinkHeader.LinksFromHeader(linkValues.First());
if (links?.NextLink == null)
{
break;
}
response = await _client.GetAsync(links.NextLink);
response.AssertSuccess();
pages.Add(response.Content);
}
var accumulated = new List <JToken>();
foreach (var content in pages)
{
var responseStr = await content.ReadAsStringAsync();
accumulated.AddRange(JToken.Parse(responseStr));
}
return(accumulated);
}
private async IAsyncEnumerable <T[]> GetList <T>(string?startUrl, int maxNumber, [EnumeratorCancellation] CancellationToken cancellationToken, Func <T[], Task <T[]> >?processing = null)
{
var counter = 0;
while (!string.IsNullOrEmpty(startUrl))
{
if (cancellationToken.IsCancellationRequested || counter > maxNumber)
{
yield break;
}
using var request = new HttpRequestMessage(HttpMethod.Get, startUrl);
using var httpResponse = await _httpClient.SendAsync(request, cancellationToken /* throws exception on cancellation */);
if (httpResponse.IsSuccessStatusCode)
{
startUrl = null;
var links = LinkHeader.LinksFromHeader(httpResponse);
if (!string.IsNullOrEmpty(links?.NextLink))
{
startUrl = links.NextLink;
}
await using var stream = await httpResponse.Content.ReadAsStreamAsync(cancellationToken);
var responseArray = await JsonSerializer.DeserializeAsync <T[]>(stream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }, cancellationToken);
if (responseArray == null)
{
LogResponseError("Failed to parse response", httpResponse);
yield break;
}
if (processing != null)
{
responseArray = await processing(responseArray);
}
counter += responseArray.Length;
yield return(responseArray);
}
else
{
LogResponseError("Failed to fetch a page", httpResponse);
}
}
}
/// <inheritdoc/>
public async Task <SearchResults> SearchRepo(SearchOptions searchOption)
{
string giteaSearchUriString = BuildSearchQuery(searchOption);
SearchResults searchResults = null;
HttpResponseMessage response = await _httpClient.GetAsync(giteaSearchUriString);
if (response.StatusCode == HttpStatusCode.OK)
{
searchResults = await response.Content.ReadAsAsync <SearchResults>();
if (response.Headers.TryGetValues("X-Total-Count", out IEnumerable <string> countValues))
{
searchResults.TotalCount = Convert.ToInt32(countValues.First());
}
if (response.Headers.TryGetValues("Link", out IEnumerable <string> linkValues))
{
LinkHeader linkHeader = LinkHeader.LinksFromHeader(linkValues.First());
if (!string.IsNullOrEmpty(linkHeader.LastLink))
{
Uri linkUri = new Uri(linkHeader.LastLink);
string page = HttpUtility.ParseQueryString(linkUri.Query).Get("page");
if (int.TryParse(page, out int lastPage))
{
searchResults.TotalPages = lastPage;
}
}
else
{
searchResults.TotalPages = searchOption.Page;
}
}
if (searchResults.TotalCount > 0 && searchResults.TotalPages == 0)
{
searchResults.TotalPages = 1;
}
}
else
{
_logger.LogError("User " + AuthenticationHelper.GetDeveloperUserName(_httpContextAccessor.HttpContext) + " SearchRepository failed with statuscode " + response.StatusCode);
}
return(searchResults);
}
private static async Task <GithubResponse> ProcessRepositories(string requestUri, string oAuthToken)
{
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.v3+json"));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", oAuthToken);
client.DefaultRequestHeaders.Add("User-Agent", "Github Repo Lister");
var response = await client.GetAsync(requestUri);
response.EnsureSuccessStatusCode();
var serializedRepos = await JsonSerializer.DeserializeAsync <List <Repository> >(await response.Content.ReadAsStreamAsync());
if (response.Headers.Contains("Link"))
{
return(new GithubResponse(serializedRepos, LinkHeader.LinksFromHeader(response.Headers.GetValues("Link").FirstOrDefault())));
}
else
{
return(new GithubResponse(serializedRepos, new LinkHeader()));
}
}
private async IAsyncEnumerable <JToken> StreamPages(HttpResponseMessage response)
{
var firstPage = JToken.Parse(await response.AssertSuccess()
.Content
.ReadAsStringAsync());
yield return(firstPage);
while (response.Headers.TryGetValues("Link", out IEnumerable <string> linkValues))
{
var links = LinkHeader.LinksFromHeader(linkValues.First());
if (links?.NextLink == null)
{
break;
}
response = await _client.GetAsync(links.NextLink);
var content = response.AssertSuccess().Content;
yield return(JToken.Parse(await content.ReadAsStringAsync()));
}
}