private async Task <IEnumerable <PagingResult> > GetAllPagesAsync(ISteamService steamService, int totalPages)
{
const int parallelDownloads = 16;
var finalResults = new List <PagingResult>(totalPages);
var segmentedPages = Enumerable
.Range(0, totalPages)
.Segmentate(parallelDownloads)
.ToList();
foreach (var segment in segmentedPages)
{
var tasks = new List <Task <PagingResult> >();
foreach (var page in segment)
{
tasks.Add(Task.Run(async() =>
{
var start = page * DeveloperListResultsPerPage;
var address = $"https://store.steampowered.com/curators/ajaxgettopcreatorhomes/render/?start={start}&count={DeveloperListResultsPerPage}";
try
{
logger.LogInformation("Attempting to download result page from address '{@Uri}'...", address);
return(await steamService.GetJsonAsync <PagingResult>(new Uri(address, UriKind.Absolute)));
}
catch (SteamPageRemovedException e)
{
logger.LogWarning(e, "Could not download from address '{@Uri}' because the page is gone.", address);
// Can't do anything but ignore and skip this item.
return(null);
}
catch (Exception e)
{
logger.LogWarning(e, "Could not download from address '{@Uri}'.", address);
throw;
}
}));
}
try
{
var results = await Task.WhenAll(tasks);
finalResults.AddRange(results.Where(x => x is not null));
}
catch (AggregateException e)
{
var collectedExceptions = e.Unwrap();
var rateLimitException = collectedExceptions.FirstOrDefault(x => x is SteamRateLimitExceededException);
if (rateLimitException is not null)
{
throw rateLimitException;
}
throw;
}
}
return(finalResults);
}
