public async Task <IEnumerable <RedditImage> > Scrap(string subreddit, int pagesCount, Action <ScrapOptions> options)
{
var opts = new ScrapOptions();
options(opts);
var result = new List <RedditImage>();
var currentPage = 0;
while (++currentPage <= pagesCount)
{
await Task.Delay(DelayBetweenRequests);
var pageResponse = await FetchPage(subreddit, opts);
if (!pageResponse.IsSuccessful)
{
throw new HttpRequestException(pageResponse.ErrorMessage);
}
//Check if subreddit page has any posts
if (!pageResponse.Data.PostIds.Any())
{
break;
}
opts.After = pageResponse.Data.PostIds.Last();
var pageImages = ExtractImages(pageResponse.Data);
result.AddRange(pageImages);
}
return(result);
}
private Task <IRestResponse <RedditPage> > FetchPage(string subreddit, ScrapOptions options)
{
var request = new RestRequest($"/desktopapi/v1/subreddits/{subreddit}", Method.GET);
request.AddQueryParameter("redditWebClient", "web2x");
request.AddQueryParameter("app", "web2x-client-production");
request.AddQueryParameter("sort", options.SortMode.ToString().ToLower());
request.AddQueryParameter("dist", options.PerPage.ToString());
if (!string.IsNullOrEmpty(options.After))
{
request.AddQueryParameter("after", options.After);
}
if (options.AllowOver18)
{
request.AddQueryParameter("allow_over18", "true");
}
return(_client.ExecuteTaskAsync <RedditPage>(request));
}