private async Task <int> GetPageCount(CancellationToken token)
{
string content = await GetPageContent(0, token);
if (token.IsCancellationRequested)
{
return(0);
}
HumbleBundleData data = HumbleBundleData.FromJson(content);
return((int)data.NumPages);
}
async Task <IEnumerable <IDeal> > IScraper.Scrape(CancellationToken token)
{
List <IDeal> deals = new List <IDeal>();
logger.Info("Getting page count");
int pageCount = await GetPageCount(token);
if (token.IsCancellationRequested)
{
return(null);
}
logger.Info($"Found {pageCount} pages");
for (int i = 0; i < pageCount; i++)
{
await Task.Delay(1500, token);
if (token.IsCancellationRequested)
{
return(null);
}
logger.Info($"Scraping page {i + 1}");
string content = await GetPageContent(i, token);
if (token.IsCancellationRequested)
{
return(null);
}
HumbleBundleData data = HumbleBundleData.FromJson(content);
foreach (Result result in data.Results)
{
int discount = GetDiscount(result);
if (discount != 100)
{
continue;
}
if (result.HumanName.ToLower().Contains("demo") || result.HumanUrl.ToLower().Contains("demo"))
{
logger.Info($"Skipping {result.HumanName} because it's a demo");
continue;
}
Deal deal = new Deal
{
Discount = discount,
Image = result.FeaturedImageRecommendation,
Link = $"https://humblebundle.com/store/{result.HumanUrl}",
Title = result.HumanName,
Start = null,
End = DateTimeOffset.FromUnixTimeSeconds(result.SaleEnd).UtcDateTime
};
deals.Add(deal);
}
}
return(deals);
}
public static string ToJson(this HumbleBundleData self) => JsonConvert.SerializeObject(self, Converter.Settings);