private async Task <int> GetPageCount(CancellationToken token) { string content = await GetPageContent(0, token); if (token.IsCancellationRequested) { return(0); } HumbleBundleData data = HumbleBundleData.FromJson(content); return((int)data.NumPages); }
async Task <IEnumerable <IDeal> > IScraper.Scrape(CancellationToken token) { List <IDeal> deals = new List <IDeal>(); logger.Info("Getting page count"); int pageCount = await GetPageCount(token); if (token.IsCancellationRequested) { return(null); } logger.Info($"Found {pageCount} pages"); for (int i = 0; i < pageCount; i++) { await Task.Delay(1500, token); if (token.IsCancellationRequested) { return(null); } logger.Info($"Scraping page {i + 1}"); string content = await GetPageContent(i, token); if (token.IsCancellationRequested) { return(null); } HumbleBundleData data = HumbleBundleData.FromJson(content); foreach (Result result in data.Results) { int discount = GetDiscount(result); if (discount != 100) { continue; } if (result.HumanName.ToLower().Contains("demo") || result.HumanUrl.ToLower().Contains("demo")) { logger.Info($"Skipping {result.HumanName} because it's a demo"); continue; } Deal deal = new Deal { Discount = discount, Image = result.FeaturedImageRecommendation, Link = $"https://humblebundle.com/store/{result.HumanUrl}", Title = result.HumanName, Start = null, End = DateTimeOffset.FromUnixTimeSeconds(result.SaleEnd).UtcDateTime }; deals.Add(deal); } } return(deals); }