private void ScrapePage(string url) { var web = new HtmlWeb(); var document = web.Load(url); var response = document.DocumentNode; var finalUrl = GetOgPropertyValue(response, "url"); if (!BundlesTab.Any()) { BundlesTab = GetBundlesTab(response).ToList(); } _visitedUrls.Add(url); _visitedUrls.Add(finalUrl); if (url == BaseUrl) { VisitOtherPages(BundlesTab); } else { try { var bundle = new HumbleBundle { Name = GetOgPropertyValue(response, "title"), Description = GetOgPropertyValue(response, "description"), ImageUrl = GetOgPropertyValue(response, "image"), URL = finalUrl, Type = GetBundleType(url) }; ScrapeSections(bundle, response); _foundBundles.Add(bundle); } catch (Exception e) { Debug.WriteLine(e.Message); // Do nothing } } }
private static void ScrapeSections(HumbleBundle bundle, HtmlNode response) { foreach (var parsedSection in response.CssSelect(".dd-game-row")) { string sectionTitle; try { try { sectionTitle = parsedSection.CssSelect(".dd-header-headline").First().InnerText .CleanInnerText(); } catch { sectionTitle = parsedSection.CssSelect(".fi-content-header").First().InnerText.CleanInnerText(); } } catch (Exception) { sectionTitle = string.Empty; } if (sectionTitle.Contains("average")) { sectionTitle = "Beat the Average!"; } if (string.IsNullOrEmpty(sectionTitle)) { continue; } var sectionToAdd = new HumbleSection() { Title = sectionTitle }; FindGamesInSection(parsedSection, sectionToAdd); bundle.Sections.Add(sectionToAdd); } }
private static void AddToQueues(ICollector <BundleQueue> bundleQueue, ICollector <BundleQueue> jsonMessageQueue, HumbleBundle bundle) { bundleQueue.Add(new BundleQueue() { Bundle = bundle, IsUpdate = true }); jsonMessageQueue.Add(new BundleQueue() { Bundle = bundle, IsUpdate = true }); }
public DiscordWebhookPayload(BundleQueue queuedBundle, HumbleBundle bundle) { content = "New Bundle: " + bundle.Name; if (queuedBundle.IsUpdate) { content = "Bundle Updated: " + bundle.Name; } embeds.Add(new DiscordEmbed() { url = bundle.URL, title = bundle.Description, image = new ImageField() { url = bundle.ImageUrl }, author = new AuthorField() { name = "Humble Bundle", url = bundle.URL }, fields = new List <EmbedField>() { new EmbedField { name = "Powered By", value = "https://github.com/cswendrowski/HumbleBundleBot" } } }); var random = new Random(); foreach (var section in bundle.Sections) { var embed = new DiscordEmbed { title = section.Title, url = bundle.URL + "?dedupe=" + random.Next(), description = "" }; var itemsAdded = 0; foreach (var item in section.Items) { embed.description += GetItemName(item, queuedBundle.UpdatedItems); itemsAdded++; // Create a new embed every 25 items if (itemsAdded % 25 == 0) { embeds.Add(embed); embed = new DiscordEmbed { title = section.Title + " (Continued)", url = bundle.URL + "?dedupe=" + random.Next(), description = "" }; } } // Add last embed embeds.Add(embed); } }