private void ScrapePage(string url)
{
var web = new HtmlWeb();
var document = web.Load(url);
var response = document.DocumentNode;
var finalUrl = GetOgPropertyValue(response, "url");
if (!BundlesTab.Any())
{
BundlesTab = GetBundlesTab(response).ToList();
}
_visitedUrls.Add(url);
_visitedUrls.Add(finalUrl);
if (url == BaseUrl)
{
VisitOtherPages(BundlesTab);
}
else
{
try
{
var bundle = new HumbleBundle
{
Name = GetOgPropertyValue(response, "title"),
Description = GetOgPropertyValue(response, "description"),
ImageUrl = GetOgPropertyValue(response, "image"),
URL = finalUrl,
Type = GetBundleType(url)
};
ScrapeSections(bundle, response);
_foundBundles.Add(bundle);
}
catch (Exception e)
{
Debug.WriteLine(e.Message);
// Do nothing
}
}
}
private static void ScrapeSections(HumbleBundle bundle, HtmlNode response)
{
foreach (var parsedSection in response.CssSelect(".dd-game-row"))
{
string sectionTitle;
try
{
try
{
sectionTitle = parsedSection.CssSelect(".dd-header-headline").First().InnerText
.CleanInnerText();
}
catch
{
sectionTitle = parsedSection.CssSelect(".fi-content-header").First().InnerText.CleanInnerText();
}
}
catch (Exception)
{
sectionTitle = string.Empty;
}
if (sectionTitle.Contains("average"))
{
sectionTitle = "Beat the Average!";
}
if (string.IsNullOrEmpty(sectionTitle))
{
continue;
}
var sectionToAdd = new HumbleSection()
{
Title = sectionTitle
};
FindGamesInSection(parsedSection, sectionToAdd);
bundle.Sections.Add(sectionToAdd);
}
}
private static void AddToQueues(ICollector <BundleQueue> bundleQueue, ICollector <BundleQueue> jsonMessageQueue, HumbleBundle bundle)
{
bundleQueue.Add(new BundleQueue()
{
Bundle = bundle,
IsUpdate = true
});
jsonMessageQueue.Add(new BundleQueue()
{
Bundle = bundle,
IsUpdate = true
});
}
public DiscordWebhookPayload(BundleQueue queuedBundle, HumbleBundle bundle)
{
content = "New Bundle: " + bundle.Name;
if (queuedBundle.IsUpdate)
{
content = "Bundle Updated: " + bundle.Name;
}
embeds.Add(new DiscordEmbed()
{
url = bundle.URL,
title = bundle.Description,
image = new ImageField()
{
url = bundle.ImageUrl
},
author = new AuthorField()
{
name = "Humble Bundle",
url = bundle.URL
},
fields = new List <EmbedField>()
{
new EmbedField
{
name = "Powered By",
value = "https://github.com/cswendrowski/HumbleBundleBot"
}
}
});
var random = new Random();
foreach (var section in bundle.Sections)
{
var embed = new DiscordEmbed
{
title = section.Title,
url = bundle.URL + "?dedupe=" + random.Next(),
description = ""
};
var itemsAdded = 0;
foreach (var item in section.Items)
{
embed.description += GetItemName(item, queuedBundle.UpdatedItems);
itemsAdded++;
// Create a new embed every 25 items
if (itemsAdded % 25 == 0)
{
embeds.Add(embed);
embed = new DiscordEmbed
{
title = section.Title + " (Continued)",
url = bundle.URL + "?dedupe=" + random.Next(),
description = ""
};
}
}
// Add last embed
embeds.Add(embed);
}
}
