static Dictionary <string, string> GrabSpellDetails(string html)
{
var detailNodes = GatherHTML.ParseHtml(
html, "div", "class", "ddb-statblock-item")
.Where(node => node.HasClass("ddb-statblock-item")).ToList();
var spellDetails = new Dictionary <string, string>();
foreach (var detailNode in detailNodes)
{
var label = GatherHTML.GetElementContent(detailNode.Descendants()
.First(node => node.HasClass("ddb-statblock-item-label")));
var value = GatherHTML.GetElementContent(detailNode.Descendants()
.First(node => node.HasClass("ddb-statblock-item-value")));
var fix = value.Split(' ');
value = "";
foreach (var word in fix)
{
value += string.IsNullOrWhiteSpace(word) ? "" : word + " ";
}
spellDetails.Add(spellFormat[label.Replace('\n', ' ').Trim()], value.Replace('\n', ' ').Trim());
}
return(spellDetails);
}
public static string CreateSpellFromScraping(string pageName)
{
var url = CallSpellPage(pageName);
var spellName = GatherHTML.ParseHtml(url, "h1", "class", "page-title").FirstOrDefault();
var spellDetails = GrabSpellDetails(url);
string details =
spellDetails.Aggregate("", (current, detail) => current + $"'{detail.Key}' : '{detail.Value}',");
string descriptionSection = GrabSpellDescription(url);
bool hasAdditionalInfo = descriptionSection.Contains('*');
string description = $"'Description' : '{(hasAdditionalInfo ? descriptionSection.Split("*")[0] : descriptionSection)}',";
string additionInfo = $"'AdditionalInfo' : '{(hasAdditionalInfo ? ("*" + descriptionSection.Split("*")[1]) : "")}',";
string listOfClasses = $"'ListOfClassesSpellIsIn' : [{GrabClassSpellListTags(url)}],";
string spellTags = $"'SpellTags' : [{GrabSpellTags(url)}]";
return("{" +
$"'Name' : '{spellName.InnerText.Replace('\n', ' ').Trim()}', " +
$"{details}" +
$"{description}" +
$"{additionInfo}" +
$"{listOfClasses}" +
$"{spellTags}" +
"}");
}
static List <string> GrabTags(string html, string fliterClassName)
{
var detailNodes = GatherHTML.ParseHtml(
html, "span", "class", fliterClassName);
return((from detailNode in detailNodes select detailNode.InnerText).ToList());
}
public void ParseHtmlElementsFromHtmlPage()
{
url = "https://www.dndbeyond.com/classes/barbarian";
var response = GatherHTML.CallUrl(url);
Assert.IsNotEmpty(GatherHTML.ParseHtml(response.Result, "div", "class", "subitems-list-details-item"));
url = "https://www.dndbeyond.com/spells/absorb-elements";
response = GatherHTML.CallUrl(url);
Assert.IsNotEmpty(GatherHTML.ParseHtml(response.Result, "div", "class", "ddb-statblock-spell"));
}
static string GrabSpellDescription(string html)
{
var detailNodes = GatherHTML.ParseHtml(
html, "div", "class", "more-info-content");
string description = "";
var words = detailNodes.FirstOrDefault()?.InnerText.Split(' ');
if (words != null)
{
foreach (var word in words)
{
if (!string.IsNullOrWhiteSpace(word) && word != "\n")
{
description += word + " ";
}
}
}
return(description.Replace("\'", ""));
}
public void GetWebPageHTML()
{
url = "https://www.dndbeyond.com/";
Assert.IsNotEmpty(GatherHTML.CallUrl(url).Result);
}
static string CallSpellPage(string pageName)
{
return(GatherHTML.CallUrl(SPELL_URL_ROOT + "/" + pageName).Result);
}