protected virtual void AddMultipleValues(HtmlNode node, ScrapeItemSetting scrapeItem, List <ResultItem> list)
{
var itemNodes = node.QuerySelectorAll(scrapeItem.Selector).ToList();
if (itemNodes.Any())
{
foreach (var itemNode in itemNodes)
{
if (!string.IsNullOrEmpty(scrapeItem.AttributeName))
{
var value = GetByAttribute(itemNode, scrapeItem);
list.Add(new ResultItem
{
Name = scrapeItem.Key,
Value = value,
IsMultiple = true
});
}
else
{
var value = GetByInnerHtml(itemNode, scrapeItem);
list.Add(new ResultItem
{
Name = scrapeItem.Key,
Value = !string.IsNullOrEmpty(value) ? value : scrapeItem.DefaultValue,
IsMultiple = true
});
}
}
}
}
/// <summary>
/// Single Value could potentially contains the "key" value, so we return the value that we add here
/// </summary>
/// <param name="node"></param>
/// <param name="scrapeItem"></param>
/// <param name="list"></param>
/// <returns></returns>
protected virtual string AddSingleValue(HtmlNode node, ScrapeItemSetting scrapeItem, List <ResultItem> list)
{
string value;
var itemNode = node.QuerySelector(scrapeItem.Selector);
if (!string.IsNullOrEmpty(scrapeItem.AttributeName))
{
value = GetByAttribute(itemNode, scrapeItem);
list.Add(new ResultItem
{
Name = scrapeItem.Key,
Value = value
});
}
else
{
value = GetByInnerHtml(itemNode, scrapeItem);
list.Add(new ResultItem
{
Name = scrapeItem.Key,
Value = !string.IsNullOrEmpty(value) ? value : scrapeItem.DefaultValue
});
}
return(value);
}
protected virtual string GetByAttribute(HtmlNode itemNode, ScrapeItemSetting scrapeItem)
{
if (scrapeItem.ValueRequired && !itemNode.HasAttributes)
{
throw new ScrapperException($"Attribute {scrapeItem.Key} does not exists");
}
return(itemNode.GetAttributeValue(scrapeItem.AttributeName, scrapeItem.DefaultValue));
}
protected virtual string GetByInnerHtml(HtmlNode itemNode, ScrapeItemSetting scrapeItem)
{
var value = itemNode.InnerHtml;
if (scrapeItem.ValueRequired && string.IsNullOrEmpty(value))
{
throw new ScrapperException($"Value {scrapeItem.Key} does not exists");
}
return(!string.IsNullOrEmpty(value) ? value : scrapeItem.DefaultValue);
}
