private ProductCategoryObject ParseProductObject(HtmlNode node, ArgumentObject args)
{
return(new ProductCategoryObject(
singleProperties: SinglePropertiesProduct.ToDictionary(x => x.Key, x => x.Value(node, args)),
pluralProperties: PluralPropertiesProduct?.ToDictionary(x => x.Key, x => x.Value(node, args))
));
}
public IEnumerable <ProductCategoryObject> GetProductOrCategory(ArgumentObject args)
{
var stack = new Stack <ArgumentObject>();
stack.Push(args);
while (stack.Count != 0)
{
if (Debug)
{
Console.Write(stack.Peek().ToString());
}
var result = ProccessUrl(stack.Pop());
if (result.IsCategory)
{
foreach (var product in result.Products)
{
stack.Push(product);
}
foreach (var subctalog in result.Subcatalogs)
{
stack.Push(subctalog);
}
}
yield return(result);
}
}
private ProductCategoryObject ParseCategoryObject(HtmlNode node, ArgumentObject args)
{
return(new ProductCategoryObject(
pluralProperties: PluralPropertiesCategory?.ToDictionary(x => x.Key, x => x.Value(node, args)),
singleProperties: SinglePropertiesCategory.ToDictionary(x => x.Key, x => x.Value(node, args)),
subcatalogs: ParseFromAllPages(node, args, _findSubcatalogs).ToList(),
products: ParseFromAllPages(node, args, _findProducts).ToList()
));
}
private ProductCategoryObject ProccessUrl(ArgumentObject args)
{
var web = new HtmlWeb {
OverrideEncoding = _encoding
};
var node = web.Load(args.Url).DocumentNode;
//File.WriteAllText("tmp.html", node.InnerHtml);
return(IsCategory(node)
? ParseCategoryObject(node, args)
: ParseProductObject(node, args));
}
public IEnumerable <ArgumentObject> GetLinks(ArgumentObject args,
string xPath,
string prefix = "",
string suffix = "")
{
var web = new HtmlWeb {
OverrideEncoding = _encoding
};
var node = web.Load(args.Url).DocumentNode;
return(node._SelectNodes(xPath)
.Select(x => new ArgumentObject(url: prefix + x.Attributes["href"].Value + suffix,
args: args.Args)));
}
private IEnumerable <ArgumentObject> ParseFromAllPages(
HtmlNode node,
ArgumentObject args,
Search <ArgumentObject[]> func)
{
foreach (var argument in func(node, args))
{
yield return(argument);
}
var links = _xPathPagintaion(node, args);
var web = new HtmlWeb {
OverrideEncoding = _encoding
};
foreach (var link in links)
{
var htmlNode = web.Load(link.Url).DocumentNode;
foreach (var argument in func(htmlNode, args))
{
yield return(argument);
}
}
}
