private ProductCategoryObject ParseProductObject(HtmlNode node, ArgumentObject args) { return(new ProductCategoryObject( singleProperties: SinglePropertiesProduct.ToDictionary(x => x.Key, x => x.Value(node, args)), pluralProperties: PluralPropertiesProduct?.ToDictionary(x => x.Key, x => x.Value(node, args)) )); }
public IEnumerable <ProductCategoryObject> GetProductOrCategory(ArgumentObject args) { var stack = new Stack <ArgumentObject>(); stack.Push(args); while (stack.Count != 0) { if (Debug) { Console.Write(stack.Peek().ToString()); } var result = ProccessUrl(stack.Pop()); if (result.IsCategory) { foreach (var product in result.Products) { stack.Push(product); } foreach (var subctalog in result.Subcatalogs) { stack.Push(subctalog); } } yield return(result); } }
private ProductCategoryObject ParseCategoryObject(HtmlNode node, ArgumentObject args) { return(new ProductCategoryObject( pluralProperties: PluralPropertiesCategory?.ToDictionary(x => x.Key, x => x.Value(node, args)), singleProperties: SinglePropertiesCategory.ToDictionary(x => x.Key, x => x.Value(node, args)), subcatalogs: ParseFromAllPages(node, args, _findSubcatalogs).ToList(), products: ParseFromAllPages(node, args, _findProducts).ToList() )); }
private ProductCategoryObject ProccessUrl(ArgumentObject args) { var web = new HtmlWeb { OverrideEncoding = _encoding }; var node = web.Load(args.Url).DocumentNode; //File.WriteAllText("tmp.html", node.InnerHtml); return(IsCategory(node) ? ParseCategoryObject(node, args) : ParseProductObject(node, args)); }
public IEnumerable <ArgumentObject> GetLinks(ArgumentObject args, string xPath, string prefix = "", string suffix = "") { var web = new HtmlWeb { OverrideEncoding = _encoding }; var node = web.Load(args.Url).DocumentNode; return(node._SelectNodes(xPath) .Select(x => new ArgumentObject(url: prefix + x.Attributes["href"].Value + suffix, args: args.Args))); }
private IEnumerable <ArgumentObject> ParseFromAllPages( HtmlNode node, ArgumentObject args, Search <ArgumentObject[]> func) { foreach (var argument in func(node, args)) { yield return(argument); } var links = _xPathPagintaion(node, args); var web = new HtmlWeb { OverrideEncoding = _encoding }; foreach (var link in links) { var htmlNode = web.Load(link.Url).DocumentNode; foreach (var argument in func(htmlNode, args)) { yield return(argument); } } }