public ParsingResults Parse(System.Net.WebClient client, List <Models.Product> ProductList) { HtmlDocument doc = new HtmlDocument(); for (int i = 0; i < PagesLinks_.Count; i++) { try { string source = client.DownloadString(PagesLinks_[i]); Console.Write(PagesLinks_[i] + " " + i.ToString() + "\n"); doc.LoadHtml(source); HtmlNodeNavigator navigator = (HtmlNodeNavigator)doc.CreateNavigator(); var productNodes = navigator.Select(Rules_.DetailsXpath); AddProducts(productNodes); var pagesNodes = navigator.Select(Rules_.PaginationXpath); AddPages(pagesNodes); } catch (System.Net.WebException) { } } ParsingResults res = new ParsingResults(); for (int i = 0; i < ProductsLinks_.Count; i++) { string source = client.DownloadString(ProductsLinks_[i]); Console.Write(ProductsLinks_[i] + " " + i.ToString() + "\n"); doc.LoadHtml(source); HtmlNodeNavigator navigator = (HtmlNodeNavigator)doc.CreateNavigator(); var product = IsAlreadyParsed(source, ProductList); if (product != null) { string priceValue = ConnectStrings(navigator.Select(Rules_.PriceXpath)); Models.Price price = new Models.Price() { Product = product, Date = DateTime.Now, PriceValue = priceValue }; product.Price = priceValue; res.AddedPrices.Add(price); } else { ParseNewProduct(navigator, product, res, ProductsLinks_[i]); } } return(res); }
public void Add(ParsingResults results) { AddedProducts.AddRange(results.AddedProducts); AddedPrices.AddRange(results.AddedPrices); AddedPictures.AddRange(results.AddedPictures); }
private void ParseNewProduct(HtmlNodeNavigator navigator, Models.Product product, ParsingResults res, string link) { string header = ConnectStrings(navigator.Select(Rules_.HeaderXpath)); string desctiption = ConnectStrings(navigator.Select(Rules_.DescriptionXpath)); string priceValue = ConnectStrings(navigator.Select(Rules_.PriceXpath)); List <string> pictures = ListValue(navigator.Select(Rules_.PicturesXpath)); product = new Models.Product() { Description = desctiption, Name = header, Link = link, ImageUrl = (pictures.Count > 0 ? pictures[0] : ""), Price = priceValue }; Models.Price price = new Models.Price() { Product = product, Date = DateTime.Now, PriceValue = priceValue }; List <Models.Picture> thumbs = new List <Models.Picture>(); foreach (string url in pictures) { thumbs.Add(new Models.Picture() { Product = product, PictureUrl = url }); } res.AddedPictures.AddRange(thumbs); res.AddedPrices.Add(price); res.AddedProducts.Add(product); }