public ParsingResults Parse(System.Net.WebClient client, List <Models.Product> ProductList)
{
HtmlDocument doc = new HtmlDocument();
for (int i = 0; i < PagesLinks_.Count; i++)
{
try
{
string source = client.DownloadString(PagesLinks_[i]);
Console.Write(PagesLinks_[i] + " " + i.ToString() + "\n");
doc.LoadHtml(source);
HtmlNodeNavigator navigator = (HtmlNodeNavigator)doc.CreateNavigator();
var productNodes = navigator.Select(Rules_.DetailsXpath);
AddProducts(productNodes);
var pagesNodes = navigator.Select(Rules_.PaginationXpath);
AddPages(pagesNodes);
}
catch (System.Net.WebException) { }
}
ParsingResults res = new ParsingResults();
for (int i = 0; i < ProductsLinks_.Count; i++)
{
string source = client.DownloadString(ProductsLinks_[i]);
Console.Write(ProductsLinks_[i] + " " + i.ToString() + "\n");
doc.LoadHtml(source);
HtmlNodeNavigator navigator = (HtmlNodeNavigator)doc.CreateNavigator();
var product = IsAlreadyParsed(source, ProductList);
if (product != null)
{
string priceValue = ConnectStrings(navigator.Select(Rules_.PriceXpath));
Models.Price price = new Models.Price()
{
Product = product, Date = DateTime.Now, PriceValue = priceValue
};
product.Price = priceValue;
res.AddedPrices.Add(price);
}
else
{
ParseNewProduct(navigator, product, res, ProductsLinks_[i]);
}
}
return(res);
}
public void Add(ParsingResults results)
{
AddedProducts.AddRange(results.AddedProducts);
AddedPrices.AddRange(results.AddedPrices);
AddedPictures.AddRange(results.AddedPictures);
}
private void ParseNewProduct(HtmlNodeNavigator navigator, Models.Product product, ParsingResults res, string link)
{
string header = ConnectStrings(navigator.Select(Rules_.HeaderXpath));
string desctiption = ConnectStrings(navigator.Select(Rules_.DescriptionXpath));
string priceValue = ConnectStrings(navigator.Select(Rules_.PriceXpath));
List <string> pictures = ListValue(navigator.Select(Rules_.PicturesXpath));
product = new Models.Product()
{
Description = desctiption,
Name = header,
Link = link,
ImageUrl = (pictures.Count > 0 ? pictures[0] : ""),
Price = priceValue
};
Models.Price price = new Models.Price()
{
Product = product, Date = DateTime.Now, PriceValue = priceValue
};
List <Models.Picture> thumbs = new List <Models.Picture>();
foreach (string url in pictures)
{
thumbs.Add(new Models.Picture()
{
Product = product, PictureUrl = url
});
}
res.AddedPictures.AddRange(thumbs);
res.AddedPrices.Add(price);
res.AddedProducts.Add(product);
}