private void ParseNewProduct(HtmlNodeNavigator navigator, Models.Product product, ParsingResults res, string link)
{
string header = ConnectStrings(navigator.Select(Rules_.HeaderXpath));
string desctiption = ConnectStrings(navigator.Select(Rules_.DescriptionXpath));
string priceValue = ConnectStrings(navigator.Select(Rules_.PriceXpath));
List <string> pictures = ListValue(navigator.Select(Rules_.PicturesXpath));
product = new Models.Product()
{
Description = desctiption,
Name = header,
Link = link,
ImageUrl = (pictures.Count > 0 ? pictures[0] : ""),
Price = priceValue
};
Models.Price price = new Models.Price()
{
Product = product, Date = DateTime.Now, PriceValue = priceValue
};
List <Models.Picture> thumbs = new List <Models.Picture>();
foreach (string url in pictures)
{
thumbs.Add(new Models.Picture()
{
Product = product, PictureUrl = url
});
}
res.AddedPictures.AddRange(thumbs);
res.AddedPrices.Add(price);
res.AddedProducts.Add(product);
}
private void GetPurchaseObjects(HtmlNodeNavigator lotNav, MySqlConnection connect,
int idLot,
int customerId)
{
var okpd2 = (lotNav
.SelectSingleNode(
"./following-sibling::tr/th[.contains(text(), 'Код ОКРБ')]/following-sibling::td")
?.Value ?? "").Trim();
var purObjects = lotNav.Select(
"./td[2]/text()");
if (purObjects is null)
{
return;
}
foreach (XPathNavigator po in purObjects)
{
var namePo = po?.Value?.ReplaceHtmlEntyty()?.Trim() ?? "";
var insertLotitem =
$"INSERT INTO {AppBuilder.Prefix}purchase_object SET id_lot = @id_lot, id_customer = @id_customer, name = @name, quantity_value = @quantity_value, okei = @okei, customer_quantity_value = @customer_quantity_value, price = @price, sum = @sum, okpd2_code = @okpd2_code";
var cmd19 = new MySqlCommand(insertLotitem, connect);
cmd19.Prepare();
cmd19.Parameters.AddWithValue("@id_lot", idLot);
cmd19.Parameters.AddWithValue("@id_customer", customerId);
cmd19.Parameters.AddWithValue("@name", namePo);
cmd19.Parameters.AddWithValue("@quantity_value", "");
cmd19.Parameters.AddWithValue("@okei", "");
cmd19.Parameters.AddWithValue("@customer_quantity_value", "");
cmd19.Parameters.AddWithValue("@price", "");
cmd19.Parameters.AddWithValue("@sum", "");
cmd19.Parameters.AddWithValue("@okpd2_code", okpd2);
cmd19.ExecuteNonQuery();
}
}
public ParsingResults Parse(System.Net.WebClient client, List <Models.Product> ProductList)
{
HtmlDocument doc = new HtmlDocument();
for (int i = 0; i < PagesLinks_.Count; i++)
{
try
{
string source = client.DownloadString(PagesLinks_[i]);
Console.Write(PagesLinks_[i] + " " + i.ToString() + "\n");
doc.LoadHtml(source);
HtmlNodeNavigator navigator = (HtmlNodeNavigator)doc.CreateNavigator();
var productNodes = navigator.Select(Rules_.DetailsXpath);
AddProducts(productNodes);
var pagesNodes = navigator.Select(Rules_.PaginationXpath);
AddPages(pagesNodes);
}
catch (System.Net.WebException) { }
}
ParsingResults res = new ParsingResults();
for (int i = 0; i < ProductsLinks_.Count; i++)
{
string source = client.DownloadString(ProductsLinks_[i]);
Console.Write(ProductsLinks_[i] + " " + i.ToString() + "\n");
doc.LoadHtml(source);
HtmlNodeNavigator navigator = (HtmlNodeNavigator)doc.CreateNavigator();
var product = IsAlreadyParsed(source, ProductList);
if (product != null)
{
string priceValue = ConnectStrings(navigator.Select(Rules_.PriceXpath));
Models.Price price = new Models.Price()
{
Product = product, Date = DateTime.Now, PriceValue = priceValue
};
product.Price = priceValue;
res.AddedPrices.Add(price);
}
else
{
ParseNewProduct(navigator, product, res, ProductsLinks_[i]);
}
}
return(res);
}
private List <ResultEntity> GetNodeValue(string html, string xpath)
{
var list = new List <ResultEntity>();
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
var root = doc.DocumentNode;
HtmlNodeNavigator navigator = (HtmlNodeNavigator)root.CreateNavigator();
var nodes = navigator.Select(xpath);
; foreach (HtmlNodeNavigator node in nodes)
{
list.Add(new ResultEntity()
{
Value = node.Value, Path = node.CurrentNode.XPath
});
}
return(list);
}
private IEnumerable <string> GetProjects(string html)
{
//var list = Regex.Match(result, )
//project-full-name
var doc = new HtmlDocument();
doc.LoadHtml(html);
//var value = doc.DocumentNode
// .SelectNodes("//span[class=\"project-full-name\"]");
HtmlNodeNavigator navigator = (HtmlNodeNavigator)doc.CreateNavigator();
//Get value from given xpath
var xpath = "//span[@class='project-full-name']";
var nodes = navigator.Select(xpath);
foreach (XPathNavigator node in nodes)
{
yield return((string)node.TypedValue);
//yield return node
}
yield return(string.Empty);
}