private void GetPage(int num) { var data = $"{{\"filter\":{{\"auctionSpecificFilter\":{{}},\"needSpecificFilter\":{{}},\"tenderSpecificFilter\":{{}}}},\"order\":[{{\"field\":\"PublishDate\",\"desc\":true}}],\"withCount\":true,\"take\":50,\"skip\":{num * 50}}}"; var url = $"https://old.zakupki.mos.ru/api/Cssp/Purchase/Query?queryDto={{\"filter\":{{\"auctionSpecificFilter\":{{}},\"needSpecificFilter\":{{}},\"tenderSpecificFilter\":{{}}}},\"order\":[{{\"field\":\"PublishDate\",\"desc\":true}}],\"withCount\":true,\"take\":50,\"skip\":{num * 50}}}"; url = Uri.EscapeUriString(url); var s = DownloadString.DownLHttpPostWithCookiesB2b(url, cookie: null, useProxy: AppBuilder.UserProxy); if (string.IsNullOrEmpty(s)) { Log.Logger($"Empty string in {GetType().Name}.{System.Reflection.MethodBase.GetCurrentMethod().Name}", _url); return; } var jObj = JObject.Parse(s); var tenders = GetElements(jObj, "items"); foreach (var t in tenders) { try { ParserTenderObj(t); } catch (Exception e) { Log.Logger($"Error in {GetType().Name}.{System.Reflection.MethodBase.GetCurrentMethod().Name}", e, t.ToString()); } } }
public void ParsingTender() { using (var connect = ConnectToDb.GetDbConnection()) { var dateUpd = DateTime.Now; connect.Open(); if (TenderExist(connect)) { return; } var s = DownloadString.DownLHttpPostWithCookiesB2b(_tn.Href, ParserB2BWeb.CookieCollection, useProxy: AppBuilder.UserProxy); if (string.IsNullOrEmpty(s)) { Log.Logger("Empty string in ParsingTender()", _tn.Href); return; } if (s.Contains("не допускает использование ботов")) { Log.Logger("Google Captcha"); return; } var htmlDoc = new HtmlDocument(); htmlDoc.LoadHtml(s); var navigator = (HtmlNodeNavigator)htmlDoc.CreateNavigator(); UpdateCancelStatus(connect, dateUpd, out var updated, out var cancelStatus); var printForm = _tn.Href; AddOrganizer(connect, navigator, out var organiserId); AddCustomer(connect, out var customerId); GetEtp(connect, out var idEtp); GetPlacingWay(connect, out var idPlacingWay); FillPurName(navigator); FillBidAndScorDates(navigator, out var scoringDate, out var biddingDate); FillNoticeVer(navigator, out var noticeVer); AddTender(connect, organiserId, idPlacingWay, idEtp, scoringDate, biddingDate, cancelStatus, dateUpd, noticeVer, printForm, updated, out var idTender); AddAttachments(htmlDoc, connect, idTender); AddLots(htmlDoc, navigator, connect, idTender, customerId); TenderKwords(connect, idTender); AddVerNumber(connect, _tn.PurNum, TypeFz); } }
private void GetPage(string url) { var result = DownloadString.DownLHttpPostWithCookiesB2b(url, CookieCollection, useProxy: AppBuilder.UserProxy); if (string.IsNullOrEmpty(result)) { Log.Logger($"Empty string in {GetType().Name}.{System.Reflection.MethodBase.GetCurrentMethod().Name}", url); return; } if (result.Contains("не допускает использование ботов")) { Log.Logger("Google Captcha"); return; } var htmlDoc = new HtmlDocument(); htmlDoc.LoadHtml(result); var tens = htmlDoc.DocumentNode.SelectNodes( "//table[contains(@class, 'search-results')]/tbody/tr") ?? new HtmlNodeCollection(null); foreach (var a in tens) { try { ParserTender(a, url); } catch (Exception e) { Log.Logger(e); } } }
private void AddLot(HtmlNodeNavigator navigator, MySqlConnection connect, int idTender, int customerId, HtmlNode lot, out bool lotWasAdded) { lotWasAdded = false; var lotNumT = lot.InnerText?.Trim().GetDataFromRegex(@"Лот № (\d+)"); var succesLotNum = int.TryParse(lotNumT, out var lotNum); if (!succesLotNum) { lotNum = 1; } var lotHref = lot.Attributes["href"]?.Value ?? ""; if (!string.IsNullOrEmpty(lotHref)) { return; } lotHref = $"https://www.b2b-center.ru{lotHref}"; var sLot = DownloadString.DownLHttpPostWithCookiesB2b(lotHref, ParserB2BWeb.CookieCollection, useProxy: AppBuilder.UserProxy); if (string.IsNullOrEmpty(sLot)) { Log.Logger("Empty string in lot ParsingTender()", lotHref); return; } var htmlLot = new HtmlDocument(); htmlLot.LoadHtml(sLot); var navLot = (HtmlNodeNavigator)htmlLot.CreateNavigator(); var priceT = navLot.SelectSingleNode( "//td[contains(., 'Цена за единицу продукции:')]/following-sibling::td/b") ?.Value?.Trim() ?? ""; var currency = navigator.SelectSingleNode( "//td[contains(., 'Вид валюты:')]/following-sibling::td") ?.Value?.Trim() ?? "руб."; var price = priceT.ExtractPrice(); var nmckT = navigator.SelectSingleNode( "//td[contains(., 'Начальная (максимальная) цена договора:' ) or contains(., 'Начальная цена всего лота:') or contains(., 'Общая стоимость')]/following-sibling::td/b") ?.Value?.Trim() ?? ""; var nmck = nmckT.ExtractPrice(); var lotName = navLot.SelectSingleNode( "//div[@class = 'expandable-text short']/span") ?.Value?.Trim() ?? ""; if (string.IsNullOrEmpty(lotName.DelAllWhitespace())) { lotName = _tn.PurName; } var insertLot = $"INSERT INTO {AppBuilder.Prefix}lot SET id_tender = @id_tender, lot_number = @lot_number, max_price = @max_price, currency = @currency, finance_source = @finance_source, lot_name = @lot_name"; var cmd18 = new MySqlCommand(insertLot, connect); cmd18.Prepare(); cmd18.Parameters.AddWithValue("@id_tender", idTender); cmd18.Parameters.AddWithValue("@lot_number", lotNum); cmd18.Parameters.AddWithValue("@max_price", nmck); cmd18.Parameters.AddWithValue("@currency", currency); cmd18.Parameters.AddWithValue("@finance_source", ""); cmd18.Parameters.AddWithValue("@lot_name", lotName); cmd18.ExecuteNonQuery(); var idLot = (int)cmd18.LastInsertedId; lotWasAdded = true; AddPurObject(connect, customerId, navLot, idLot, lotName, price); AddCustRequirements(connect, customerId, navLot, idLot, nmck); }
private void ParserNeed(MySqlConnection connect) { var url = $"https://old.zakupki.mos.ru/api/Cssp/Need/GetEntity?id={_tn.NeedId}"; var s = DownloadString.DownLHttpPostWithCookiesB2b(url, cookie: null, useProxy: AppBuilder.UserProxy); if (string.IsNullOrEmpty(s)) { Log.Logger( $"Empty string in {GetType().Name}.{System.Reflection.MethodBase.GetCurrentMethod().Name}", _tn.Href); return; } var tender = JObject.Parse(s); var purNum = _tn.PurNum; var noticeVersion = _tn.Status; var dateUpd = DateTime.Now; var(updated, cancelStatus) = UpdateTenderVersion(connect, purNum, dateUpd); var printForm = _tn.Href; var organiserId = 0; if (_tn.OrgName != "") { var selectOrg = $"SELECT id_organizer FROM {AppBuilder.Prefix}organizer WHERE full_name = @full_name"; var cmd3 = new MySqlCommand(selectOrg, connect); cmd3.Prepare(); cmd3.Parameters.AddWithValue("@full_name", _tn.OrgName); var dt3 = new DataTable(); var adapter3 = new MySqlDataAdapter { SelectCommand = cmd3 }; adapter3.Fill(dt3); if (dt3.Rows.Count > 0) { organiserId = (int)dt3.Rows[0].ItemArray[0]; } else { var phone = ((string)tender.SelectToken("contactPhone") ?? "").Trim(); var email = ""; var contactPerson = ((string)tender.SelectToken("contactPerson") ?? "").Trim(); var inn = _tn.OrgInn; var kpp = ((string)tender.SelectToken("createdByCompany.kpp") ?? "").Trim(); var postAddr = ((string)tender.SelectToken("createdByCompany.legalAddress") ?? "").Trim(); var addOrganizer = $"INSERT INTO {AppBuilder.Prefix}organizer SET full_name = @full_name, contact_phone = @contact_phone, contact_person = @contact_person, contact_email = @contact_email, inn = @inn, kpp = @kpp, post_address = @post_address"; var cmd4 = new MySqlCommand(addOrganizer, connect); cmd4.Prepare(); cmd4.Parameters.AddWithValue("@full_name", _tn.OrgName); cmd4.Parameters.AddWithValue("@contact_phone", phone); cmd4.Parameters.AddWithValue("@contact_person", contactPerson); cmd4.Parameters.AddWithValue("@contact_email", email); cmd4.Parameters.AddWithValue("@inn", inn); cmd4.Parameters.AddWithValue("@kpp", kpp); cmd4.Parameters.AddWithValue("@post_address", postAddr); cmd4.ExecuteNonQuery(); organiserId = (int)cmd4.LastInsertedId; } } var idPlacingWay = 0; GetEtp(connect, out var idEtp); var idRegion = GetRegionFromString(_tn.RegionName, connect); var insertTender = $"INSERT INTO {AppBuilder.Prefix}tender SET id_region = @id_region, id_xml = @id_xml, purchase_number = @purchase_number, doc_publish_date = @doc_publish_date, href = @href, purchase_object_info = @purchase_object_info, type_fz = @type_fz, id_organizer = @id_organizer, id_placing_way = @id_placing_way, id_etp = @id_etp, end_date = @end_date, scoring_date = @scoring_date, bidding_date = @bidding_date, cancel = @cancel, date_version = @date_version, num_version = @num_version, notice_version = @notice_version, xml = @xml, print_form = @print_form"; var cmd9 = new MySqlCommand(insertTender, connect); cmd9.Prepare(); cmd9.Parameters.AddWithValue("@id_region", idRegion); cmd9.Parameters.AddWithValue("@id_xml", purNum); cmd9.Parameters.AddWithValue("@purchase_number", purNum); cmd9.Parameters.AddWithValue("@doc_publish_date", _tn.DatePub); cmd9.Parameters.AddWithValue("@href", _tn.Href); cmd9.Parameters.AddWithValue("@purchase_object_info", _tn.PurName); cmd9.Parameters.AddWithValue("@type_fz", TypeFz); cmd9.Parameters.AddWithValue("@id_organizer", organiserId); cmd9.Parameters.AddWithValue("@id_placing_way", idPlacingWay); cmd9.Parameters.AddWithValue("@id_etp", idEtp); cmd9.Parameters.AddWithValue("@end_date", _tn.DateEnd); cmd9.Parameters.AddWithValue("@scoring_date", DateTime.MinValue); cmd9.Parameters.AddWithValue("@bidding_date", DateTime.MinValue); cmd9.Parameters.AddWithValue("@cancel", cancelStatus); cmd9.Parameters.AddWithValue("@date_version", dateUpd); cmd9.Parameters.AddWithValue("@num_version", 1); cmd9.Parameters.AddWithValue("@notice_version", noticeVersion); cmd9.Parameters.AddWithValue("@xml", _tn.Href); cmd9.Parameters.AddWithValue("@print_form", printForm); var resInsertTender = cmd9.ExecuteNonQuery(); var idTender = (int)cmd9.LastInsertedId; Counter(resInsertTender, updated); var attacments = GetElements(tender, "files"); WriteAttachments(connect, attacments, idTender); var lotNum = 1; var insertLot = $"INSERT INTO {AppBuilder.Prefix}lot SET id_tender = @id_tender, lot_number = @lot_number, max_price = @max_price, currency = @currency"; var cmd18 = new MySqlCommand(insertLot, connect); cmd18.Prepare(); cmd18.Parameters.AddWithValue("@id_tender", idTender); cmd18.Parameters.AddWithValue("@lot_number", lotNum); cmd18.Parameters.AddWithValue("@max_price", _tn.Nmck); cmd18.Parameters.AddWithValue("@currency", ""); cmd18.ExecuteNonQuery(); var idLot = (int)cmd18.LastInsertedId; _tn.Customers.ForEach(cus => { var customerId = 0; if (!string.IsNullOrEmpty(cus.CusName)) { var selectCustomer = $"SELECT id_customer FROM {AppBuilder.Prefix}customer WHERE full_name = @full_name"; var cmd13 = new MySqlCommand(selectCustomer, connect); cmd13.Prepare(); cmd13.Parameters.AddWithValue("@full_name", cus.CusName); var reader7 = cmd13.ExecuteReader(); if (reader7.HasRows) { reader7.Read(); customerId = (int)reader7["id_customer"]; reader7.Close(); } else { reader7.Close(); var insertCustomer = $"INSERT INTO {AppBuilder.Prefix}customer SET reg_num = @reg_num, full_name = @full_name, inn = @inn, is223=1"; var cmd14 = new MySqlCommand(insertCustomer, connect); cmd14.Prepare(); var customerRegNumber = Guid.NewGuid().ToString(); cmd14.Parameters.AddWithValue("@reg_num", customerRegNumber); cmd14.Parameters.AddWithValue("@full_name", cus.CusName); cmd14.Parameters.AddWithValue("@inn", cus.CusInn); cmd14.ExecuteNonQuery(); customerId = (int)cmd14.LastInsertedId; } } var delivPl = ((string)tender.SelectToken( "deliveryPlace") ?? "").Trim(); var delivTerm = ((string)tender.SelectToken( "paymentTerms") ?? "").Trim(); if (delivTerm != "" || delivPl != "") { var insertCustomerRequirement = $"INSERT INTO {AppBuilder.Prefix}customer_requirement SET id_lot = @id_lot, id_customer = @id_customer, max_price = @max_price, delivery_place = @delivery_place, delivery_term = @delivery_term"; var cmd16 = new MySqlCommand(insertCustomerRequirement, connect); cmd16.Prepare(); cmd16.Parameters.AddWithValue("@id_lot", idLot); cmd16.Parameters.AddWithValue("@id_customer", customerId); cmd16.Parameters.AddWithValue("@delivery_place", delivPl); cmd16.Parameters.AddWithValue("@delivery_term", delivTerm); cmd16.Parameters.AddWithValue("@max_price", _tn.Nmck); cmd16.ExecuteNonQuery(); } var purObjects = GetElements(tender, "items"); purObjects.ForEach(po => { var okpdName = ((string)po.SelectToken( "okpd.name") ?? "").Trim(); var okpdCode = ((string)po.SelectToken( "okpd.code") ?? "").Trim(); var okei = ((string)po.SelectToken( "okei.name") ?? "").Trim(); var quantity = (decimal?)po.SelectToken( "amount") ?? 0.0m; var price = (decimal?)po.SelectToken( "cost") ?? 0.0m; var sum = (decimal?)po.SelectToken( "totalCost") ?? 0.0m; var poName = ((string)po.SelectToken( "name") ?? "").Trim(); var insertLotitem = $"INSERT INTO {AppBuilder.Prefix}purchase_object SET id_lot = @id_lot, id_customer = @id_customer, name = @name, sum = @sum, okpd2_code = @okpd2_code, okpd2_group_code = @okpd2_group_code, okpd2_group_level1_code = @okpd2_group_level1_code, okpd_name = @okpd_name, quantity_value = @quantity_value, customer_quantity_value = @customer_quantity_value, okei = @okei, price = @price"; var cmd19 = new MySqlCommand(insertLotitem, connect); cmd19.Prepare(); cmd19.Parameters.AddWithValue("@id_lot", idLot); cmd19.Parameters.AddWithValue("@id_customer", customerId); cmd19.Parameters.AddWithValue("@name", poName); cmd19.Parameters.AddWithValue("@sum", sum); cmd19.Parameters.AddWithValue("@okpd2_code", okpdCode); cmd19.Parameters.AddWithValue("@okpd2_group_code", ""); cmd19.Parameters.AddWithValue("@okpd2_group_level1_code", ""); cmd19.Parameters.AddWithValue("@okpd_name", okpdName); cmd19.Parameters.AddWithValue("@quantity_value", quantity); cmd19.Parameters.AddWithValue("@customer_quantity_value", quantity); cmd19.Parameters.AddWithValue("@okei", okei); cmd19.Parameters.AddWithValue("@price", price); cmd19.ExecuteNonQuery(); }); }); TenderKwords(connect, idTender); AddVerNumber(connect, purNum, TypeFz); }