public void ExecuteImport() { string fileNameToken = "token.xml"; XmlDocument xDoc = new XmlDocument(); xDoc.Load(fileNameToken); XmlElement xRoot = xDoc.DocumentElement; ParserBot.SetToken(xRoot.ChildNodes[0].Attributes.GetNamedItem("Token").Value); }
private void SaveInBaseListDescriptionMessage(List <MessageModel> descriptionMessage, DateTime date) { var con = ConnetionToSqlServer.Default(); var regex = new Regex(@"\d+:\d+"); string format = "yyyy-MM-dd"; foreach (var message in descriptionMessage) { string dateStart = "null"; string dateEnd = "null"; string dateStartBargaining = "null"; if (message.DateStart.HasValue) { dateStart = $"'{((DateTime)message.DateStart).ToString(format)}'"; } if (message.DateEnd.HasValue) { dateEnd = $"'{((DateTime)message.DateEnd).ToString(format)}'"; } if (message.DateStartBargaining.HasValue) { dateStartBargaining = $"'{((DateTime)message.DateStartBargaining).ToString(format)}'"; } string insert = $@"exec dbo.SaveMessage '{message.Id}', '{message.Number}',{dateStart}, {dateEnd},{dateStartBargaining}, '{message.Url}', '{message.Description}', '{message.DateMesage.ToString(format)}', '{message.TypeOfBidding}'"; //Log.Debug(insert); con.ExecuteNonQuery(insert); foreach (var lot in message.Lots) { insert = $@"exec dbo.SaveLot '{lot.Id}', {lot.Number}, '{lot.Description.Replace("'", "")}', {lot.StartPrice.ToString(System.Globalization.CultureInfo.GetCultureInfo("en-US"))}, '{lot.Step}', '{lot.Deposit}', '{lot.PriceReductionInformation}', '{lot.PropertyClassification}', '{message.Id}'"; con.ExecuteNonQuery(insert); } } string typeMessage = GetTypeMessage(descriptionMessage.Count); ParserBot.SendMessage($"За {date.ToShortDateString()} спарсено {descriptionMessage.Count} {typeMessage}"); }
private void ExecuteParse(DateTime date) { int pageNum; GetCountPages(date, out pageNum); var descriptionMessages = new List <MessageModel>(); var cookieContainer = new CookieContainer(); for (int page = 1; page <= pageNum; page++) { using (var handler = new HttpClientHandler() { CookieContainer = cookieContainer }) { HttpClient client = new HttpClient(handler); SetCookiesAndHeaders(client, cookieContainer, date); var content = GetContent(page, date); HttpResponseMessage response = null; try { response = client.PostAsync("http://bankrot.fedresurs.ru/Messages.aspx?attempt=1", content).Result; } catch (AggregateException ex) { //Log.Error("Ошибка при открии страницы с сообщениями"); //Log.Error(ex.Message); handler.Dispose(); return; } if (response == null) { return; } if (!response.IsSuccessStatusCode) { //Log.Debug("Не удалось выполнить запрос к странице с сообщениями"); return; } var dataObjects = response.Content.ReadAsStringAsync(); HtmlParser parser = new HtmlParser(); var document = parser.ParseDocument(dataObjects.Result); var table = document.GetElementById("ctl00_cphBody_gvMessages"); if (table == null) { //Log.Debug($"Страница - {page} за дату {date.ToShortDateString()} не содержит сообщений"); return; } var trs = table.GetElementsByTagName("tr"); if (trs.Length > 0) { var length = trs.Length; for (int i = 1; i < length; i++) { var tr = trs[i]; var tds = tr.GetElementsByTagName("td"); if (tds.Length > 0) { var elemsA = tds[1].GetElementsByTagName("a"); if (elemsA.Length == 0) { continue; } var href = elemsA[0].GetAttribute("href"); Thread.Sleep(2000); var uriStr = $@"{MainUri}{href}"; Uri uri; if (Uri.TryCreate(uriStr, UriKind.RelativeOrAbsolute, out uri)) { try { response = client.GetAsync(uri).Result; } catch (AggregateException ex) { //Log.Error("Ошибка при открии страницы с сообщениями"); //Log.Error(ex.Message); handler.Dispose(); return; } if (!response.IsSuccessStatusCode) { //Log.Debug($"Не удалось выполнить запрос к странице с сообщениям - {uri.ToString()}"); continue; } dataObjects = response.Content.ReadAsStringAsync(); Thread.Sleep(1000); var result = dataObjects.Result; if (result.Contains("setting cookie")) { //Log.Debug("setting cookies"); ParserBot.SendMessage($"Необходимо обновить куки для парсера."); return; } document = parser.ParseDocument(result); var elemNumMessage = document.GetElementsByClassName("even"); if (elemNumMessage.Length > 0) { var numMessage = elemNumMessage[0].GetElementsByTagName("td")[1].TextContent.Trim(); var descriptionMessage = new MessageModel() { Number = numMessage, Url = uriStr, DateMesage = date }; var descriptions = document.GetElementsByClassName("msg"); if (descriptions.Length > 0) { for (int k = 0; k < descriptions.Length; k++) { var str = descriptions[k].TextContent; if (str.Contains("Текст:")) { descriptionMessage.Description = str.Replace("Текст:", string.Empty).Replace("'", string.Empty).Trim(); break; } } } var typeOfBidding = document.GetElementsByClassName("odd"); if (typeOfBidding.Length > 0) { for (int k = 0; k < typeOfBidding.Length; k++) { var str = typeOfBidding[k].TextContent; if (str.Contains("Вид торгов:")) { descriptionMessage.TypeOfBidding = str.Replace("Вид торгов:", string.Empty).Replace("'", string.Empty).Trim(); break; } } } if (string.IsNullOrEmpty(descriptionMessage.TypeOfBidding)) { typeOfBidding = document.GetElementsByClassName("even"); for (int k = 0; k < typeOfBidding.Length; k++) { var str = typeOfBidding[k].TextContent; if (str.Contains("Вид торгов:")) { descriptionMessage.TypeOfBidding = str.Replace("Вид торгов:", string.Empty).Replace("'", string.Empty).Trim(); break; } } } ParseLot(document, descriptionMessage); descriptionMessages.Add(descriptionMessage); } } } } } } } SaveInBaseListDescriptionMessage(descriptionMessages, date); }