private void SaveInBaseListDescriptionMessage(List <MessageModel> descriptionMessage, DateTime date)
{
var con = ConnetionToSqlServer.Default();
var regex = new Regex(@"\d+:\d+");
string format = "yyyy-MM-dd";
foreach (var message in descriptionMessage)
{
string dateStart = "null";
string dateEnd = "null";
string dateStartBargaining = "null";
if (message.DateStart.HasValue)
{
dateStart = $"'{((DateTime)message.DateStart).ToString(format)}'";
}
if (message.DateEnd.HasValue)
{
dateEnd = $"'{((DateTime)message.DateEnd).ToString(format)}'";
}
if (message.DateStartBargaining.HasValue)
{
dateStartBargaining = $"'{((DateTime)message.DateStartBargaining).ToString(format)}'";
}
string insert = $@"exec dbo.SaveMessage '{message.Id}', '{message.Number}',{dateStart}, {dateEnd},{dateStartBargaining}, '{message.Url}', '{message.Description}', '{message.DateMesage.ToString(format)}', '{message.TypeOfBidding}'";
//Log.Debug(insert);
con.ExecuteNonQuery(insert);
foreach (var lot in message.Lots)
{
insert = $@"exec dbo.SaveLot '{lot.Id}', {lot.Number}, '{lot.Description.Replace("'", "")}', {lot.StartPrice.ToString(System.Globalization.CultureInfo.GetCultureInfo("en-US"))}, '{lot.Step}', '{lot.Deposit}', '{lot.PriceReductionInformation}', '{lot.PropertyClassification}', '{message.Id}'";
con.ExecuteNonQuery(insert);
}
}
string typeMessage = GetTypeMessage(descriptionMessage.Count);
ParserBot.SendMessage($"За {date.ToShortDateString()} спарсено {descriptionMessage.Count} {typeMessage}");
}
private void ExecuteParse(DateTime date)
{
int pageNum;
GetCountPages(date, out pageNum);
var descriptionMessages = new List <MessageModel>();
var cookieContainer = new CookieContainer();
for (int page = 1; page <= pageNum; page++)
{
using (var handler = new HttpClientHandler()
{
CookieContainer = cookieContainer
})
{
HttpClient client = new HttpClient(handler);
SetCookiesAndHeaders(client, cookieContainer, date);
var content = GetContent(page, date);
HttpResponseMessage response = null;
try
{
response = client.PostAsync("http://bankrot.fedresurs.ru/Messages.aspx?attempt=1", content).Result;
}
catch (AggregateException ex)
{
//Log.Error("Ошибка при открии страницы с сообщениями");
//Log.Error(ex.Message);
handler.Dispose();
return;
}
if (response == null)
{
return;
}
if (!response.IsSuccessStatusCode)
{
//Log.Debug("Не удалось выполнить запрос к странице с сообщениями");
return;
}
var dataObjects = response.Content.ReadAsStringAsync();
HtmlParser parser = new HtmlParser();
var document = parser.ParseDocument(dataObjects.Result);
var table = document.GetElementById("ctl00_cphBody_gvMessages");
if (table == null)
{
//Log.Debug($"Страница - {page} за дату {date.ToShortDateString()} не содержит сообщений");
return;
}
var trs = table.GetElementsByTagName("tr");
if (trs.Length > 0)
{
var length = trs.Length;
for (int i = 1; i < length; i++)
{
var tr = trs[i];
var tds = tr.GetElementsByTagName("td");
if (tds.Length > 0)
{
var elemsA = tds[1].GetElementsByTagName("a");
if (elemsA.Length == 0)
{
continue;
}
var href = elemsA[0].GetAttribute("href");
Thread.Sleep(2000);
var uriStr = $@"{MainUri}{href}";
Uri uri;
if (Uri.TryCreate(uriStr, UriKind.RelativeOrAbsolute, out uri))
{
try
{
response = client.GetAsync(uri).Result;
}
catch (AggregateException ex)
{
//Log.Error("Ошибка при открии страницы с сообщениями");
//Log.Error(ex.Message);
handler.Dispose();
return;
}
if (!response.IsSuccessStatusCode)
{
//Log.Debug($"Не удалось выполнить запрос к странице с сообщениям - {uri.ToString()}");
continue;
}
dataObjects = response.Content.ReadAsStringAsync();
Thread.Sleep(1000);
var result = dataObjects.Result;
if (result.Contains("setting cookie"))
{
//Log.Debug("setting cookies");
ParserBot.SendMessage($"Необходимо обновить куки для парсера.");
return;
}
document = parser.ParseDocument(result);
var elemNumMessage = document.GetElementsByClassName("even");
if (elemNumMessage.Length > 0)
{
var numMessage = elemNumMessage[0].GetElementsByTagName("td")[1].TextContent.Trim();
var descriptionMessage = new MessageModel()
{
Number = numMessage, Url = uriStr, DateMesage = date
};
var descriptions = document.GetElementsByClassName("msg");
if (descriptions.Length > 0)
{
for (int k = 0; k < descriptions.Length; k++)
{
var str = descriptions[k].TextContent;
if (str.Contains("Текст:"))
{
descriptionMessage.Description = str.Replace("Текст:", string.Empty).Replace("'", string.Empty).Trim();
break;
}
}
}
var typeOfBidding = document.GetElementsByClassName("odd");
if (typeOfBidding.Length > 0)
{
for (int k = 0; k < typeOfBidding.Length; k++)
{
var str = typeOfBidding[k].TextContent;
if (str.Contains("Вид торгов:"))
{
descriptionMessage.TypeOfBidding = str.Replace("Вид торгов:", string.Empty).Replace("'", string.Empty).Trim();
break;
}
}
}
if (string.IsNullOrEmpty(descriptionMessage.TypeOfBidding))
{
typeOfBidding = document.GetElementsByClassName("even");
for (int k = 0; k < typeOfBidding.Length; k++)
{
var str = typeOfBidding[k].TextContent;
if (str.Contains("Вид торгов:"))
{
descriptionMessage.TypeOfBidding = str.Replace("Вид торгов:", string.Empty).Replace("'", string.Empty).Trim();
break;
}
}
}
ParseLot(document, descriptionMessage);
descriptionMessages.Add(descriptionMessage);
}
}
}
}
}
}
}
SaveInBaseListDescriptionMessage(descriptionMessages, date);
}