private List <string[]> processHtml(string html)
{
List <string[]> outputList = new List <string[]>();
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(html);
var dom = htmlDocument.DocumentNode.Descendants().First().Name;
var element = htmlDocument.DocumentNode.Descendants().First().Attributes.ToList();
foreach (var elem in element)
{
var elementName = elem.Name;
var value = elem.Value;
string[] outputString = { dom, elementName, value };
outputList.Add(outputString);
}
string text = htmlDocument.DocumentNode.Descendants().First().InnerText;
if (!String.IsNullOrEmpty(text))
{
string[] domOutput = { dom, text };
outputList.Add(domOutput);
}
return(outputList);
}
public override string[,] SearchMovies(string keywords)
{
string searchAddress = SEARCH_ADDRESS + keywords;
HtmlWeb web = new HtmlWeb();
HtmlAgilityPack.HtmlDocument document = web.Load(searchAddress);
HtmlNode[] items = document.DocumentNode.SelectNodes("//div[@class='tray-item ']").ToArray();
int lengthNode = items.Length;
HtmlNode nodeSlug;
HtmlNode nodeTitle;
HtmlNode nodeEpisodes;
if (lengthNode > 10)
{
lengthNode = 10;
}
string[,] ListItems = new string[lengthNode, 3];
for (int i = 0; i < lengthNode; i++)
{
nodeSlug = items[i].SelectSingleNode(".//a");
nodeTitle = items[i].SelectSingleNode(".//div[@class='tray-item-title']");
nodeEpisodes = items[i].SelectSingleNode(".//div[@class='tray-film-update']");
ListItems[i, 0] = nodeTitle.InnerText.Trim();
ListItems[i, 1] = nodeEpisodes.InnerText.Trim();
ListItems[i, 2] = nodeSlug.Attributes["href"].Value.Trim();
}
return(ListItems);
}
private void UpdateDoviz()
{
try
{
client = new WebClient();
client.Headers.Add("User-Agent: Other");
html = client.DownloadString(url);
document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(html);
HtmlNodeCollection Currency =
document.DocumentNode.SelectNodes("//div");
/*
* HtmlNodeCollection dovizler = document.DocumentNode.SelectNodes("//span[@class='menu-row2']");
* UpdateScreen(dovizler[1].InnerText, dovizler[2].InnerText);
*/
if (Currency.Count < 33)
{
return;
}
if (InvokeRequired)
{
BeginInvoke(new DelegateUpdateScreen(() => UpdateScreen(
Currency[26].InnerText, Currency[31].InnerText, Currency[29].InnerText, Currency[28].InnerText,
Currency[16].InnerText, Currency[21].InnerText, Currency[19].InnerText, Currency[18].InnerText,
Currency[24].InnerText, Currency[25].InnerText, Currency[14].InnerText,
Currency[15].InnerText)));
}
else
{
UpdateScreen(
Currency[26].InnerText, Currency[31].InnerText, Currency[29].InnerText, Currency[28].InnerText,
Currency[16].InnerText, Currency[21].InnerText, Currency[19].InnerText, Currency[18].InnerText,
Currency[24].InnerText, Currency[25].InnerText, Currency[14].InnerText, Currency[15].InnerText);
}
txtLastUpdate.ForeColor = defaultTimeColor;
}
catch
{
if (InvokeRequired)
{
BeginInvoke(new DelegateUpdateScreen(RefreshError));
}
else
{
RefreshError();
}
}
}
private void RefreshPrtInfo(string sBusDate, string sCheckCode)
{
Stopwatch sw = new Stopwatch();
sw.Start();
if (string.IsNullOrEmpty(sCheckCode))
{
return;
}
if (doc == null)
{
doc = new HtmlWeb().Load(WbPrtStatic.PRT_TEMPLATE_FILE_PATH + @"so" + WbPrtStatic.PRT_TEMPLATE_FILE_NAME_SUFFIX);
}
string strSqlWhere = "";
DynamicParameters dynamicParams = new DynamicParameters();
strSqlWhere = "CheckCode=@CheckCode AND BusDate=@BusDate";
dynamicParams.Add("BusDate", sBusDate);
dynamicParams.Add("CheckCode", sCheckCode);
//var lstOI = CommonData.TaOrderItem.Where(s => s.CheckCode.Equals(strChkOrder) && s.BusDate.Equals(checkBusDate)).ToList();
var lstOI = new SQLiteDbHelper().QueryMultiByWhere <TaOrderItemInfo>("Ta_OrderItem", strSqlWhere, dynamicParams);
string htmlText = doc.Text;
wbPtl = GetAllPrtInfo();
if (string.IsNullOrEmpty(htmlText))
{
webBrowser2.DocumentText = "";
}
htmlText = WbPrtPrint.ReplaceHtmlPrtKeysShop(htmlText, wbPtl);
htmlText = WbPrtPrint.GetOrderItemInfo(doc, htmlText, lstOI, false);
webBrowser2.DocumentText = htmlText;
sw.Stop();
TimeSpan ts = sw.Elapsed;
Console.WriteLine(@"FrmTaShowOrder/RefreshPrtInfo Time:{0}", ts.TotalMilliseconds);
//webBrowser2.Refresh();
}
private List <string> checkHtml(string html, List <string[]> outputList)
{
seleniumList.Columns.Clear();
seleniumList.Items.Clear();
if (previousElement != null)
{
previousElement.Style = previousStyle;
}
List <string> actualOutput = new List <string>();
foreach (var output in outputList)
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(html);
string fullOutput;
if (output.Length == 3)
{
fullOutput = $"//{output[0]}[@{output[1]}='{output[2]}']";
}
else
{
fullOutput = $"//{output[0]}[text()='{output[1]}']";
}
HtmlNodeCollection nodes = null;
try
{
nodes = htmlDocument.DocumentNode.SelectNodes(fullOutput);
}
catch
{
//Nothing
}
try
{
int count = 0;
foreach (var node in nodes)
{
try
{
var style = node.Attributes["Style"].Value.Replace(" ", "");
if (languageSelection.Text == "Python")
{
if (style.Contains(highlightColor) &&
!fullOutput.Contains(highlightColor))
{
if (count == 0)
{
if (output[1] == "class")
{
actualOutput.Add(
"find_element_by_class_name(\"" + output[2] +
"\")");
}
else
{
actualOutput.Add(
"find_element_by_xpath(\"" + fullOutput + "\")");
}
}
else
{
if (output[1] == "class")
{
actualOutput.Add(
"find_elements_by_class_name(\"" + output[2] +
"\")[" +
count + "]");
}
else
{
actualOutput.Add(
"find_elements_by_xpath(\"" + fullOutput + "\")[" +
count +
"]");
}
}
}
}
if (languageSelection.Text == "Java")
{
if (style.Contains(highlightColor) &&
!fullOutput.Contains(highlightColor))
{
if (count == 0)
{
actualOutput.Add(
"findElement(By.xpath(\"" + fullOutput + "\")");
}
else
{
actualOutput.Add(
"findElements(By.xpath(\"" + fullOutput + "\")[" +
count +
"]");
}
}
}
}
catch
{
//Nothing
}
count++;
}
}
catch
{
//Nothing
}
}
return(actualOutput);
}
public void generateEarnings()
{
List <Earnings> lstEarning = new List <Earnings>();
WebClient webClient = new WebClient();
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
magentadbDataContext dbProd = new magentadbDataContext();
DateTime currentDate = Convert.ToDateTime(txtDateStart.Value.ToString("yyyy-MM-dd"));
DateTime endDate = Convert.ToDateTime(txtDateEnd.Value.ToString("yyyy-MM-dd"));
int dateDiff = Convert.ToInt32(endDate.Subtract(currentDate).Days.ToString());
int offset = 0;
progressBar1.Maximum = dateDiff + 1;
progressBar1.Value = 0;
while (currentDate <= endDate)
{
string urlDateString = currentDate.ToString("yyyy-MM-dd");
string url = webClient.DownloadString("https://finance.yahoo.com/calendar/earnings?day=" + urlDateString + "&offset=" + offset);
doc.LoadHtml(url);
if (doc.DocumentNode.SelectNodes("//table[contains(@class, 'data-table W(100%) Bdcl(c) Pos(r) BdB Bdc($c-fuji-grey-c)')]") == null)
{
txtLogActivity.Text += urlDateString + ": No data. \r\n";
}
else
{
foreach (HtmlNode table in doc.DocumentNode.SelectNodes("//table[contains(@class, 'data-table W(100%) Bdcl(c) Pos(r) BdB Bdc($c-fuji-grey-c)')]"))
{
foreach (HtmlNode tableBody in table.SelectNodes("tbody"))
{
txtLogActivity.Text += urlDateString + ": " + tableBody.SelectNodes("tr").Count() + " symbols \r\n";
if (tableBody.SelectNodes("tr").Count() == 100)
{
offset = offset + 100;
}
else
{
offset = 0;
}
foreach (HtmlNode tableRow in tableBody.SelectNodes("tr"))
{
HtmlNodeCollection tableRows = tableRow.SelectNodes("td");
if (tableRows.Any())
{
try
{
txtLogActivity.Text += tableRows[1].InnerText + "\r\n";
txtLogActivity.SelectionStart = txtLogActivity.Text.Length;
txtLogActivity.ScrollToCaret();
//txtLogActivity.Text = "Symbol: " + tableRows[1].InnerText + "\r\n" + txtLogActivity.Text;
//txtLogActivity.Text = "Company: " + tableRows[2].InnerText + "\r\n" + txtLogActivity.Text;
//txtLogActivity.Text = "Earnings Call Time: " + tableRows[3].InnerText + "\r\n" + txtLogActivity.Text;
//txtLogActivity.Text = "EPS Estimate: " + tableRows[4].InnerText + "\r\n" + txtLogActivity.Text;
//txtLogActivity.Text = "Reported EPS: " + tableRows[5].InnerText + "\r\n" + txtLogActivity.Text;
//txtLogActivity.Text = "Surprise (%): " + tableRows[6].InnerText + "\r\n" + txtLogActivity.Text;
//txtLogActivity.Text = "\r\n\n" + txtLogActivity.Text;
TrnStockEarning newTrnStockEarning = new TrnStockEarning();
string earningSymbol = tableRows[1].InnerText.ToUpper();
string earningPosition = "Before Market Open";
if (tableRows[3].InnerText.Equals("After Market Close"))
{
earningPosition = "After Market Close";
}
if (dbProd.TrnStockEarnings.Where(e => e.Symbol == earningSymbol && e.EarningDate.Date == currentDate.Date).Count() == 0)
{
var MstSymbol = from s in dbProd.MstSymbols
where s.Symbol == earningSymbol &&
(s.Exchange == "NASDAQ" || s.Exchange == "NYSE" || s.Exchange == "AMEX")
select new
{
Id = s.Id,
};
if (MstSymbol.Any())
{
newTrnStockEarning.Symbol = earningSymbol;
newTrnStockEarning.SymbolId = MstSymbol.FirstOrDefault().Id;
newTrnStockEarning.EarningDate = currentDate.Date;
newTrnStockEarning.EarningTime = earningPosition;
dbProd.TrnStockEarnings.InsertOnSubmit(newTrnStockEarning);
dbProd.SubmitChanges();
}
}
}
catch
{
txtLogActivity.Text += "Error saving. \r\n";
}
}
}
}
}
}
if (offset == 0)
{
currentDate = currentDate.AddDays(1);
progressBar1.Value += 1;
}
}
MessageBox.Show("Completed ", "", MessageBoxButtons.OK, MessageBoxIcon.Information);
}
public ActionResult Index(string startId) //int startId, int finishId
{
HtmlAgilityPack.HtmlWeb web = new HtmlWeb();
for (int i = 0; i < 11; i++)
{
HtmlAgilityPack.HtmlDocument doc = web.Load("https://www.avito.ru/moskva/kvartiry/prodam/1-komnatnye/vtorichka?p=" + i + "&user=1&view=list");
//foreach (HtmlNode row in doc.DocumentNode.SelectNodes("//div[@class='item item_list js-catalog-item-enum clearfix item-highlight cat_24']"))
//foreach (HtmlNode row in doc.DocumentNode.SelectNodes("//div[@class='item item_list js-catalog-item-enum clearfix c-b-0 cat_24']"))
foreach (HtmlNode row in doc.DocumentNode.SelectNodes("//div[contains(@class, 'js-catalog-item-enum') and contains(@class, 'clearfix')]"))
{
// ссылка на подробную объяву
var link = row.SelectSingleNode("div[@class='title description-title']/h3[@class='h3 fader description-title-h3']/a").GetAttributeValue("href", "");
HtmlAgilityPack.HtmlDocument childDoc = web.Load("https://www.avito.ru" + link);
var opisanie = childDoc.DocumentNode.SelectSingleNode("//div[@class = 'item-description-text']/p").InnerHtml;
// id
var adId = row.Id;
var sAdId = adId.Replace("i", "");
// цена
var price = row.SelectSingleNode("div[@class='price']/p").InnerText;
var bPrice = new StringBuilder();
foreach (var p in price)
{
if (p == '&')
{
break;
}
if (p == '0' | p == '1' | p == '2' | p == '3' | p == '4' | p == '5' | p == '6' | p == '7' | p == '8' | p == '9')
{
bPrice.Append(p);
}
}
var sPrice = int.Parse(bPrice.ToString());
// проверка существующей записи в базе
var infoAds = _context.InfoAds.FirstOrDefault(m => m.AdId == sAdId);
if (infoAds == null)
{
//получить номер телефона
var avitoParser = new AvitoParser();
var img = avitoParser.AvitoParserPhoneImage(link);
var recognizer = new PhoneRecognizer(System.Web.HttpContext.Current.Server.MapPath("~/Content/digits/"));
var sTel = recognizer.Recognize(img);
var tel = long.Parse(sTel.Replace("-", string.Empty));
// площадь
var area = row.SelectSingleNode("div[@class='params clearfix']/div[@class='param area']").InnerText;
var bArea = new StringBuilder();
foreach (var a in area)
{
if (a == ' ')
{
break;
}
if (a == '.')
{
bArea.Append(',');
}
else
{
bArea.Append(a);
}
}
var sArea = float.Parse(bArea.ToString());
// этаж и этажность
var floor = row.SelectSingleNode("div[@class='params clearfix']/div[@class='param floor']").InnerText;
// этажность
var bFloors = new StringBuilder();
var indexFloor = floor.IndexOf("/");
var floorToGetFloors = floor.Substring(indexFloor + 1);
foreach (var f in floorToGetFloors)
{
if (f == ' ')
{
break;
}
bFloors.Append(f);
}
var sFloors = int.Parse(bFloors.ToString());
// этаж
var bFloor = new StringBuilder();
foreach (var f in floor)
{
if (f == '/')
{
break;
}
if (f == '0' | f == '1' | f == '2' | f == '3' | f == '4' | f == '5' | f == '6' | f == '7' | f == '8' | f == '9')
{
bFloor.Append(f);
}
}
var sFloor = int.Parse(bFloor.ToString());
// адрес
var address = row.SelectSingleNode("div[@class='params clearfix']/div[@class='param address']/div[@class='fader']").InnerText;
var sAddress = address.Replace("\n ", "");
// ближайшее метро к адресу
var nearestMetro = row.SelectSingleNode("div[@class='params clearfix']" +
"/div[@class='param address']" +
"/div[@class='metro-info__wrap']" +
"/div[@class='metro-info__wrap-2']" +
"/div[@class='metro-name__wrap']" +
"/div[@class='metro-name__wrap-2']/span[@class='metro-name']")?.InnerText;
var sNearestMetro = "";
if (nearestMetro == null)
{
sNearestMetro = "Не указано";
}
else
{
sNearestMetro = nearestMetro.Replace("\n ", "").Trim();
}
// расстояние до ближайшего метро (в метрах)
var nearestMetroDistance = row.SelectSingleNode("div[@class='params clearfix']" +
"/div[@class='param address']" +
"/div[@class='metro-info__wrap']" +
"/div[@class='metro-info__wrap-2']" +
"/div[@class='metro-distance']")?.InnerText;
var sNearestMetroDistance = 0;
if (nearestMetroDistance == null)
{
sNearestMetroDistance = 0;
}
else
{
var bNearestMetroDistance = new StringBuilder();
foreach (var n in nearestMetroDistance)
{
if (n == 'к' || n == 'м' || n == ' ' || n == '+')
{
continue;
}
bNearestMetroDistance.Append(n);
}
if (bNearestMetroDistance.Length == 1 | bNearestMetroDistance.Length == 2)
{
bNearestMetroDistance = bNearestMetroDistance.Append('0');
bNearestMetroDistance = bNearestMetroDistance.Append('0');
bNearestMetroDistance = bNearestMetroDistance.Append('0');
}
var distanceClean = bNearestMetroDistance.ToString();
var nearestMetroDistanceInMetres = "";
var indexNearestMetroDistance = distanceClean.IndexOf(".");
if (indexNearestMetroDistance == -1)
{
nearestMetroDistanceInMetres = distanceClean;
}
else
{
nearestMetroDistanceInMetres = distanceClean.Insert(indexNearestMetroDistance + 2, "00");
}
var finalResult = new StringBuilder();
foreach (var r in nearestMetroDistanceInMetres)
{
if (r == '.')
{
continue;
}
finalResult.Append(r);
}
sNearestMetroDistance = int.Parse(finalResult.ToString());
}
var infoAd = new InfoAd()
{
AdId = sAdId,
Rooms = "1",
Price = sPrice,
Area = sArea,
Floor = sFloor,
Floors = sFloors,
Address = sAddress,
NearestMetro = sNearestMetro,
NearestMetroDistance = sNearestMetroDistance,
AdAdded = DateTime.Now,
Description = opisanie,
TelNumber = tel
};
_context.InfoAds.Add(infoAd);
_context.SaveChanges();
}
Thread.Sleep(8000);
}
Thread.Sleep(5000);
}
return(View());
}