public IWebDriver CreateDriver(string extension_directory, AuthCredetials credentials, Proxy proxy = null) { ChromeOptions options = new ChromeOptions(); //options.AddExtension(extension_directory + "\\Buster.crx"); options.AddExtension(extension_directory + "\\VPN.crx"); options.AddExtension(extension_directory + "\\Add.crx"); if (proxy != null) { options.AddArguments($"proxy-server={proxy.HttpProxy}"); } IWebDriver driver = new ChromeDriver(options); Console.WriteLine("If you ready type anything"); Console.ReadKey(); driver.Url = "https://www.rusprofile.ru/"; Console.Clear(); if (proxy == null) { Console.WriteLine("Used proxy : No proxy"); } else { Console.WriteLine("Used proxy : " + proxy.HttpProxy); } driver.FindElement(By.Id("tologin")).Click(); Thread.Sleep(200); driver.FindElement(By.Id("mw-l_mail")).SendKeys(credentials.UserName); driver.FindElement(By.Id("mw-l_pass")).SendKeys(credentials.Password); driver.FindElement(By.Id("mw-l_entrance")).Click(); return(driver); }
public static void ExecuteFromSearch() { Parser parser = new Parser(CaptchaApi); Exporter exporter = new Exporter(ExportedDir); AuthCredetials credentials = new AuthCredetials { UserName = "******", Password = "******" }; IWebDriver driver = parser.CreateDriver(ExtDir, credentials); FileInfo searchFI = new FileInfo(ExportedDir + "\\Search.xlsx"); ExcelPackage excel = new ExcelPackage(searchFI); ExcelWorksheet sheet = excel.Workbook.Worksheets.FirstOrDefault(); int last_row = sheet.Cells.Where(c => c.Start.Column == 2 && c.Value != null).Last().End.Row; string search_page_url = "https://www.rusprofile.ru"; for (int i = 631; i <= last_row; i++) { error_link: try { forward_base: if (driver.Url != search_page_url) { driver.Url = "https://www.rusprofile.ru"; } if (driver.PageSource.Contains("«Я не робот».")) { parser.SolveCaptcha(driver, driver.Url); goto forward_base; } string searchString = sheet.Cells[i, 2].Value.ToString() + ", " + sheet.Cells[i, 6].Value.ToString(); var searchInput = driver.FindElement(By.ClassName("index-search-input")); if (searchInput != null) { searchInput.SendKeys(searchString); } Thread.Sleep(1000); var finded = driver.FindElements(By.XPath("//a[@class='search-drop__item']")); if (finded.Count == 0) { Console.WriteLine(searchString + "- нет результатов поиска"); continue; } string finded_URL = finded.FirstOrDefault().GetAttribute("href"); CompanyCard card = parser.ParseCard(driver, finded_URL).Result; sheet.Cells[i, 3].Value = card.Phones; sheet.Cells[i, 4].Value = card.Mails; sheet.Cells[i, 5].Value = card.Sites; sheet.Cells[i, 7].Value = card.PrimaryOccupation; sheet.Cells[i, 8].Value = card.OrganizationStatus; excel.Save(); } catch { goto error_link; } } Console.WriteLine("Done"); Console.ReadKey(); }
public static void ExecuteFromLinks(string[] args) { Proxy proxy = null; string LinksFileName = string.Empty; foreach (string arg in args) { if (arg.Contains("Links =")) { LinksFileName = arg.Replace("Links = ", string.Empty).Trim(); } if (arg.Contains("Proxy =")) { proxy = new Proxy() { HttpProxy = arg.Replace("Proxy = ", string.Empty).Trim() }; } } if (string.IsNullOrEmpty(LinksFileName)) { LinksFileName = "Links2.xlsx"; } SoundPlayer player = new SoundPlayer(Environment.CurrentDirectory + "\\Speech On.wav"); Dictionary <string, string> urls = new Dictionary <string, string>() { //{"411000", "https://www.rusprofile.ru/codes/411000/sankt-peterburg/" } , //{"412000", "https://www.rusprofile.ru/codes/412000/sankt-peterburg/" } , //{"421100", "https://www.rusprofile.ru/codes/421100/sankt-peterburg/" } , //{"421200", "https://www.rusprofile.ru/codes/421200/sankt-peterburg/" } , //{"421300", "https://www.rusprofile.ru/codes/421300/sankt-peterburg/" } , //{"422100", "https://www.rusprofile.ru/codes/422100/sankt-peterburg/" } , //{"422200", "https://www.rusprofile.ru/codes/422200/sankt-peterburg/" } , //{"422210", "https://www.rusprofile.ru/codes/422210/sankt-peterburg/" } , //{"422220", "https://www.rusprofile.ru/codes/422220/sankt-peterburg/" } , //{"422230", "https://www.rusprofile.ru/codes/422230/sankt-peterburg/" } , //{"429100", "https://www.rusprofile.ru/codes/429100/sankt-peterburg/" } , //{"429110", "https://www.rusprofile.ru/codes/429110/sankt-peterburg/" } , //{"429120", "https://www.rusprofile.ru/codes/429120/sankt-peterburg/" } , //{"429140", "https://www.rusprofile.ru/codes/429140/sankt-peterburg/" } , //{"429150", "https://www.rusprofile.ru/codes/429150/sankt-peterburg/" } , //{"429900", "https://www.rusprofile.ru/codes/429900/sankt-peterburg/" } , //{"431100", "https://www.rusprofile.ru/codes/431100/sankt-peterburg/" } , //{"431200", "https://www.rusprofile.ru/codes/431200/sankt-peterburg/" } , //{"431210", "https://www.rusprofile.ru/codes/431210/sankt-peterburg/" } , //{"431220", "https://www.rusprofile.ru/codes/431220/sankt-peterburg/" } , //{"431230", "https://www.rusprofile.ru/codes/431230/sankt-peterburg/" } , //{"431240", "https://www.rusprofile.ru/codes/431240/sankt-peterburg/" } , //{"431300", "https://www.rusprofile.ru/codes/431300/sankt-peterburg/" } , //{"432000", "https://www.rusprofile.ru/codes/432000/sankt-peterburg/" } , //{"432100", "https://www.rusprofile.ru/codes/432100/sankt-peterburg/" } , //{"432200", "https://www.rusprofile.ru/codes/432200/sankt-peterburg/" } , //{"432900", "https://www.rusprofile.ru/codes/432900/sankt-peterburg/" } , //{"433100", "https://www.rusprofile.ru/codes/433100/sankt-peterburg/" } , //{"433200", "https://www.rusprofile.ru/codes/433200/sankt-peterburg/" } , //{"433210", "https://www.rusprofile.ru/codes/433210/sankt-peterburg/" } , //{"433220", "https://www.rusprofile.ru/codes/433220/sankt-peterburg/" } , //{"433230", "https://www.rusprofile.ru/codes/433230/sankt-peterburg/" } , //{"433300", "https://www.rusprofile.ru/codes/433300/sankt-peterburg/" } , //{"433400", "https://www.rusprofile.ru/codes/433400/sankt-peterburg/" } , //{"433410", "https://www.rusprofile.ru/codes/433410/sankt-peterburg/" } , //{"433420", "https://www.rusprofile.ru/codes/433420/sankt-peterburg/" } , //{"433900", "https://www.rusprofile.ru/codes/433900/sankt-peterburg/" } , //{"439100", "https://www.rusprofile.ru/codes/439100/sankt-peterburg/" } , //{"439900", "https://www.rusprofile.ru/codes/439900/sankt-peterburg/" } , //{"439910", "https://www.rusprofile.ru/codes/439910/sankt-peterburg/" } , //{"439920", "https://www.rusprofile.ru/codes/439920/sankt-peterburg/" } , //{"439930", "https://www.rusprofile.ru/codes/439930/sankt-peterburg/" } , //{"439940", "https://www.rusprofile.ru/codes/439940/sankt-peterburg/" } , //{"439950", "https://www.rusprofile.ru/codes/439950/sankt-peterburg/" } , //{"439960", "https://www.rusprofile.ru/codes/439960/sankt-peterburg/" } , //{"439970", "https://www.rusprofile.ru/codes/439970/sankt-peterburg/" } , //{"439990", "https://www.rusprofile.ru/codes/439990/sankt-peterburg/" } , }; Parser parser = new Parser(CaptchaApi); Exporter exporter = new Exporter(ExportedDir); AuthCredetials credentials = new AuthCredetials { UserName = "******", Password = "******" }; IWebDriver driver = null; if (proxy == null) { driver = parser.CreateDriver(ExtDir, credentials); } else { driver = parser.CreateDriver(ExtDir, credentials, proxy); } DirectoryInfo profiles = new DirectoryInfo(ExportedDir + "\\Profiles\\"); FileInfo[] files = profiles.GetFiles(); FileInfo links = new FileInfo(ExportedDir + "\\" + LinksFileName); ExcelPackage pack = new ExcelPackage(links); for (int sheetNum = 0; sheetNum < pack.Workbook.Worksheets.Count; sheetNum++) { foreach (ExcelWorksheet sheet in pack.Workbook.Worksheets) { second_trying: ExcelWorksheet linkSheet = sheet; int last_row = linkSheet.Cells.Where(c => c.Start.Column == 1 && !c.Value.ToString().Equals("")).Last().End.Row; int last_profile_row = 0; Console.WriteLine("START PARSING " + linkSheet.Name + "At time: " + DateTime.Now.ToString()); ExcelPackage profilePack = null; try { profilePack = new ExcelPackage(files.FirstOrDefault(f => f.Name.Contains(linkSheet.Name))); ExcelWorksheet profileSheet = profilePack.Workbook.Worksheets.FirstOrDefault(); last_profile_row = profileSheet.Cells.Where(c => c.Start.Column == 2 && !c.Value.ToString().Equals("")).Last().End.Row + 1; IEnumerable <ExcelRangeBase> query = from row in linkSheet.Cells["C:XFD"] select row; ExcelRangeBase trying_find_last = query.ToList().FirstOrDefault(c => c.Value.ToString() == "wait"); if (trying_find_last == null) { Console.WriteLine("SKIP PARSING " + linkSheet.Name + "At time: " + DateTime.Now.ToString()); continue; } int last_not_parsed_row = trying_find_last.Start.Row; for (int row_index = last_not_parsed_row; row_index <= last_row; row_index++) { string url = linkSheet.Cells[row_index, 2].Value.ToString(); if (linkSheet.Cells[row_index, 3].Value.ToString() == "ok") { continue; } CompanyCard card = parser.ParseCard(driver, url).Result; #region map profileSheet.Cells[last_profile_row, 2].Value = card.ShortName; profileSheet.Cells[last_profile_row, 3].Value = card.FullName; profileSheet.Cells[last_profile_row, 4].Value = card.Phones; profileSheet.Cells[last_profile_row, 5].Value = card.Mails; profileSheet.Cells[last_profile_row, 6].Value = card.Sites; profileSheet.Cells[last_profile_row, 7].Value = card.INN; profileSheet.Cells[last_profile_row, 8].Value = card.PrimaryOccupation; profileSheet.Cells[last_profile_row, 9].Value = card.OrganizationStatus; #endregion last_profile_row++; profilePack.Save(); linkSheet.Cells[row_index, 3].Value = "ok"; } pack.Save(); Console.WriteLine("STOP PARSING " + linkSheet.Name + "At time: " + DateTime.Now.ToString()); } catch (Exception) { Console.WriteLine("Error on links page number = " + sheetNum); Console.WriteLine(" last profile row = " + last_profile_row); Console.WriteLine(" executed again"); save_mark: try { pack.Save(); } catch (ArgumentOutOfRangeException) { goto save_mark; } goto second_trying; } } } driver.Close(); player.Play(); Console.WriteLine("Done"); Console.ReadKey(); }