private static List <PropertyCrime> GetCrimes(PhantomJSDriver driver, long ID)
{
try
{
List <PropertyCrime> crimes = new List <PropertyCrime>();
CultureInfo ci = new System.Globalization.CultureInfo("en-US");
driver.ExecuteScript("window.scroll(0, document.querySelector('#nearbySubtitle').offsetTop + 500);");
System.Threading.Thread.Sleep(2000);
//Screenshot screenshot = ((ITakesScreenshot)driver).GetScreenshot().SaveAsFile("D:\\ScreenShot.png",System.Drawing.Imaging.ImageFormat.Png);
var crimesList = driver.FindElementsByCssSelector("div.crimeDataList.bbs.mbs > table > tbody > tr");
foreach (var crime in crimesList)
{
PropertyCrime temp = new PropertyCrime();
temp.Date = Convert.ToDateTime(crime.FindElement(By.CssSelector("td:nth-child(1) > div")).Text, ci);
temp.Type = crime.FindElement(By.CssSelector("td:nth-child(2) > div")).Text;
temp.Description = crime.FindElement(By.CssSelector("td:nth-child(3) > div")).Text;
temp.HomeId = ID;
crimes.Add(temp);
}
return(crimes);
}
catch (Exception ex)
{
Console.WriteLine("Не удалось получить список преступлений: {}, {}", ex.Message, ex.StackTrace);
return(null);
throw;
}
}
public void RegistrationRequiredCheck()
{
try
{
_driver.Manage().Timeouts().SetScriptTimeout(TimeSpan.FromSeconds(30));
_driver.Navigate().GoToUrl("http://localhost:5000/Login/Register");
var alltexts = _driver.FindElementsByCssSelector("[data-val='true']");
System.Collections.Generic.List <string> controls = new System.Collections.Generic.List <string>();
foreach (var item in alltexts)
{
controls.Add(item.GetAttribute("Id"));
}
Assert.IsTrue(controls.Contains("Fname"));
Assert.IsTrue(controls.Contains("Lname"));
Assert.IsTrue(controls.Contains("Phone"));
Assert.IsTrue(controls.Contains("EmailID"));
Assert.IsTrue(controls.Contains("Password"));
Assert.IsTrue(controls.Contains("ConfirmPassword"));
Assert.IsTrue(controls.Contains("Address"));
Console.Out.WriteLine("Required Field check is succeed");
}
catch (Exception ex)
{
Console.Out.WriteLine(ex.Message);
}
}
private static List <PropertySchool> GetSchools(PhantomJSDriver driver, long ID)
{
try
{
string Type = String.Empty; //у нас три типа школ, по умолчанию открывается начальная
List <PropertySchool> schools = new List <PropertySchool>();
driver.ExecuteScript("document.querySelector('#schoolsCard .clickable').click()"); //открываем список школ, чтобы он прогрузился
System.Threading.Thread.Sleep(1000);
for (int i = 0; i < 3; i++)
{
driver.ExecuteScript(String.Format("document.querySelectorAll('div.mbl.schoolListContainer > div > ul > li > div')[{0}].click()", i)); //нажимаем на кнопку с нуным списком школ
switch (i)
{
case 0:
Type = Constants.SchoolType.Elementary;
break;
case 1:
Type = Constants.SchoolType.Middle;
break;
case 2:
Type = Constants.SchoolType.HighSchool;
break;
}
System.Threading.Thread.Sleep(1000);
var schoolsList = driver.FindElementsByCssSelector(".line.pls.bbs.pvm"); //выбираем список
foreach (var school in schoolsList)
{
PropertySchool temp = new PropertySchool();
temp.homeID = ID;
temp.Type = Type;
temp.SchoolName = school.FindElement(By.CssSelector("a")).Text;
temp.Grades = school.FindElement(By.CssSelector(".line > div")).Text.Replace("Grades: ", String.Empty);
temp.Distance = Convert.ToDouble(school.FindElement(By.CssSelector(".line > div:nth-child(2)")).Text.Replace(" mi", String.Empty));
temp.Address = school.FindElement(By.CssSelector(".typeLowlight")).Text;
temp.Rank = school.FindElement(By.CssSelector(".txtC")).Text;
}
}
return(schools);
}
catch (Exception ex)
{
Console.WriteLine("Не удалось получить список школ: {}, {}", ex.Message, ex.StackTrace);
return(null);
throw;
}
}
void TestDriver()
{
string link = "https://www.trulia.com/property-sitemap/AL/";
var driver = new PhantomJSDriver();
driver.Manage().Window.Size = new Size(1920, 1080);
driver.Navigate().GoToUrl(link);
var links = driver.FindElementsByCssSelector("dl a");
foreach (var a in links)
{
//Console.WriteLine("City: {0}, Link: {1}, Count: {2}", a.Text.Replace(a.Text.Split(' ')[0],String.Empty).Replace(" Homes",String.Empty), a.GetAttribute(Constants.WebAttrsNames.href), Int32.Parse(a.Text.Split(' ')[0].Replace(",",String.Empty)));
}
driver.Quit();
}
/// <summary>
/// Получает данные о штатах со страницы https://www.trulia.com/property-sitemap/ и записывает в БД
/// </summary>
public static void GetStateListToDb()
{
var driver = new PhantomJSDriver(ProxySolver.GetServiceForDriver());
driver.Manage().Window.Size = new System.Drawing.Size(1920, 1080);
string statesLink = "https://www.trulia.com/property-sitemap/";
driver.Navigate().GoToUrl(statesLink);
var linksElemsList = driver.FindElementsByCssSelector(".content .cols12 li a");
foreach (var linkElem in linksElemsList)
{
Console.WriteLine(linkElem.GetAttribute(Constants.WebAttrsNames.href) + ", " + linkElem.Text.Remove(linkElem.Text.Length - 7, 6));
SqlCommand insertState = DataProvider.Instance.CreateSQLCommandForSP(Resources.SP_AddNewState);
insertState.Parameters.AddWithValue("@StateName", linkElem.Text.Remove(linkElem.Text.Length - 7, 6));
insertState.Parameters.AddWithValue("@Link", linkElem.GetAttribute(Constants.WebAttrsNames.href));
DataProvider.Instance.ExecureSP(insertState);
}
driver.Quit();
}
public static PlayerHeroesPreferenceModel ScrapeHeroPreferencesPhantomJs(string playerId, int days = 60)
{
if (string.IsNullOrWhiteSpace(playerId))
{
return(null);
}
var retries = 5;
while (--retries > 0)
{
try
{
using (var driver = new PhantomJSDriver())
{
var url = BuildPlayerStatsUrl(playerId);
driver.Navigate().GoToUrl(url);
// Open the 'Days to show' selector
driver.FindElementById("ctl00_MainContent_DropDownProfileTimeSpan").Click();
var daySelectParent = new WebDriverWait(driver, TimeSpan.FromSeconds(2)).Until(e =>
{
var elem = e.FindElement(By.Id("ctl00_MainContent_DropDownProfileTimeSpan_DropDown"));
return(elem.GetAttribute("style").Contains("display: block;") ? elem : null);
});
// Select the 'Last X Days' option to show only heroes played in the last so many days
var daySelectText = $"Last {days} Days";
daySelectParent.FindElements(By.CssSelector("li")).First(e => e.Text == daySelectText).Click();
// Do the same for the range (i.e. only show heroes that have been played more than x times)
driver.FindElementById("ctl00_MainContent_DropDownGamesPlayedRequired").Click();
var rangeSelectParent = new WebDriverWait(driver, TimeSpan.FromSeconds(2)).Until(e =>
{
var elem = e.FindElement(By.Id("ctl00_MainContent_DropDownGamesPlayedRequired_DropDown"));
return(elem.GetAttribute("style").Contains("display: block;") ? elem : null);
});
var rangeSelectText = "5+ Games Played";
new WebDriverWait(driver, TimeSpan.FromSeconds(2)).Until(e =>
{
var elem = rangeSelectParent.FindElements(By.CssSelector("li")).FirstOrDefault(li => li.Text == rangeSelectText);
return(elem ?? elem);
}).Click();
// Find the hero stats in the table and parse the stats out
var rows = driver.FindElementsByCssSelector("#RadGridReplayCharacterScoreResultsAverage table tbody tr");
var stats = new List <HeroStatsModel>();
foreach (var row in rows)
{
var dataElems = row.FindElements(By.TagName("td"));
var heroNameElem = dataElems[0];
var gamesPlayedElem = dataElems[2];
var winPercentElem = dataElems[3];
var heroName = heroNameElem.FindElement(By.TagName("a")).GetAttribute("title");
var gamesPlayed = gamesPlayedElem.Text;
var winPercent = winPercentElem.Text.Replace("%", "");
stats.Add(new HeroStatsModel {
Name = WebUtility.HtmlDecode(heroName), GamesPlayed = Convert.ToInt32(gamesPlayed), WinPercent = Convert.ToDouble(winPercent)
});
}
return(new PlayerHeroesPreferenceModel {
HeroStats = stats.OrderByDescending(e => e.GamesPlayed).ThenByDescending(e => e.WinPercent).ToList(), PlayerId = playerId
});
}
}
catch (StaleElementReferenceException)
{
continue;
}
catch
{
return(new PlayerHeroesPreferenceModel {
HeroStats = new List <HeroStatsModel>(), PlayerId = playerId
});
}
}
return(new PlayerHeroesPreferenceModel {
HeroStats = new List <HeroStatsModel>(), PlayerId = playerId
});
}
private bool savePageInfo(string path, CourtCase case1)
{
DocumentWriter doc = null;
try
{
doc = new DocumentWriter(path + "/" + case1.caseNum);
doc.addHeading(case1.caseNum);
#region Div Case Information
string caseInfoSel = "#divCaseInformation_body";
var caseInfoDiv = FindElementIfExists(By.CssSelector(caseInfoSel));
if (caseInfoDiv != null)
{
var caseInfochildDivs = driver.FindElementsByCssSelector(caseInfoSel + " > div");
doc.addHeading("Case Information");
foreach (var cicd in caseInfochildDivs)
{
var attr = cicd.GetAttribute("class");
doc.addText(cicd.Text);
}
}
#endregion
#region Case Parties
string casePartiessel = "#partyInformationDiv";
var casepartiesDiv = FindElementIfExists(By.CssSelector(caseInfoSel));
if (casepartiesDiv != null)
{
var caseInfohead = FindElementIfExists(By.CssSelector("#divPartyInformation_header"));
if (caseInfohead != null)
{
doc.addHeading(caseInfohead.Text);
}
var casePartiesBody = FindElementIfExists(By.CssSelector("#divPartyInformation_body"));
if (casePartiesBody != null)
{
var casepartieschildDivs = driver.FindElementsByCssSelector("#divPartyInformation_body > div");
foreach (var cicd in casepartieschildDivs)
{
doc.addText(cicd.Text);
}
}
}
#endregion
#region Disposition Events
string dispositionEventsSel = "#dispositionInformationDiv";
var dispositionEventDiv = FindElementIfExists(By.CssSelector(dispositionEventsSel));
if (dispositionEventDiv != null)
{
doc.addHeading("Disposotion Events");
var dispositionEventBody = FindElementIfExists(By.CssSelector("#dispositionInformationDiv > div:nth-child(2)"));
if (dispositionEventBody != null)
{
var childDivs = dispositionEventBody.FindElements(By.CssSelector("div > div > div"));
foreach (var div in childDivs)
{
doc.addText(div.Text);
}
}
}
#endregion
#region Events and Hearings
string eventInfoSel = "#eventsInformationDiv";
var eventInfoDiv = FindElementIfExists(By.CssSelector(eventInfoSel));
if (eventInfoSel != null)
{
doc.addHeading("Events and Hearings");
var eventInfoBody = FindElementIfExists(By.CssSelector(".list-group"));
if (eventInfoBody != null)
{
var childLi = eventInfoBody.FindElements(By.TagName("li"));
foreach (var ch in childLi)
{
doc.addText(ch.Text);
}
}
}
#endregion
#region Financial
string financialSel = "#financialSlider";
var financialDiv = FindElementIfExists(By.CssSelector(financialSel));
if (financialDiv != null)
{
doc.addHeading("Financial");
var financialBody = FindElementIfExists(By.CssSelector("#financialSlider > div:nth-child(1)"));
var childs = driver.FindElementsByCssSelector("#financialSlider > div:nth-child(1) > div");
foreach (var ch in childs)
{
var attr = ch.GetAttribute("class");
doc.addText(ch.Text);
}
}
#endregion
doc.Save();
return(true);
}
catch (Exception ex)
{
if (doc != null)
{
doc.Save();
}
return(false);
}
}
public static List <string> ParseHomes(Street street)
//public static List<string> ParseHomes(string Link)
{
PhantomJSDriver driver = CreateDriver();
try
{
if (driver.SessionId == null)
{
driver = CreateDriver();
}
//переход по стартовой ссылке города
while (true)
{
try
{
driver.Navigate().GoToUrl(street.Link);
break;
}
catch (Exception ex)
{
logger.Trace(ex, "Ошибка получения страницы, время ожидания истекло. {0},{1}", ex.Message, ex.StackTrace);
driver.Quit();
driver = CreateDriver();
}
}
//забор элементов со страниц
List <string> homeHrefs = new List <string>(); //создаем список для ссылок на дома(property)
//сначала в цикле while соберем ссылки на дома в список
while (true)
{
var homelinks = driver.FindElementsByCssSelector(".mvm a"); //выбираем элементы, в которых есть нужные нам ссылки
if (homelinks.Count == 0) //т.е. улица пуста
{
return(homeHrefs);
}
foreach (var a in homelinks)
{
homeHrefs.Add(a.GetAttribute(Constants.WebAttrsNames.href));
}
var nextButtons = driver.FindElementsByCssSelector("*[rel=next]");
if (nextButtons.Count > 0)
{
while (true)
{
try
{
nextButtons[0].Click();
break;
}
catch (OpenQA.Selenium.WebDriverTimeoutException ex)
{
//throw new Exception();
logger.Trace(ex, "Ошибка получения страницы, время ожидания истекло.");
logger.Error(ex, "Ошибка получения страницы, время ожидания истекло.");
}
catch (OpenQA.Selenium.StaleElementReferenceException ex)
{
//throw new Exception();
logger.Trace(ex, "Отсутствует данный Элемент на странице(кнопка Next)");
logger.Error("Отсутствует данный Элемент на странице(кнопка Next)");
break;
}
catch (Exception ex)
{
logger.Error(ex, "Неизвестная ошибка: {1}, {0}", ex.StackTrace, ex.Message);
logger.Trace(ex, "Неизвестная ошибка: {1}, {0}", ex.StackTrace, ex.Message);
throw;
}
}
}
else
{
break;
}
}
foreach (string link in homeHrefs) //а теперь сам процесс обработки домов по ссылкам
{
Console.WriteLine(link);
}
return(homeHrefs); //все прошло успешно
}
catch (OpenQA.Selenium.WebDriverException ex)
{
logger.Error(ex, "Возникло исключение web-драйвера: {1}, {0}", ex.StackTrace, ex.Message);
return(null);
}
catch (Exception ex)
{
logger.Error(ex, "Возникло неизвестное исключение: {1}, {0}", ex.StackTrace, ex.Message);
return(null);
}
finally
{
driver.Quit();
}
}
public static bool GetSteetsToDb(City city)
{
PhantomJSDriver driver = CreateDriver();
try
{
if (driver.SessionId == null)
{
driver = CreateDriver();
}
//переход по стартовой ссылке города
while (true)
{
try
{
driver.Navigate().GoToUrl(city.Link);
break;
}
catch (Exception ex)
{
logger.Trace(ex, "Ошибка получения страницы, время ожидания истекло. {0},{1}", ex.Message, ex.StackTrace);
driver.Quit();
driver = CreateDriver();
}
}
//забор элементов со страниц
while (true)
{
var streetlinks = driver.FindElementsByCssSelector(".mvm a");
foreach (var a in streetlinks)
{
string Text = a.Text;
Text = Text.Replace("homes in ", String.Empty);
string href = a.GetAttribute(Constants.WebAttrsNames.href);
Console.WriteLine("State: {2}, City: {3} Street: {0}, Link: {1}", Text, href, city.StateName, city.CityName);
SqlCommand insertStreet = DataProvider.Instance.CreateSQLCommandForSP(Resources.SP_AddNewStreet);
insertStreet.Parameters.AddWithValue("@StateName", city.StateName);
insertStreet.Parameters.AddWithValue("@Link", href);
insertStreet.Parameters.AddWithValue("@CityName", city.CityName);
insertStreet.Parameters.AddWithValue("@StreetName", Text);
insertStreet.Parameters.AddWithValue("@ZIP", Text.Split(' ')[Text.Split(' ').Length - 1]);
DataProvider.Instance.ExecureSP(insertStreet);
}
var nextButtons = driver.FindElementsByCssSelector("*[rel=next]");
if (nextButtons.Count > 0)
{
while (true)
{
try
{
nextButtons[0].Click();
break;
}
catch (OpenQA.Selenium.WebDriverTimeoutException ex)
{
//throw new Exception();
logger.Trace(ex, "Ошибка получения страницы, время ожидания истекло.");
logger.Error(ex, "Ошибка получения страницы, время ожидания истекло.");
}
catch (OpenQA.Selenium.StaleElementReferenceException ex)
{
//throw new Exception();
logger.Trace(ex, "Отсутствует данный Элемент на странице(кнопка Next)");
logger.Error("Отсутствует данный Элемент на странице(кнопка Next)");
break;
}
catch (Exception ex)
{
logger.Error(ex, "Неизвестная ошибка: {1}, {0}", ex.StackTrace, ex.Message);
logger.Trace(ex, "Неизвестная ошибка: {1}, {0}", ex.StackTrace, ex.Message);
throw;
}
}
}
else
{
break;
}
}
return(true); //все прошло успешно
}
catch (OpenQA.Selenium.WebDriverException ex)
{
logger.Error(ex, "Возникло исключение web-драйвера: {1}, {0}", ex.StackTrace, ex.Message);
return(false);
}
catch (Exception ex)
{
logger.Error(ex, "Возникло неизвестное исключение: {1}, {0}", ex.StackTrace, ex.Message);
return(false);
}
finally
{
driver.Quit();
}
}
public static void GetCityListToDb(State state)
{
while (true)
{
PhantomJSDriver driver;
while (true)
{
try
{
driver = new PhantomJSDriver(ProxySolver.GetServiceForDriver());
break;
}
catch (Exception ex)
{
logger.Error(ex, "Ошибка создания драйвера: {0},{1}", ex.Message, ex.StackTrace);
}
}
try
{
driver.Manage().Window.Size = new System.Drawing.Size(1920, 1080);
driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(20);
while (true)
{
try
{
driver.Navigate().GoToUrl(state.Link);
break;
}
catch (Exception ex)
{
logger.Trace(ex, "Ошибка получения страницы, время ожидания истекло. {0},{1}", ex.Message, ex.StackTrace);
driver.Quit();
driver = new PhantomJSDriver(ProxySolver.GetServiceForDriver());
driver.Manage().Window.Size = new System.Drawing.Size(1920, 1080);
driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(20);
}
}
while (true)
{
var citylinks = driver.FindElementsByCssSelector("dl a");
foreach (var a in citylinks)
{
string Text = a.Text;
Text = Text.Remove(Text.Length - 7, 6);
string href = a.GetAttribute(Constants.WebAttrsNames.href);
//Console.WriteLine("City: {0}, Link: {1}, Count: {2}", Text.Replace(Text.Split(' ')[0], String.Empty), href, Int32.Parse(Text.Split(' ')[0].Replace(",", String.Empty)));
SqlCommand insertCity = DataProvider.Instance.CreateSQLCommandForSP(Resources.SP_AddNewCity);
insertCity.Parameters.AddWithValue("@StateName", state.StateName);
insertCity.Parameters.AddWithValue("@Link", href);
insertCity.Parameters.AddWithValue("@CityName", Text.Replace(Text.Split(' ')[0], String.Empty));
insertCity.Parameters.AddWithValue("@Count", Int32.Parse(Text.Split(' ')[0].Replace(",", String.Empty)));
DataProvider.Instance.ExecureSP(insertCity);
}
var nextButtons = driver.FindElementsByCssSelector("*[rel=next]");
if (nextButtons.Count > 0)
{
while (true)
{
try
{
nextButtons[0].Click();
break;
}
catch (OpenQA.Selenium.WebDriverTimeoutException ex)
{
//throw new Exception();
logger.Trace(ex, "Ошибка получения страницы, время ожидания истекло.");
logger.Error(ex, "Ошибка получения страницы, время ожидания истекло.");
}
catch (OpenQA.Selenium.StaleElementReferenceException ex)
{
//throw new Exception();
logger.Trace(ex, "Отсутствует данный Элемент на странице(кнопка Next)");
logger.Error("Отсутствует данный Элемент на странице(кнопка Next)");
break;
}
catch (Exception ex)
{
logger.Error(ex, "Неизвестная ошибка: {1}, {0}", ex.StackTrace, ex.Message);
logger.Trace(ex, "Неизвестная ошибка: {1}, {0}", ex.StackTrace, ex.Message);
throw;
}
}
}
else
{
break;
}
}
driver.Quit();
break;
}
catch (OpenQA.Selenium.WebDriverException ex)
{
driver.Quit();
logger.Error(ex, "Возникло исключение web-драйвера: {1}, {0}", ex.StackTrace, ex.Message);
}
}
//foreach (var linkElem in linksElemsList)
//{
// Console.WriteLine(linkElem.GetAttribute(Constants.WebAttrsNames.href) + ", " + linkElem.Text.Replace(" homes", String.Empty));
// SqlCommand insertState = DataProvider.Instance.CreateSQLCommandForSP(Resources.SP_AddNewState);
// insertState.Parameters.AddWithValue("@StateName", linkElem.Text.Replace(" homes", String.Empty));
// insertState.Parameters.AddWithValue("@Link", linkElem.GetAttribute(Constants.WebAttrsNames.href));
// DataProvider.Instance.ExecureSP(insertState);
//}
}
