/// <summary>
/// Crawl main page and get state links
/// </summary>
/// <returns></returns>
private IList <ListingInformation> CrawlMainPage()
{
IList <ListingInformation> stateList = new List <ListingInformation>();
driver.Navigate().GoToUrl(@url);
IWebElement selectElement = driver.FindElementById("states_home_search");
IList <IWebElement> optionElements = selectElement.FindElements(By.TagName("option"));
foreach (IWebElement element in optionElements)
{
if (String.IsNullOrEmpty(element.GetAttribute("value")))
{
continue;
}
ListingInformation state = new ListingInformation
{
Name = element.Text,
Url = GetAbsoluteUrl(url, element.GetAttribute("value"))
};
if (!state.IsEmpty())
{
stateList.Add(state);
}
}
return(stateList);
}
/// <summary>
/// Crawl city pages
/// </summary>
/// <param name="cityList"></param>
/// <returns></returns>
private IList <ListingInformation> CrawlCityPages(IList <ListingInformation> cityList)
{
IList <ListingInformation> shelters = new List <ListingInformation>();
foreach (var city in cityList)
{
try
{
driver.Navigate().GoToUrl(city.Url);
IList <IWebElement> h4Element = driver.FindElementsById("h4");
foreach (IWebElement element in h4Element)
{
IWebElement ele = element.FindElement(By.TagName("a"));
ListingInformation shelter = new ListingInformation
{
Name = ele.Text,
Url = ele.GetAttribute("href")
};
if (!shelter.IsEmpty())
{
shelters.Add(shelter);
}
}
}
catch (Exception ex)
{
log.LogError(ex, $"Exception while crawling shelter in state {city.Name} and url {city.Url}");
}
}
return(shelters);
}
/// <summary>
/// Crawl state pages
/// </summary>
/// <param name="stateList"></param>
/// <returns></returns>
private IList <ListingInformation> CrawlStatePages(IList <ListingInformation> stateList)
{
IList <ListingInformation> cities = new List <ListingInformation>();
foreach (var state in stateList)
{
try
{
driver.Navigate().GoToUrl(state.Url);
IWebElement tripleElement = driver.FindElementById("triple");
/*
* IList<IWebElement> cityElements = tripleElement.FindElements(By.XPath("//li/a"));
* foreach (IWebElement cityElement in cityElements)
* {
* ListingInformation city = new ListingInformation
* {
* Name = cityElement.Text,
* Url = cityElement.GetAttribute("href")
* };
* cities.Add(city);
*
* }*/
IList <IWebElement> lis = tripleElement.FindElements(By.TagName("li"));
foreach (IWebElement li in lis)
{
IWebElement ele = li.FindElement(By.TagName("a"));
ListingInformation city = new ListingInformation
{
Name = ele.Text,
Url = ele.GetAttribute("href")
};
if (!city.IsEmpty())
{
cities.Add(city);
}
}
}
catch (Exception ex)
{
log.LogError(ex, $"Exception while crawling shelter in state {state.Name} and url {state.Url}");
}
}
return(cities);
}
