/// <summary>
/// Calls a method to fetch urls based on query string parameters
/// state and city. Loops through the collection of urls and if one of the
/// urls searched for returns false, I will retry new city values within the same
/// state until it works.
/// </summary>
public void ScrapeWebsite()
{
var yelpBuilder = BuildYelpElements();
var scraper = new Scraper();
var locations = db.GetLocationsToSearchFor();
foreach (var state in locations)
{
Console.WriteLine("Starting to search location: {0}, {1}", state.Key, state.Value);
PagesExist = scraper.FetchPageUrls(string.Format(baseUrl, state.Key.Replace(" ", "+"), state.Value.Replace(" ", "+")), paginatorXpath, yelpBuilder);
if (!PagesExist)
{
Console.WriteLine("Could not find that location {0}, {1} from yelp, checking new location", state.Key, state.Value);
while (!PagesExist)
{
var newLocation = db.RetryNewLocation(state.Key);
Console.WriteLine("Trying {0}, {1} ", state.Key, newLocation);
PagesExist = scraper.FetchPageUrls(
string.Format(baseUrl, state.Key.Replace(" ", "+"), newLocation.Replace(" ", "+")),
paginatorXpath, yelpBuilder);
}
}
Console.WriteLine("Finished saving {0}, {1} in the database", state.Key, state.Value);
}
}