public static async Task CrawlFoodPantrySateSiteAsync(ChromeDriver driver, FoodPantryState state) { IList <FoodPantryCity> cityList = new List <FoodPantryCity>(); driver.Navigate().GoToUrl(state.Url); IWebElement tableElement = driver.FindElement(By.TagName("table")); IList <IWebElement> tableRow = tableElement.FindElements(By.TagName("tr")); IList <IWebElement> rowTD; foreach (IWebElement row in tableRow) { rowTD = row.FindElements(By.XPath("//td/a")); foreach (IWebElement td in rowTD) { FoodPantryCity city = new FoodPantryCity(); city.Name = td.Text; city.Url = td.GetAttribute("href"); city.State = state; cityList.Add(city); } } state.Cities = cityList; driver.Navigate().Back(); }
public static FoodPantry ProcessFoodPantry(string html, FoodPantryCity city) { FoodPantry pantry = new FoodPantry(); string processhtml = html.Replace(System.Environment.NewLine, @""); string[] parts = processhtml.Split(@","); foreach (string part in parts) { string[] pair = part.Split(@":"); if (pair[0].Contains(city.Name)) { pantry.Name = pair[1]; } } return(pantry); }
public static async Task CrawlFoodPantryCityWebSite( ChromeDriver driver, FoodPantryCity city) { driver.Navigate().GoToUrl(city.Url); IList <IWebElement> tableElements = driver.FindElements(By.XPath("//script[@type='application/ld+json']")); foreach (IWebElement block in tableElements) { string html = block.GetAttribute("innerHTML"); if (html.Contains(@"postalCode") && html.Contains(city.Name)) { city.Pantries.Add(ProcessFoodPantry(html, city)); } } }