public ScrapedCommunities ScrapCommunities(string state)
{
GoToCaringCom();
GoToState(state);
ScrapedCommunities scrapedCommunities = new ScrapedCommunities();
scrapedCommunities.CareType = this.CareType;
scrapedCommunities.State = state;
scrapedCommunities.CommunitiesByCity = new Dictionary <string, int>();
var cityList = webDriver.FindElement(By.Id("cities"));
var content = cityList.FindElements(By.ClassName("lrtr-list-item"));
foreach (var item in content)
{
string cityName = item.FindElement(By.ClassName("text-subtitle1")).Text;
int communitiesNumber = Convert.ToInt32(item.FindElement(By.ClassName("text-subtitle2")).Text);
scrapedCommunities.CommunitiesByCity.Add(cityName, communitiesNumber);
}
scrapedCommunities.CommunitiesByCounty = new Dictionary <string, int>();
var countyList = webDriver.FindElement(By.Id("counties"));
content = countyList.FindElements(By.ClassName("lrtr-list-item"));
foreach (var item in content)
{
string countyName = item.FindElement(By.ClassName("text-subtitle1")).Text;
int communitiesNumber = Convert.ToInt32(item.FindElement(By.ClassName("text-subtitle2")).Text);
scrapedCommunities.CommunitiesByCounty.Add(countyName, communitiesNumber);
}
return(scrapedCommunities);
}
static void Main(string[] args)
{
ScrapedCommunities communities = null;
List <ScrapedCompany> companies = null;
Scraper scraper = new Scraper();
Console.WriteLine("Write name of state");
string stateName = Console.ReadLine();
try
{
Console.WriteLine("Star of scraping at " + DateTime.Now.ToString());
communities = scraper.ScrapCommunities(stateName);
companies = scraper.ScrapCompanies(stateName);
Console.WriteLine("End of scraping at " + DateTime.Now.ToString());
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
string date = DateTime.UtcNow.ToLongDateString();
Guid guid = Guid.NewGuid();
string path = "Scraped_data_" + date + "_" + guid.ToString();
Directory.CreateDirectory(path);
var root = Directory.GetCurrentDirectory() + "\\" + path;
ExcelHelper excel = new ExcelHelper();
try
{
excel.LoadScrapedCommunities(root, communities);
excel.LoadScrapedCommpanies(root, companies);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
scraper.Close();
}
public void LoadScrapedCommunities(string path, ScrapedCommunities communities)
{
byte[] fileContent;
using (var package = new ExcelPackage())
{
var sheet = package.Workbook.Worksheets.Add(communities.CareType + "Of" + communities.State);
int row = 1;
sheet.Cells[row, 1].Value = communities.State;
row++;
sheet.Cells[row, 1].Value = "City/County";
sheet.Cells[row, 2].Value = "Communities";
row++;
var keys = communities.CommunitiesByCity.Keys;
foreach (var key in keys)
{
sheet.Cells[row, 1].Value = key;
sheet.Cells[row, 2].Value = communities.CommunitiesByCity[key];
row++;
}
keys = communities.CommunitiesByCounty.Keys;
foreach (var key in keys)
{
sheet.Cells[row, 1].Value = key;
sheet.Cells[row, 2].Value = communities.CommunitiesByCounty[key];
row++;
}
fileContent = package.GetAsByteArray();
}
if (fileContent == null || fileContent.Length == 0)
{
throw new Exception("fileContent is null or empty");
}
string fileName = "CommunitiesOf" + communities.State + ".xlsx";
using (FileStream stream = new FileStream($"{path}\\" + fileName, FileMode.Create))
{
stream.Write(fileContent, 0, fileContent.Length);
}
}
