private static void ScrapeAll()
{
var homesFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Homes_{DateTime.Now.ToString("yyyy_MM_dd_HH_mm_ss")}.csv");
var webScraperContent = File.ReadAllText("WebScraperConfig.xml");
OutputCsvHeader(homesFilePath, webScraperContent);
var singlePageSelectorGroups = GetSelectorRows();
var webScraper = new SinglePageScraper();
foreach (var singlePageSelectors in singlePageSelectorGroups)
{
var url = singlePageSelectors.First(s => s.Name == "Url").Value;
var selectorRow = GetSelectorRow(webScraperContent, singlePageSelectors);
webScraper.Scrape(url, selectorRow);
GetRoute(selectorRow);
Output(selectorRow, homesFilePath);
}
}
private static void ScrapeTest()
{
var webScraperContent = File.ReadAllText("WebScraperConfig.xml");
var webScraper = new SinglePageScraper();
var singlePageUrl = "https://www.redfin.com/CA/San-Jose/127-Herlong-Ave-95123/home/1287770";
var selectors = new List <Selector> {
new Selector {
Name = "Url", Value = singlePageUrl
}
};
var webScraperConfig = GetSelectorRow(webScraperContent, selectors);
webScraper.Scrape(singlePageUrl, webScraperConfig);
var rowValues = new List <string>();
GenerateCsvRow(webScraperConfig, rowValues);
Console.WriteLine(string.Join(",", rowValues));
}
