public void ScrapData()
{
var config = _configService.GetAppConfig();
_logger.Info("Loaded configuration");
var pages = config.ScrapPages;
using (var driver = new ChromeDriver())
{
foreach (var page in pages)
{
if (page.Ignore)
{
_logger.Info("Ignoring Page: " + page.URL);
continue;
}
var tableData = ScrapTableData(page, driver);
_logger.Info(tableData);
var dateCol = page.ColumnMappings.Find(x => x.IsDate == true);
if (dateCol == null)
{
throw new Exception("Cannot find date column in the column mappings");
}
var filtered = FilterNewRecords(tableData, page, dateCol);
_logger.Info("new records = " + filtered.Count);
filtered.Reverse();
if (filtered.Count > 0)
{
_logger.Info("FOUND new data on web, appending to file");
var headings = page.ColumnMappings.Select(x => x.HeadingInFile).ToList();
_csvWriter.AppendToFile(page.FilePath, filtered, headings);
_logger.Info("done");
}
else
{
_logger.Info("NOT found new data on web");
}
}
}
}