private static void OnlineScrap()
{
List <Dictionary <string, string> > nodesDataList = new List <Dictionary <string, string> >();//Scraped data
string[] links = FileManagment.LoadLinesFromFile(settings.GetInputFilePath());
raport.AllFiles = links.Count();
foreach (string link in links)
{
ProcessFile(LoadSiteFromUrl(link), ref nodesDataList);
}
FileManagment.SaveScrapedDataToFile(nodesDataList, settings);
}
private static void LocalScrap()
{
if (!Directory.Exists("sites"))
{
Directory.CreateDirectory("sites");
}
string[] filePaths = Directory.GetFiles(settings.GetInputPath());
if (filePaths.Count() < 1)
{
throw new System.Exception(string.Format("There are no sites to scrap in {0} folder at root directory", settings.inputPath));
}
List <Dictionary <string, string> > nodesDataList = new List <Dictionary <string, string> >();//Scraped data
raport.AllFiles = filePaths.Count();
foreach (string filePath in filePaths)
{
ProcessFile(LoadSiteFromPath(filePath), ref nodesDataList);
}
FileManagment.SaveScrapedDataToFile(nodesDataList, settings);
}
