/// <summary>
/// All crawlers is configed as text file and put in crawlerFolder
/// CrawlerName|rule for fetch url
/// Name of Url | Url | Output | Dept| Keywords split by ";"
/// Example for Crawler config
/// VMFCrawler|http://diendantoanhoc.net/|VMF.txt|topic;
/// </summary>
/// <param name="crawlerFolder"></param>
public void LoadCrawlers(string crawlerFolder)
{
SmartFolder _folder = new SmartFolder(crawlerFolder, false);
var _smartFiles = _folder.GeSmartFiles(false);
if (_smartFiles != null && _smartFiles.Count() > 0)
{
foreach (var smartFile in _smartFiles)
{
var _crawler = LoadCrawler(smartFile.FullPath);
Crawlers.Add(_crawler);
}
}
}