/// <summary> /// All crawlers is configed as text file and put in crawlerFolder /// CrawlerName|rule for fetch url /// Name of Url | Url | Output | Dept| Keywords split by ";" /// Example for Crawler config /// VMFCrawler|http://diendantoanhoc.net/|VMF.txt|topic; /// </summary> /// <param name="crawlerFolder"></param> public void LoadCrawlers(string crawlerFolder) { SmartFolder _folder = new SmartFolder(crawlerFolder, false); var _smartFiles = _folder.GeSmartFiles(false); if (_smartFiles != null && _smartFiles.Count() > 0) { foreach (var smartFile in _smartFiles) { var _crawler = LoadCrawler(smartFile.FullPath); Crawlers.Add(_crawler); } } }