public Crawler(ApplicationSettings applicationSettings, WebSettings webSettings, CrawlMode crawlMode, CookieContainer cookieContainer, CredentialCache credentialCache, List <CrawlerPeer> crawlerPeers, List <DatabasePeer> databasePeers, bool enableRenderers)
: this(applicationSettings, webSettings, crawlMode, crawlerPeers, databasePeers, enableRenderers)
{
CookieContainer = cookieContainer;
CredentialCache = credentialCache;
}
public Crawler(ApplicationSettings applicationSettings, WebSettings webSettings, CrawlMode crawlMode, CookieContainer cookieContainer, CredentialCache credentialCache, bool enableRenderers)
: this(applicationSettings, webSettings, crawlMode, enableRenderers)
{
CookieContainer = cookieContainer;
CredentialCache = credentialCache;
}
/// <summary>
/// Initializes a new instance of the <see cref = "Crawler" /> class.
/// </summary>
public Crawler(ApplicationSettings applicationSettings, WebSettings webSettings, CrawlMode crawlMode, List <CrawlerPeer> crawlerPeers, List <DatabasePeer> databasePeers, bool enableRenderers)
{
Guid = Guid.NewGuid();
try
{
_applicationSettings = applicationSettings;
_webSettings = webSettings;
_arachnodeDAO = (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true);
_applicationSettings = _arachnodeDAO.ApplicationSettings;
_consoleManager = new ConsoleManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_consoleManager.OutputString("arachnode.net " + Assembly.GetExecutingAssembly().GetName().Version, ConsoleColor.Green, ConsoleColor.Gray);
_actionManager = new ActionManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_ruleManager = new RuleManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_memoryManager = new MemoryManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_cacheManager = new CacheManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_cookieManager = new CookieManager();
_cacheManager = new CacheManager <TArachnodeDAO>(_applicationSettings, _webSettings);
CrawlerPeers = crawlerPeers;
DatabasePeers = databasePeers;
_crawlerPeerManager = new CrawlerPeerManager <TArachnodeDAO>(_applicationSettings, _webSettings, CrawlerPeers, (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true));
_databasePeerManager = new DatabasePeerManager <TArachnodeDAO>(_applicationSettings, _webSettings, DatabasePeers);
_cache = new Cache <TArachnodeDAO>(_applicationSettings, _webSettings, this, _actionManager, _cacheManager, _crawlerPeerManager, _memoryManager, _ruleManager);
_dataTypeManager = new DataTypeManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_discoveryManager = new DiscoveryManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache, _actionManager, _cacheManager, _memoryManager, _ruleManager);
_crawlRequestManager = new CrawlRequestManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache, _consoleManager, _discoveryManager);
_encodingManager = new EncodingManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_htmlManager = new HtmlManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager);
_politenessManager = new PolitenessManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache);
_proxyManager = new ProxyManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_reportingManager = new ReportingManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
//create required directories...
if (!Directory.Exists(_applicationSettings.ConsoleOutputLogsDirectory))
{
Directory.CreateDirectory(_applicationSettings.ConsoleOutputLogsDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedFilesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedFilesDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedImagesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedImagesDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedWebPagesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedWebPagesDirectory);
}
QueryProcessor = new QueryProcessor <TArachnodeDAO>();
_consoleManager.OutputString("Crawler: Initializing Configuration/Database Connection.", ConsoleColor.White, ConsoleColor.Gray);
LoadCrawlActions(_arachnodeDAO);
LoadCrawlRules(_arachnodeDAO);
AreRenderersEnabled = enableRenderers;
Engine = new Engine <TArachnodeDAO>(_applicationSettings, _webSettings, this, _cache, _actionManager, _cacheManager, _consoleManager, _cookieManager, _crawlRequestManager, _dataTypeManager, _discoveryManager, _encodingManager, _htmlManager, _memoryManager, _politenessManager, _proxyManager, _reportingManager, _ruleManager, enableRenderers, (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true));
CrawlMode = crawlMode;
/**/
if (CrawlerPeerManager != null && CrawlerPeerManager.CrawlerPeers != null && CrawlerPeerManager.CrawlerPeers.Count != 0)
{
ConsoleManager.OutputString("Crawler: Starting CrawlerPeerManager Server", ConsoleColor.White, ConsoleColor.Gray);
CrawlerPeerManager.StartServer(this, _arachnodeDAO);
_crawlerPeerManager.SendStatusMessageToCrawlerPeers(_arachnodeDAO);
}
/**/
if (Debugger.IsAttached)
{
_consoleManager.OutputString("Debugger: Attached - Expect Performance Degradation.", ConsoleColor.Yellow, ConsoleColor.Gray);
}
//update all core/components/managers with the updated ApplicationSettings...
#if DEMO
Engine.CrawlRequestCompleted += Engine_CrawlRequestCompleted;
_stopwatch.Start();
#endif
}
catch (InvalidConfigurationException invalidConfigurationException)
{
ProcessException(invalidConfigurationException);
throw new InvalidConfigurationException(invalidConfigurationException.ApplicationSettings, invalidConfigurationException.WebSettings, invalidConfigurationException.Message, InvalidConfigurationExceptionSeverity.Error);
}
catch (Exception exception)
{
ProcessException(exception);
throw new Exception(exception.Message, exception);
}
}
public Crawler(ApplicationSettings applicationSettings, WebSettings webSettings, CrawlMode crawlMode, bool enableRenderers)
: this(applicationSettings, webSettings, crawlMode, null, null, null, enableRenderers)
{
}
public void Constructor2(ApplicationSettings applicationSettings, WebSettings webSettings, CrawlMode crawlMode, bool enableRenderers)
{
Crawler = new Crawler <ArachnodeDAO>(applicationSettings, webSettings, crawlMode, enableRenderers);
}
