public string StopCrawling() { try { _storageManager.IssueCmd("CLEAR", _robotsTxtUrl); _storageManager.ClearAll(); return("Stopping and clearing crawl of " + _robotsTxtUrl); } catch (Exception ex) { return("Error: " + ex.ToString()); } }
private async Task RunAsync(CancellationToken cancellationToken) { // Set up queues, tables, data helper, status helper InitializeCrawlrComponents(); // Get the current cmd from the cmd table; // Re-execute cmd query periodically until current cmd exists while (_storageManager.GetCurrentCmd() != "START") { Thread.Sleep(5000); } // If start cmd given, initialize download of robots.txt // and populate the xmlQueue and _disallowed list if (_storageManager.GetCurrentCmd() == "START") { Startup(); } // Recurring work while (!cancellationToken.IsCancellationRequested) { Trace.TraceInformation("Working"); // Do work if current cmd is still "start" if (_storageManager.GetCurrentCmd() == "START") { // Process all XMLs (sitemaps) found while (_crawlrData.NumXmlsQueued > 0 && _storageManager.GetCurrentCmd() == "START") { CloudQueueMessage nextXmlMsg = _storageManager.XmlQueue.GetMessage(); string nextXml = nextXmlMsg.AsString; XmlCrawlr.CrawlXml(ref _crawlrData, ref _storageManager, nextXml); _storageManager.XmlQueue.DeleteMessage(nextXmlMsg); _crawlrData.NumXmlsQueued--; // Update worker role status _statusManager.UpdateCrawlrStatus( "Loading", _crawlrData, _storageManager ); _statusManager.UpdateQueueSize(_storageManager, _crawlrData.NumXmlsQueued, _crawlrData.NumUrlsQueued); Thread.Sleep(50); } // Process all URLs in queue while (_crawlrData.NumUrlsQueued > 0 && _storageManager.GetCurrentCmd() == "START") { CloudQueueMessage nextUrlMsg = _storageManager.UrlQueue.GetMessage(); string nextUrl = nextUrlMsg.AsString; UrlCrawlr.CrawlUrl(ref _crawlrData, ref _storageManager, nextUrl); _storageManager.UrlQueue.DeleteMessage(nextUrlMsg); _crawlrData.NumUrlsQueued--; // Update worker role status _statusManager.UpdateCrawlrStatus( "Crawling", _crawlrData, _storageManager ); _statusManager.UpdateQueueSize(_storageManager, _crawlrData.NumXmlsQueued, _crawlrData.NumUrlsQueued); Thread.Sleep(50); } } else if (_storageManager.GetCurrentCmd() == "CLEAR") { // If the "CLEAR" command is found, clear all queues and tables. _storageManager.ClearAll(); _statusManager.UpdateCrawlrStatus( "CLEAR", _crawlrData, _storageManager ); // Give Azure time to delete tables. Thread.Sleep(20000); try { // Idle while waiting for next command. while (_storageManager.GetCurrentCmd() == "CLEAR") { Thread.Sleep(10000); } } finally { // Reinitialize worker role. InitializeCrawlrComponents(); Startup(); } } else { // Idle worker role (for unimplemented 'pause' functionality). _statusManager.UpdateCrawlrStatus( "Idle", _crawlrData, _storageManager ); Thread.Sleep(5000); } } Thread.Sleep(1000); }