// constructor public CrawlerEngine(CrawlerSettings cSetting, CrawlerInput cInput, CrawlerOutput cOutput) { this.crawlerSettings_ = cSetting; this.crawlerInput_ = cInput; this.crawlerOutput_ = cOutput; queueUrls_ = new Queue(); this.threadsRunning_ = new Thread[cSetting.maxThreadCount_]; }
private void ClearCrawlerTree() { CrawlerOutput.Clear(); treeNodesDict.Clear(); RaisePropertyChanged(nameof(CrawlerOutput)); LoggerViewModel.Instance.LogMessage("Output cleared."); }
static void Main(string[] args) { string strMIMETypes = @"text/richtext[0,0];text/html[0,0];audio/x-aiff[0,0];"; strMIMETypes += @"audio/basic[0,0];audio/wav[0,0];image/gif[0,0];image/jpeg[0,0];"; strMIMETypes += @"image/pjpeg[0,0];image/tiff[0,0];image/x-png[0,0];image/x-xbitmap[0,0];"; strMIMETypes += @"image/bmp[0,0];image/x-jg[0,0];image/x-emf[0,0];image/x-wmf[0,0];"; strMIMETypes += @"video/avi[0,0];video/mpeg[0,0];application/postscript[0,0];application/base64[0,0];"; strMIMETypes += @"application/macbinhex40[0,0];application/pdf[0,0];application/x-compressed[0,0];"; strMIMETypes += @"application/x-zip-compressed[0,0];application/x-gzip-compressed[0,0];"; strMIMETypes += @"application/java[0,0];application/x-msdownload[0,0];"; CrawlerSettings settings = new CrawlerSettings(); settings.allowAllMIMETypes_ = false; settings.downloadfolder_ = "downloadfolder1"; settings.excludeFiles_ = new string[] { ".gif", ".jpg", ".css", ".zip", ".exe" }; settings.excludeHosts_ = new string[] { "" }; settings.excludeWords_ = new string[] { "" }; settings.keepAlive_ = false; settings.keepSameServer_ = false; settings.lastRequestCount_ = 0; settings.allowedMIMETypes_ = strMIMETypes; settings.requestTimeout_ = 10; settings.sleepConnectTime_ = 0; settings.sleepFetchTime_ = 0; settings.threadsCount_ = 1; settings.maxThreadCount_ = 20; settings.maxDepth_ = 1; settings.filePath_ = "CrawlerConsoleSettings.txt"; settings.lastModified_ = DateTime.Now; settings.version_ = 1; settings.dataTypeName_ = "CrawlerConsoleSettings"; settings.WriteToFile(); settings.ReadFromFile("CrawlerConsoleSettings.txt"); CrawlerInput input = new CrawlerInput(); input.domain_ = "baidu.com"; input.fullUrl_ = "www.baidu.com"; CrawlerOutput output = new CrawlerOutput(); CrawlerEngine engine = new CrawlerEngine(settings, input, output); engine.RunCrawling(); }
public CrawlerTreeViewItem AddCrawlerOutputTreeNode(CrawlerTreeViewItem parent, string item, WebCrawlerOutput attachment) { // Construct element CrawlerTreeViewItem element = new CrawlerTreeViewItem(); element.Header = item; element.AttachedData = attachment; // Attach to the parent if (parent != null) { parent.Items.Add(element); } else { CrawlerOutput.Add(element); } treeNodesDict.Add(attachment.SourceId, element); // Register for quick access RaisePropertyChanged(nameof(CrawlerOutput)); return(element); }