/// <summary> /// Queue a new step on the crawler queue /// </summary> /// <param name="uri">url to crawl</param> /// <param name="depth">depth of the url</param> /// <param name="referrer">Step which the url was located</param> /// <param name="properties">Custom properties</param> public void AddStep(Uri uri, int depth, CrawlStep referrer, Dictionary <string, object> properties) { if (!m_Crawling) { throw new InvalidOperationException("Crawler must be running before adding steps"); } if (m_CrawlStopped) { return; } if ((uri.Scheme != Uri.UriSchemeHttps && uri.Scheme != Uri.UriSchemeHttp) || // Only accept http(s) schema (MaximumCrawlDepth.HasValue && MaximumCrawlDepth.Value > 0 && depth >= MaximumCrawlDepth.Value) || !IsAllowedUrl(uri, referrer)) { if (depth == 0) { StopCrawl(); } return; } if (!m_CrawlerHistory.Register(uri.GetUrlKeyString(UriSensitivity))) { return; } // Make new crawl step CrawlStep crawlStep = new CrawlStep(uri, depth) { IsExternalUrl = IsExternalUrl(uri), IsAllowed = true, }; m_CrawlerQueue.Push(new CrawlerQueueEntry { CrawlStep = crawlStep, Referrer = referrer, Properties = properties }); m_Logger.Verbose("Added {0} to queue referred from {1}", crawlStep.Uri, referrer.IsNull() ? string.Empty : referrer.Uri.ToString()); StartNew(); }
/// <summary> /// Queue a new step on the crawler queue /// </summary> /// <param name = "uri">url to crawl</param> /// <param name = "depth">depth of the url</param> /// <param name = "referrer">Step which the url was located</param> /// <param name = "properties">Custom properties</param> public async Task AddStepAsync(Uri uri, int depth, CrawlStep referrer, Dictionary <string, object> properties) { if (!this.m_Crawling) { throw new InvalidOperationException("Crawler must be running before adding steps"); } if (this.m_CrawlStopped) { return; } var allowedReferrer = await this.m_CrawlerRules.IsAllowedUrlAsync(uri, referrer); if ((uri.Scheme != "https" && uri.Scheme != "http") || // Only accept http(s) schema (this.MaximumCrawlDepth.HasValue && this.MaximumCrawlDepth.Value > 0 && depth >= this.MaximumCrawlDepth.Value) || !allowedReferrer || !this.m_CrawlerHistory.Register(uri.GetUrlKeyString(this.UriSensitivity))) { if (depth == 0) { StopCrawl(); } return; } // Make new crawl step var crawlStep = new CrawlStep(uri, depth) { IsExternalUrl = this.m_CrawlerRules.IsExternalUrl(uri), IsAllowed = true, }; this.m_CrawlerQueue.Push(new CrawlerQueueEntry { CrawlStep = crawlStep, Referrer = referrer, Properties = properties }); this.m_Logger.Verbose("Added {0} to queue referred from {1}", crawlStep.Uri, referrer.IsNull() ? string.Empty : referrer.Uri.ToString()); ProcessQueue(); }