/// <summary>
/// Queue a new step on the crawler queue
/// </summary>
/// <param name="uri">url to crawl</param>
/// <param name="depth">depth of the url</param>
/// <param name="referrer">Step which the url was located</param>
/// <param name="properties">Custom properties</param>
public void AddStep(Uri uri, int depth, CrawlStep referrer, Dictionary <string, object> properties)
{
if (!m_Crawling)
{
throw new InvalidOperationException("Crawler must be running before adding steps");
}
if (m_CrawlStopped)
{
return;
}
if ((uri.Scheme != Uri.UriSchemeHttps && uri.Scheme != Uri.UriSchemeHttp) || // Only accept http(s) schema
(MaximumCrawlDepth.HasValue && MaximumCrawlDepth.Value > 0 && depth >= MaximumCrawlDepth.Value) ||
!IsAllowedUrl(uri, referrer))
{
if (depth == 0)
{
StopCrawl();
}
return;
}
if (!m_CrawlerHistory.Register(uri.GetUrlKeyString(UriSensitivity)))
{
return;
}
// Make new crawl step
CrawlStep crawlStep = new CrawlStep(uri, depth)
{
IsExternalUrl = IsExternalUrl(uri),
IsAllowed = true,
};
m_CrawlerQueue.Push(new CrawlerQueueEntry
{
CrawlStep = crawlStep,
Referrer = referrer,
Properties = properties
});
m_Logger.Verbose("Added {0} to queue referred from {1}",
crawlStep.Uri, referrer.IsNull() ? string.Empty : referrer.Uri.ToString());
StartNew();
}
/// <summary>
/// Queue a new step on the crawler queue
/// </summary>
/// <param name = "uri">url to crawl</param>
/// <param name = "depth">depth of the url</param>
/// <param name = "referrer">Step which the url was located</param>
/// <param name = "properties">Custom properties</param>
public async Task AddStepAsync(Uri uri, int depth, CrawlStep referrer, Dictionary <string, object> properties)
{
if (!this.m_Crawling)
{
throw new InvalidOperationException("Crawler must be running before adding steps");
}
if (this.m_CrawlStopped)
{
return;
}
var allowedReferrer = await this.m_CrawlerRules.IsAllowedUrlAsync(uri, referrer);
if ((uri.Scheme != "https" && uri.Scheme != "http") || // Only accept http(s) schema
(this.MaximumCrawlDepth.HasValue && this.MaximumCrawlDepth.Value > 0 && depth >= this.MaximumCrawlDepth.Value) ||
!allowedReferrer ||
!this.m_CrawlerHistory.Register(uri.GetUrlKeyString(this.UriSensitivity)))
{
if (depth == 0)
{
StopCrawl();
}
return;
}
// Make new crawl step
var crawlStep = new CrawlStep(uri, depth)
{
IsExternalUrl = this.m_CrawlerRules.IsExternalUrl(uri),
IsAllowed = true,
};
this.m_CrawlerQueue.Push(new CrawlerQueueEntry
{
CrawlStep = crawlStep,
Referrer = referrer,
Properties = properties
});
this.m_Logger.Verbose("Added {0} to queue referred from {1}",
crawlStep.Uri, referrer.IsNull() ? string.Empty : referrer.Uri.ToString());
ProcessQueue();
}