public override int GetHashCode()
{
unchecked
{
int result = (CrawlStep != null ? CrawlStep.GetHashCode() : 0);
result = (result * 397) ^ (Properties != null ? Properties.GetHashCode() : 0);
result = (result * 397) ^ (Referrer != null ? Referrer.GetHashCode() : 0);
return(result);
}
}
/// <summary>
/// Executes OnDownloadException event
/// </summary>
private void OnDownloadException(Exception exception, CrawlStep crawlStep, CrawlStep referrer)
{
long downloadErrors = Interlocked.Increment(ref m_DownloadErrors);
if (MaximumHttpDownloadErrors.HasValue && MaximumHttpDownloadErrors.Value > downloadErrors)
{
m_Logger.Error("Number of maximum failed downloads exceeded({0}), cancelling crawl", MaximumHttpDownloadErrors.Value);
StopCrawl();
}
m_Logger.Error("Download exception while downloading {0}, error was {1}", crawlStep.Uri, exception);
DownloadException.ExecuteEvent(this, () => new DownloadExceptionEventArgs(crawlStep, referrer, exception));
}
/// <summary>
/// Returns true to continue crawl of this url, else false
/// </summary>
/// <returns>True if this step should be cancelled, else false</returns>
private bool OnBeforeDownload(CrawlStep crawlStep)
{
EventHandler <BeforeDownloadEventArgs> beforeDownloadTmp = BeforeDownload;
if (beforeDownloadTmp.IsNull())
{
return(crawlStep.IsAllowed);
}
BeforeDownloadEventArgs e =
new BeforeDownloadEventArgs(!crawlStep.IsAllowed, crawlStep);
beforeDownloadTmp(this, e);
return(!e.Cancel);
}
/// <summary>
/// Returns true to continue crawl of this url, else false
/// </summary>
/// <returns>True if this step should be cancelled, else false</returns>
private bool OnAfterDownload(CrawlStep crawlStep, PropertyBag response)
{
EventHandler <AfterDownloadEventArgs> afterDownloadTmp = AfterDownload;
if (afterDownloadTmp.IsNull())
{
return(crawlStep.IsAllowed);
}
AfterDownloadEventArgs e =
new AfterDownloadEventArgs(!crawlStep.IsAllowed, response);
afterDownloadTmp(this, e);
return(!e.Cancel);
}
/// <summary>
/// Queue a new step on the crawler queue
/// </summary>
/// <param name = "uri">url to crawl</param>
/// <param name = "depth">depth of the url</param>
/// <param name = "referrer">Step which the url was located</param>
/// <param name = "properties">Custom properties</param>
public void AddStep(Uri uri, int depth, CrawlStep referrer, Dictionary <string, object> properties)
{
if (!m_Crawling)
{
throw new InvalidOperationException("Crawler must be running before adding steps");
}
if (m_CrawlStopped)
{
return;
}
if ((uri.Scheme != "https" && uri.Scheme != "http") || // Only accept http(s) schema
(MaximumCrawlDepth.HasValue && MaximumCrawlDepth.Value > 0 && depth >= MaximumCrawlDepth.Value) ||
!m_CrawlerRules.IsAllowedUrl(uri, referrer) ||
!m_CrawlerHistory.Register(uri.GetUrlKeyString(UriSensitivity)))
{
if (depth == 0)
{
StopCrawl();
}
return;
}
// Make new crawl step
CrawlStep crawlStep = new CrawlStep(uri, depth)
{
IsExternalUrl = m_CrawlerRules.IsExternalUrl(uri),
IsAllowed = true,
};
m_CrawlerQueue.Push(new CrawlerQueueEntry
{
CrawlStep = crawlStep,
Referrer = referrer,
Properties = properties
});
m_Logger.Verbose("Added {0} to queue referred from {1}",
crawlStep.Uri, referrer.IsNull() ? string.Empty : referrer.Uri.ToString());
ProcessQueue();
}
public int CompareTo(CrawlerQueueEntry other)
{
return(CrawlStep.CompareTo(other.CrawlStep));
}
