private void ProcessUri(ParallelOptions options, Uri uri)
{
if (uri == null)
return;
WebPage webpage = new WebPage(uri);
// Is it safe to visit the webpage?
if (!EnsurePoliteVisit(webpage)) return;
// Visit webpage
webpage.LoadPage();
// make sure to update or add time for visit to the dictionary
UpdateTimestamp(webpage);
// Add webpage anchors to the queue
if (!webpage.IsLoaded)
return;
if (options.CancellationToken.IsCancellationRequested)
{
Thread.CurrentThread.Abort();
}
_store.WriteFile(webpage);
_statistics.IncrementPagesCrawled();
// Add extracted anchors to the queue
_urlFrontier.AddUriRange(webpage.GetAnchors());
}
