예제 #1
0
        private void ProcessUri(ParallelOptions options, Uri uri)
        {
            if (uri == null)
                return;

            WebPage webpage = new WebPage(uri);

            // Is it safe to visit the webpage?
            if (!EnsurePoliteVisit(webpage)) return;

            // Visit webpage
            webpage.LoadPage();

            // make sure to update or add time for visit to the dictionary
            UpdateTimestamp(webpage);

            // Add webpage anchors to the queue
            if (!webpage.IsLoaded)
                return;

            if (options.CancellationToken.IsCancellationRequested)
            {
                Thread.CurrentThread.Abort();
            }
            _store.WriteFile(webpage);
            _statistics.IncrementPagesCrawled();

            // Add extracted anchors to the queue
            _urlFrontier.AddUriRange(webpage.GetAnchors());
        }