Example #1
0
        private void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;
            var         mssg        = string.Format("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);

            _logger.Debug(mssg);
        }
        public void Constructor_ValidReason_SetsPublicProperty()
        {
            string reason = "aaa";
            PageCrawlDisallowedArgs args = new PageCrawlDisallowedArgs(_context, _page, reason);

            Assert.AreSame(reason, args.DisallowedReason);
        }
Example #3
0
        private void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;
            var         result      = string.Format("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);

            log.Info(result);
        }
Example #4
0
 public static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
 {
     try
     {
         PageToCrawl pageToCrawl = e.PageToCrawl;
         Console.WriteLine("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
     }catch (Exception)
     {
     }
 }
        void crawler_PageCrawlDisallowedAsync(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;
            string      msg;

            m_linksSkipped++;
            updateCrawlingProgress(-1, -1, m_linksSkipped);

            msg = "Did not crawl the links on page " + pageToCrawl.Uri.AbsoluteUri.ToString() + " due to " + e.DisallowedReason.ToString();
            log.Warn(msg);
        }
Example #6
0
        private static void WebCrawler_PageCrawlDisallowedAsync(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;

            Console.WriteLine("Abot-Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
        }
Example #7
0
        void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;

            OnMessageReceived($"Did not crawl page {pageToCrawl.Uri.AbsoluteUri} due to {e.DisallowedReason}");
        }
 private void PageCrawlDisallowedEvent(object sender, PageCrawlDisallowedArgs e)
 {
 }
Example #9
0
        private void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;

            _logger.InfoFormat("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
        }
 private void Crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
 {
     Console.WriteLine($"Page Crawl Disallowed, Page: {e.PageToCrawl.Uri}, Reason: {e.DisallowedReason}");
 }
Example #11
0
 static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
 {
     Log("crawler_PageCrawlDisallowed", e.PageToCrawl.Uri.AbsoluteUri);
     var e1 = e;
     //Process data
 }
Example #12
0
 private static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
 {
 }
Example #13
0
        void PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;

            Log.Logger.Error($"Did not crawl page {pageToCrawl.Uri.AbsoluteUri} due to {e.DisallowedReason}");
        }
 /// <summary>
 /// Method to handle when page crawling was disallowed
 /// </summary>
 /// <param name="sender"></param>
 /// <param name="args"></param>
 private void crawlerProcessPageCrawlDisallowedHandler(object sender, PageCrawlDisallowedArgs args)
 {
     Debug.WriteLine("Did not crawl page {0}", args);
     this.PageCrawlDisallowed?.Invoke(this, args);
 }
Example #15
0
        static void Crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;

            _log.Info($"Did not crawl page {pageToCrawl.Uri.AbsoluteUri} due to {e.DisallowedReason}");
        }
Example #16
0
        //抓取页面失败
        public static void Disallowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;

            Outputer.Output(string.Format("由于产生错误:{1} 无法抓取页面{0}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason));
        }
        /// <summary>
        /// 不爬取这个页面的原因
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl pageToCrawl = e.PageToCrawl;

            log.Info("不爬取此页面 " + pageToCrawl.Uri.AbsoluteUri + " 其原因为 " + e.DisallowedReason);
        }
Example #18
0
        private void crawler_CrawlerDisalowed(object sender, PageCrawlDisallowedArgs e)
        {
            PageToCrawl page = e.PageToCrawl;

            Console.WriteLine("Disallowed: {0}", page.Uri.ToString());
        }
Example #19
0
 static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
 {
     //Process data
 }
Example #20
0
 void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
 {
 }
Example #21
0
 void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
 {
     //PageToCrawl pageToCrawl = e.PageToCrawl;
     //Console.WriteLine("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
 }
Example #22
0
 void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
 {
     Log($"Did not crawl page {e.PageToCrawl.Uri.AbsoluteUri} due to {e.DisallowedReason}");
 }