private void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
var mssg = string.Format("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
_logger.Debug(mssg);
}
public void Constructor_ValidReason_SetsPublicProperty()
{
string reason = "aaa";
PageCrawlDisallowedArgs args = new PageCrawlDisallowedArgs(_context, _page, reason);
Assert.AreSame(reason, args.DisallowedReason);
}
private void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
var result = string.Format("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
log.Info(result);
}
public static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
try
{
PageToCrawl pageToCrawl = e.PageToCrawl;
Console.WriteLine("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
}catch (Exception)
{
}
}
void crawler_PageCrawlDisallowedAsync(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
string msg;
m_linksSkipped++;
updateCrawlingProgress(-1, -1, m_linksSkipped);
msg = "Did not crawl the links on page " + pageToCrawl.Uri.AbsoluteUri.ToString() + " due to " + e.DisallowedReason.ToString();
log.Warn(msg);
}
private static void WebCrawler_PageCrawlDisallowedAsync(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
Console.WriteLine("Abot-Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
}
void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
OnMessageReceived($"Did not crawl page {pageToCrawl.Uri.AbsoluteUri} due to {e.DisallowedReason}");
}
private void PageCrawlDisallowedEvent(object sender, PageCrawlDisallowedArgs e)
{
}
private void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
_logger.InfoFormat("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
}
private void Crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
Console.WriteLine($"Page Crawl Disallowed, Page: {e.PageToCrawl.Uri}, Reason: {e.DisallowedReason}");
}
static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
Log("crawler_PageCrawlDisallowed", e.PageToCrawl.Uri.AbsoluteUri);
var e1 = e;
//Process data
}
private static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
}
void PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
Log.Logger.Error($"Did not crawl page {pageToCrawl.Uri.AbsoluteUri} due to {e.DisallowedReason}");
}
/// <summary>
/// Method to handle when page crawling was disallowed
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void crawlerProcessPageCrawlDisallowedHandler(object sender, PageCrawlDisallowedArgs args)
{
Debug.WriteLine("Did not crawl page {0}", args);
this.PageCrawlDisallowed?.Invoke(this, args);
}
static void Crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
_log.Info($"Did not crawl page {pageToCrawl.Uri.AbsoluteUri} due to {e.DisallowedReason}");
}
//抓取页面失败
public static void Disallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
Outputer.Output(string.Format("由于产生错误：{1} 无法抓取页面{0}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason));
}
/// <summary>
/// 不爬取这个页面的原因
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
log.Info("不爬取此页面 " + pageToCrawl.Uri.AbsoluteUri + " 其原因为 " + e.DisallowedReason);
}
private void crawler_CrawlerDisalowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl page = e.PageToCrawl;
Console.WriteLine("Disallowed: {0}", page.Uri.ToString());
}
static void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
//Process data
}
void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
}
void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
//PageToCrawl pageToCrawl = e.PageToCrawl;
//Console.WriteLine("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
}
void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
Log($"Did not crawl page {e.PageToCrawl.Uri.AbsoluteUri} due to {e.DisallowedReason}");
}
