/// <summary>
/// 开始线程
/// </summary>
/// <param name="data"></param>
public void DoWork(object data)
{
try
{
while (true)
{
if (_crawler.crawlRequests.Count == 0)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
continue;
}
CrawlRequest crawlRequest = null;
lock (_crawler.crawlRequests)
{
crawlRequest = _crawler.crawlRequests.Pop();
}
if (crawlRequest == null)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
continue;
}
try
{
_logger.Debug($"begin request url[{crawlRequest.Url}]");
// 获取页面
HtmlWeb web = new HtmlWeb();
web.OverrideEncoding = Encoding.GetEncoding("gb2312");
var htmlDoc = web.Load(crawlRequest.Url);
crawlRequest.Callback(new CallbackEventArgs(crawlRequest.Url, htmlDoc, crawlRequest.Metadata, Name));
}
catch (Exception ex)
{
_logger.Error(ex, $"request url[{crawlRequest.Url}]");
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
}
catch (Exception ex)
{
_logger.Error(ex, $"DoWork failed");
// 线程被放弃
}
}