/// <summary>
/// 处理队列
/// </summary>
/// <param name="encoding"></param>
/// <returns></returns>
public void ProcessQueue(Encoding encoding)
{
DateTime startTime = DateTime.Now;
logInfo.InfoFormat("[downloader]开始处理请求时间：{0}，总共需要处理请求的数量为：{1}",
startTime, HandleData.handingUrlQueue.Count);
this.handingUrlCount = 0;
CrawlerItem cI = null;
while (HandleData.handingUrlQueue.Count > 0)
{
cI = HandleData.handingUrlQueue.Dequeue();
# region 步下载
//cI = urlQueue.Dequeue();
////获取请求的内容
//string strContent = GetPageByHttpWebRequest(cI.requstUrl, encoding, cI.referer);
//if (!string.IsNullOrEmpty(strContent))
//{
// ParseResponseContent.responseContentQueue.Enqueue(strContent);
//}
#endregion
#region 异步下载
AsyncDownloadString(cI.requstUrl, cI.referer, encoding);
#endregion
//请求一次，进行时间暂停
Thread.Sleep(this.requestTimeSpan);
}
/// <summary>
/// 队列新增项
/// </summary>
/// <param name="articleCategory">文章分类</param>
/// <param name="strUrl">请求url</param>
/// <param name="refer">url的refer</param>
public void AddUrlQueue(string strUrl, string refer)
{
CrawlerItem cI = new CrawlerItem();
cI.requstUrl = strUrl;
cI.referer = refer;
HandleData.handingUrlQueue.Enqueue(cI);
//增加数量
//int count = Interlocked.Increment(ref handingUrlCount);
}
