static async void DoWork(ConcurrentQueue <HttpUrlWork> queue, HttpUrlWork workItem)
{
string html = HttpHelper.HtmlCode(workItem.URL);
if (null != CallBackAddUrl)
{
CallBackAddUrl(workItem.URL, html);
}
//var lstImgLink = HttpHelper.GetHtmlImageUrlList(html);
//for (int i = 0; i < lstImgLink.Count; i++)
//{
// ImgUrls.Add(lstImgLink[i]);
// string strimg = lstImgLink[i];
// TaskHelperImage.taskQueue.Enqueue(strimg);
//}
var lstLink = HttpHelper.GetLinks(html);
for (int i = 0; i < lstLink.Count; i++)
{
var strUrl = lstLink[i];
await TaskProducer(queue, new HttpUrlWork()
{
URL = strUrl
});
//queue.Enqueue(new HttpUrlWork() { URL = strUrl });
}
}
private void button2_Click(object sender, EventArgs e)
{
string url = "http://blog.sina.com.cn/cc918035096";
GlobalConfig.WebUrl = url;
HttpUrlWork work = new HttpUrlWork()
{
URL = url
};
HttpHelper.HtmlCode(url);
}
//生成
static async Task TaskProducer(ConcurrentQueue <HttpUrlWork> queue, HttpUrlWork workItem)
{
string strUrl = workItem.URL;
bool dequeueSuccesful = HaveBeenVisitedURL.Contains(strUrl);
if (!dequeueSuccesful)
{
HaveBeenVisitedURL.Add(strUrl);
Console.WriteLine(workItem.URL);
await Task.Delay(50);
queue.Enqueue(workItem);
}
//await Task.Delay(10);
}
private void button1_Click(object sender, EventArgs e)
{
string url = "http://588ku.com/sucai/0-pxnum-0-0-0-1/?h=bd&sem=1";
//string url = StringHelper.GetPureUrl("https://www.zhihu.com/question/266467210/answer/308342541");
GlobalConfig.WebUrl = StringHelper.GetPureUrl(url);
HttpUrlWork work = new HttpUrlWork()
{
URL = url
};
TaskHelperCopy.CallBackAddUrl = ShowUrl;
TaskHelperImage.ImageCallBack = ShowImageUrl;
Task t = TaskHelperCopy.RunProgram(work);
Task t1 = TaskHelperImage.RunProgram();
//Task t = RunProgram();
//t.Wait();
//Console.ReadKey();
}
public static async Task RunProgram(HttpUrlWork workSrc)
{
ConcurrentQueue <HttpUrlWork> taskQueue = new ConcurrentQueue <HttpUrlWork>();
var cts = new CancellationTokenSource();
//生成任务添加至并发队列
var taskSource = Task.Run(() => TaskProducer(taskQueue, workSrc));
//同时启动四个任务处理队列中的任务
Task[] processors = new Task[10];
for (int i = 1; i <= 10; i++)
{
string processId = i.ToString();
processors[i - 1] = Task.Run(() => TaskProcessor(taskQueue, cts.Token, processId));
}
await taskSource;
//向任务发送取消信号
cts.CancelAfter(TimeSpan.FromSeconds(2));
await Task.WhenAll(processors);
}
