static async void DoWork(ConcurrentQueue <HttpUrlWork> queue, HttpUrlWork workItem) { string html = HttpHelper.HtmlCode(workItem.URL); if (null != CallBackAddUrl) { CallBackAddUrl(workItem.URL, html); } //var lstImgLink = HttpHelper.GetHtmlImageUrlList(html); //for (int i = 0; i < lstImgLink.Count; i++) //{ // ImgUrls.Add(lstImgLink[i]); // string strimg = lstImgLink[i]; // TaskHelperImage.taskQueue.Enqueue(strimg); //} var lstLink = HttpHelper.GetLinks(html); for (int i = 0; i < lstLink.Count; i++) { var strUrl = lstLink[i]; await TaskProducer(queue, new HttpUrlWork() { URL = strUrl }); //queue.Enqueue(new HttpUrlWork() { URL = strUrl }); } }
private void button2_Click(object sender, EventArgs e) { string url = "http://blog.sina.com.cn/cc918035096"; GlobalConfig.WebUrl = url; HttpUrlWork work = new HttpUrlWork() { URL = url }; HttpHelper.HtmlCode(url); }
//生成 static async Task TaskProducer(ConcurrentQueue <HttpUrlWork> queue, HttpUrlWork workItem) { string strUrl = workItem.URL; bool dequeueSuccesful = HaveBeenVisitedURL.Contains(strUrl); if (!dequeueSuccesful) { HaveBeenVisitedURL.Add(strUrl); Console.WriteLine(workItem.URL); await Task.Delay(50); queue.Enqueue(workItem); } //await Task.Delay(10); }
private void button1_Click(object sender, EventArgs e) { string url = "http://588ku.com/sucai/0-pxnum-0-0-0-1/?h=bd&sem=1"; //string url = StringHelper.GetPureUrl("https://www.zhihu.com/question/266467210/answer/308342541"); GlobalConfig.WebUrl = StringHelper.GetPureUrl(url); HttpUrlWork work = new HttpUrlWork() { URL = url }; TaskHelperCopy.CallBackAddUrl = ShowUrl; TaskHelperImage.ImageCallBack = ShowImageUrl; Task t = TaskHelperCopy.RunProgram(work); Task t1 = TaskHelperImage.RunProgram(); //Task t = RunProgram(); //t.Wait(); //Console.ReadKey(); }
public static async Task RunProgram(HttpUrlWork workSrc) { ConcurrentQueue <HttpUrlWork> taskQueue = new ConcurrentQueue <HttpUrlWork>(); var cts = new CancellationTokenSource(); //生成任务添加至并发队列 var taskSource = Task.Run(() => TaskProducer(taskQueue, workSrc)); //同时启动四个任务处理队列中的任务 Task[] processors = new Task[10]; for (int i = 1; i <= 10; i++) { string processId = i.ToString(); processors[i - 1] = Task.Run(() => TaskProcessor(taskQueue, cts.Token, processId)); } await taskSource; //向任务发送取消信号 cts.CancelAfter(TimeSpan.FromSeconds(2)); await Task.WhenAll(processors); }