/// <summary> /// Begin spider /// </summary> private static void Run() { //Add other areaInfo Dictionary <string, string> areaDic = DazhongdianpingArea.GetAreaDic(); List <string> urls = new List <string>(); foreach (var key in areaDic.Keys) { for (int i = 1; i <= 50; i++) { urls.Add(string.Format(urlAreaInfo, key, i)); } } RunSpider runSpiders = new RunSpider("DazhongdianpingProcessor", "DazhongdianpingPipeline", "UTF-8", true); runSpiders.Run(urls); //RunSpider runSpider = new RunSpider("DazhongdianpingProcessor", "DazhongdianpingPipeline", "UTF-8", true); //runSpider.Run(urlInfo, 50); }
private void RunSpider() { bool res = true; List <Model.RunSpider> runspiderlist = new List <RunSpider>(); List <string> cmdlist = new List <string>(); //frist get the select one for (int i = 0; i < dgvconfigspider.Rows.Count; i++) { //check if it is select if (Convert.ToBoolean(dgvconfigspider.Rows[i].Cells["IsSelect"].EditedFormattedValue)) { //dgvconfigspider.Rows[i].DefaultCellStyle.BackColor = Color.Green; int spiderid = Convert.ToInt32(dgvconfigspider.Rows[i].Cells["Sid"].Value); int configID = Convert.ToInt32(dgvconfigspider.Rows[i].Cells["SpiderConfigID"].Value); Model.RunSpider runspider = new RunSpider(); runspider.RowIndex = i; runspider.Spider = bllspider.GetModel(spiderid); runspider.SpiderConfig = bllspiderconfig.GetModel(configID); runspiderlist.Add(runspider); } } if (runspiderlist.Count < 1) { NoteInfo(Info.Warning, "未选中任何一行爬虫,请重试!"); return; } //then generate the sqlscript runspiderlist.ForEach(x => { var cmd = ScrapyCommand.ConvertModelToCmd(x); cmdlist.Add(cmd); }); btnstartspider.Invoke((EventHandler) delegate { btnstartspider.Enabled = false; }); NoteInfo(Info.Loading, "多线程执行中..."); try { ScrapyCommand.RunSpiderCommand(cmdlist); } catch (Exception ex) { NoteInfo(Info.Error, ex.ToString()); res = true; } finally { btnstartspider.Invoke((EventHandler) delegate { btnstartspider.Enabled = true; }); } NoteInfo(Info.Right, "执行成功,请查看文件"); }