/// <summary>
/// Begin spider
/// </summary>
private static void Run()
{
//Add other areaInfo
Dictionary <string, string> areaDic = DazhongdianpingArea.GetAreaDic();
List <string> urls = new List <string>();
foreach (var key in areaDic.Keys)
{
for (int i = 1; i <= 50; i++)
{
urls.Add(string.Format(urlAreaInfo, key, i));
}
}
RunSpider runSpiders = new RunSpider("DazhongdianpingProcessor", "DazhongdianpingPipeline", "UTF-8", true);
runSpiders.Run(urls);
//RunSpider runSpider = new RunSpider("DazhongdianpingProcessor", "DazhongdianpingPipeline", "UTF-8", true);
//runSpider.Run(urlInfo, 50);
}
private void RunSpider()
{
bool res = true;
List <Model.RunSpider> runspiderlist = new List <RunSpider>();
List <string> cmdlist = new List <string>();
//frist get the select one
for (int i = 0; i < dgvconfigspider.Rows.Count; i++)
{
//check if it is select
if (Convert.ToBoolean(dgvconfigspider.Rows[i].Cells["IsSelect"].EditedFormattedValue))
{
//dgvconfigspider.Rows[i].DefaultCellStyle.BackColor = Color.Green;
int spiderid = Convert.ToInt32(dgvconfigspider.Rows[i].Cells["Sid"].Value);
int configID = Convert.ToInt32(dgvconfigspider.Rows[i].Cells["SpiderConfigID"].Value);
Model.RunSpider runspider = new RunSpider();
runspider.RowIndex = i;
runspider.Spider = bllspider.GetModel(spiderid);
runspider.SpiderConfig = bllspiderconfig.GetModel(configID);
runspiderlist.Add(runspider);
}
}
if (runspiderlist.Count < 1)
{
NoteInfo(Info.Warning, "未选中任何一行爬虫，请重试!");
return;
}
//then generate the sqlscript
runspiderlist.ForEach(x =>
{
var cmd = ScrapyCommand.ConvertModelToCmd(x);
cmdlist.Add(cmd);
});
btnstartspider.Invoke((EventHandler) delegate
{
btnstartspider.Enabled = false;
});
NoteInfo(Info.Loading, "多线程执行中...");
try
{
ScrapyCommand.RunSpiderCommand(cmdlist);
}
catch (Exception ex)
{
NoteInfo(Info.Error, ex.ToString());
res = true;
}
finally
{
btnstartspider.Invoke((EventHandler) delegate
{
btnstartspider.Enabled = true;
});
}
NoteInfo(Info.Right, "执行成功，请查看文件");
}
