public void Execute()
{
List <SpiderTemplate> list = SpiderTemplate.find("IsDelete=0").list();
DbContext.closeConnectionAll();
logger.Info("begin SpiderJob=" + list.Count);
StringBuilder log = new StringBuilder();
foreach (SpiderTemplate s in list)
{
ISpiderTool spider = getSpider(s);
spider.DownloadPage(s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo }); // 2~6秒暂停
DbContext.closeConnectionAll();
int sleepms = rd.Next(SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo);
Thread.Sleep(sleepms);
}
String[] arrLog = log.ToString().Split('\n');
StringBuilder errorLog = new StringBuilder();
foreach (String item in arrLog)
{
if (item.Trim().StartsWith("error="))
{
errorLog.AppendLine(item.Trim());
}
}
SpiderLog sg = new SpiderLog();
sg.Msg = errorLog.ToString();
sg.insert();
DbContext.closeConnectionAll();
}
public List <SpiderTemplate> GetAll()
{
return(SpiderTemplate.find("order by OrderId desc, Id asc").list());
}