public void Execute() { List<SpiderTemplate> list = SpiderTemplate.find( "IsDelete=0" ).list(); DbContext.closeConnectionAll(); logger.Info( "begin SpiderJob=" + list.Count ); StringBuilder log = new StringBuilder(); foreach (SpiderTemplate s in list) { ISpiderTool spider = getSpider( s ); spider.DownloadPage( s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo } ); // 2~6秒暂停 DbContext.closeConnectionAll(); int sleepms = rd.Next( SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo ); Thread.Sleep( sleepms ); } String[] arrLog = log.ToString().Split( '\n' ); StringBuilder errorLog = new StringBuilder(); foreach (String item in arrLog) { if (item.Trim().StartsWith( "error=" )) errorLog.AppendLine( item.Trim() ); } SpiderLog sg = new SpiderLog(); sg.Msg = errorLog.ToString(); sg.insert(); DbContext.closeConnectionAll(); }
public void Execute() { List <SpiderTemplate> list = SpiderTemplate.find("IsDelete=0").list(); DbContext.closeConnectionAll(); logger.Info("begin SpiderJob=" + list.Count); StringBuilder log = new StringBuilder(); foreach (SpiderTemplate s in list) { ISpiderTool spider = getSpider(s); spider.DownloadPage(s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo }); // 2~6秒暂停 DbContext.closeConnectionAll(); int sleepms = rd.Next(SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo); Thread.Sleep(sleepms); } String[] arrLog = log.ToString().Split('\n'); StringBuilder errorLog = new StringBuilder(); foreach (String item in arrLog) { if (item.Trim().StartsWith("error=")) { errorLog.AppendLine(item.Trim()); } } SpiderLog sg = new SpiderLog(); sg.Msg = errorLog.ToString(); sg.insert(); DbContext.closeConnectionAll(); }
public void Execute() { // List<SpiderTemplate> list = SpiderTemplate.find( "IsDelete=0" ).list(); DbContext.closeConnectionAll(); StringBuilder log = new StringBuilder(); IList userRanks = User.find("order by Hits desc, id desc").list(1000); logger.Info("begin SpiderJob=" + userRanks.Count); foreach (User user in userRanks) { if (string.IsNullOrEmpty(user.Profile.Address)) { continue; } SpiderTemplate s = new SpiderTemplate(); s.ListUrl = user.Profile.Address; s.ListEncoding = user.QQ; s.ListBodyPattern = user.Profile.Tel; s.ListPattern = user.Profile.WebSite; s.DetailPattern = user.MSN; s.IsDelete = user.Id; s.SiteName = user.Url; ISpiderTool spider = getSpider(s); spider.DownloadPage(s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo }); // 2~6秒暂停 DbContext.closeConnectionAll(); int sleepms = rd.Next(SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo); Thread.Sleep(sleepms); } //foreach (SpiderTemplate s in list) { // ISpiderTool spider = getSpider( s ); // spider.DownloadPage( s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo } ); // 2~6秒暂停 // DbContext.closeConnectionAll(); // int sleepms = rd.Next( SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo ); // Thread.Sleep( sleepms ); //} String[] arrLog = log.ToString().Split('\n'); StringBuilder errorLog = new StringBuilder(); foreach (String item in arrLog) { if (item.Trim().StartsWith("error=")) { errorLog.AppendLine(item.Trim()); } } SpiderLog sg = new SpiderLog(); sg.Msg = errorLog.ToString(); sg.insert(); DbContext.closeConnectionAll(); }
public void Execute() { // List<SpiderTemplate> list = SpiderTemplate.find( "IsDelete=0" ).list(); DbContext.closeConnectionAll(); StringBuilder log = new StringBuilder(); IList userRanks = User.find("order by Hits desc, id desc").list(1000); logger.Info("begin SpiderJob=" + userRanks.Count); foreach (User user in userRanks) { if (string.IsNullOrEmpty(user.Profile.Address)) continue; SpiderTemplate s = new SpiderTemplate(); s.ListUrl = user.Profile.Address; s.ListEncoding = user.QQ; s.ListBodyPattern = user.Profile.Tel; s.ListPattern = user.Profile.WebSite; s.DetailPattern = user.MSN; s.IsDelete = user.Id; s.SiteName = user.Url; ISpiderTool spider = getSpider(s); spider.DownloadPage(s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo }); // 2~6秒暂停 DbContext.closeConnectionAll(); int sleepms = rd.Next(SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo); Thread.Sleep(sleepms); } //foreach (SpiderTemplate s in list) { // ISpiderTool spider = getSpider( s ); // spider.DownloadPage( s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo } ); // 2~6秒暂停 // DbContext.closeConnectionAll(); // int sleepms = rd.Next( SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo ); // Thread.Sleep( sleepms ); //} String[] arrLog = log.ToString().Split( '\n' ); StringBuilder errorLog = new StringBuilder(); foreach (String item in arrLog) { if (item.Trim().StartsWith( "error=" )) errorLog.AppendLine( item.Trim() ); } SpiderLog sg = new SpiderLog(); sg.Msg = errorLog.ToString(); sg.insert(); DbContext.closeConnectionAll(); }