public void Execute()
{
List<SpiderTemplate> list = SpiderTemplate.find( "IsDelete=0" ).list();
DbContext.closeConnectionAll();
logger.Info( "begin SpiderJob=" + list.Count );
StringBuilder log = new StringBuilder();
foreach (SpiderTemplate s in list) {
ISpiderTool spider = getSpider( s );
spider.DownloadPage( s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo } ); // 2~6秒暂停
DbContext.closeConnectionAll();
int sleepms = rd.Next( SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo );
Thread.Sleep( sleepms );
}
String[] arrLog = log.ToString().Split( '\n' );
StringBuilder errorLog = new StringBuilder();
foreach (String item in arrLog) {
if (item.Trim().StartsWith( "error=" )) errorLog.AppendLine( item.Trim() );
}
SpiderLog sg = new SpiderLog();
sg.Msg = errorLog.ToString();
sg.insert();
DbContext.closeConnectionAll();
}
public void Execute()
{
List <SpiderTemplate> list = SpiderTemplate.find("IsDelete=0").list();
DbContext.closeConnectionAll();
logger.Info("begin SpiderJob=" + list.Count);
StringBuilder log = new StringBuilder();
foreach (SpiderTemplate s in list)
{
ISpiderTool spider = getSpider(s);
spider.DownloadPage(s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo }); // 2~6秒暂停
DbContext.closeConnectionAll();
int sleepms = rd.Next(SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo);
Thread.Sleep(sleepms);
}
String[] arrLog = log.ToString().Split('\n');
StringBuilder errorLog = new StringBuilder();
foreach (String item in arrLog)
{
if (item.Trim().StartsWith("error="))
{
errorLog.AppendLine(item.Trim());
}
}
SpiderLog sg = new SpiderLog();
sg.Msg = errorLog.ToString();
sg.insert();
DbContext.closeConnectionAll();
}
public void Execute()
{
// List<SpiderTemplate> list = SpiderTemplate.find( "IsDelete=0" ).list();
DbContext.closeConnectionAll();
StringBuilder log = new StringBuilder();
IList userRanks = User.find("order by Hits desc, id desc").list(1000);
logger.Info("begin SpiderJob=" + userRanks.Count);
foreach (User user in userRanks)
{
if (string.IsNullOrEmpty(user.Profile.Address))
{
continue;
}
SpiderTemplate s = new SpiderTemplate();
s.ListUrl = user.Profile.Address;
s.ListEncoding = user.QQ;
s.ListBodyPattern = user.Profile.Tel;
s.ListPattern = user.Profile.WebSite;
s.DetailPattern = user.MSN;
s.IsDelete = user.Id;
s.SiteName = user.Url;
ISpiderTool spider = getSpider(s);
spider.DownloadPage(s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo }); // 2~6秒暂停
DbContext.closeConnectionAll();
int sleepms = rd.Next(SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo);
Thread.Sleep(sleepms);
}
//foreach (SpiderTemplate s in list) {
// ISpiderTool spider = getSpider( s );
// spider.DownloadPage( s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo } ); // 2~6秒暂停
// DbContext.closeConnectionAll();
// int sleepms = rd.Next( SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo );
// Thread.Sleep( sleepms );
//}
String[] arrLog = log.ToString().Split('\n');
StringBuilder errorLog = new StringBuilder();
foreach (String item in arrLog)
{
if (item.Trim().StartsWith("error="))
{
errorLog.AppendLine(item.Trim());
}
}
SpiderLog sg = new SpiderLog();
sg.Msg = errorLog.ToString();
sg.insert();
DbContext.closeConnectionAll();
}
public void Execute()
{
// List<SpiderTemplate> list = SpiderTemplate.find( "IsDelete=0" ).list();
DbContext.closeConnectionAll();
StringBuilder log = new StringBuilder();
IList userRanks = User.find("order by Hits desc, id desc").list(1000);
logger.Info("begin SpiderJob=" + userRanks.Count);
foreach (User user in userRanks)
{
if (string.IsNullOrEmpty(user.Profile.Address))
continue;
SpiderTemplate s = new SpiderTemplate();
s.ListUrl = user.Profile.Address;
s.ListEncoding = user.QQ;
s.ListBodyPattern = user.Profile.Tel;
s.ListPattern = user.Profile.WebSite;
s.DetailPattern = user.MSN;
s.IsDelete = user.Id;
s.SiteName = user.Url;
ISpiderTool spider = getSpider(s);
spider.DownloadPage(s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo }); // 2~6秒暂停
DbContext.closeConnectionAll();
int sleepms = rd.Next(SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo);
Thread.Sleep(sleepms);
}
//foreach (SpiderTemplate s in list) {
// ISpiderTool spider = getSpider( s );
// spider.DownloadPage( s, log, new int[] { SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo } ); // 2~6秒暂停
// DbContext.closeConnectionAll();
// int sleepms = rd.Next( SpiderConfig.SuspendFrom, SpiderConfig.SuspendTo );
// Thread.Sleep( sleepms );
//}
String[] arrLog = log.ToString().Split( '\n' );
StringBuilder errorLog = new StringBuilder();
foreach (String item in arrLog) {
if (item.Trim().StartsWith( "error=" )) errorLog.AppendLine( item.Trim() );
}
SpiderLog sg = new SpiderLog();
sg.Msg = errorLog.ToString();
sg.insert();
DbContext.closeConnectionAll();
}
