public string DoOneJob(IPipeline Pipeline)
{
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
Author author = GetNextJob();
if (author != null)
{
try
{
author.RefreshStatus = Enums.CrawlStatus.Stop;
try
{
SendMsg(string.Format("正在更新用户{0}的个人信息", author.AuthorName));
//更新用户个人信息
NetDimension.Weibo.Entities.user.Entity user = WeiboAPI.GetAuthorInfo(author.AuthorID);
var author_new = AuthorDBManager.ConvertToAuthor(user, author.AuthorSource);
AuthorDBManager.InsertOrUpdateAuthorInfo(author);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (Exception ex)
{
SendMsg("获取个人信息时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.RefreshStatus = Enums.CrawlStatus.Normal;
Logger.Error(ex.ToString());
}
SendMsg(string.Format("用户{0}的个人信息更新完毕，开始刷新其最新微博", author.AuthorName));
List <NetDimension.Weibo.Entities.status.Entity> result = new List <NetDimension.Weibo.Entities.status.Entity>();
try
{
//获取最新若干微博
WeiboAPI.GetAuthorLatestStatus(author, result, author.PostSampleMethode);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (Exception ex)
{
SendMsg("获取最新微博时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.RefreshStatus = Enums.CrawlStatus.Normal;
Logger.Error(ex.ToString());
}
SendMsg(string.Format("找到{0}的{1}条最新微博，开始插入数据库并统计相关信息", author.AuthorName, result.Count));
//同时更新平均转发数和评论数
double avgForward = 0, avgReply = 0;
for (int i = 0; i < result.Count; ++i)
{
avgForward += (double)result[i].RepostsCount / (double)result.Count;
avgReply += (double)result[i].CommentsCount / (double)result.Count;
var item = ItemDBManager.ConvertToItem(result[i], author.AuthorSource, CrawlID);
ItemDBManager.InsertOrUpdateItem(item);
CntData.Tick();
}
author.AvgForward = (int)avgForward;
author.AvgReply = (int)avgReply;
SuccCount++;
continue;
}
catch (Exception ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.RefreshStatus = Enums.CrawlStatus.Normal;
SendMsg(ex.ToString());
}
finally
{
author.UpdateCount++;
author.UpdateTime = DateTime.Now;
author.NextRefreshTime = author.UpdateTime.AddDays(author.IntervalDays);
AuthorDBManager.PushbackCensusJob(author);
SendMsg(string.Format("用户{0}的普查任务完成", author.AuthorName));
}
}
}
SendMsg(string.Format("休息{0}秒", IntervalMS / 1000));
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
/// <summary>
/// 定期检查账号情况
/// </summary>
private static void NormalizeAccount()
{
string where;
try
{
#region 初始化账号
where = "IsDisabled=FALSE AND SubscribeStatus>=0 AND AuthorID IS NULL";
var result = LoginAccountBusiness.GetAllByWhere(where, null, LoginAccountMySqlDAL.OrderColumn.Default);
foreach (var account in result)
{
try
{
account.AuthorID = WeiboAPI.GetAuthorID(account.UserName);
var author = WeiboAPI.GetAuthorInfo(account.AuthorID);
var list = WeiboAPI.GetFriendsIDs(account.AuthorID, Enums.SampleMethod.All);
account.FollowCount = 0;
foreach (var id in list)
{
AuthorDBManager.SubscribeAuthor(account.UserName, id);
account.FollowCount++;
}
}
catch (Exception ex)
{
Logger.Error(string.Format("Err@ Account: {0}, message: {1}", account.UserName, ex.ToString()));
account.FailCount++;
if (account.FailCount > DefaultSettings.MaxFailureCount)
{
account.IsDisabled = true;
}
LoginAccountBusiness.UpdateByAccountID(account);
continue;
}
account.FailCount = 0;
account.SubscribeIntervalMins = 15;
account.SubscribeLastTime = Utilities.Epoch;
account.SubscribeNextTime = Utilities.Epoch;
account.SubscribeSinceID = null;
account.FollowCountLastClearTime = DateTime.Now;
account.FollowCountToday = 0;
LoginAccountBusiness.UpdateByAccountID(account);
}
#endregion
#region 关注次数和失效次数清零（每天一次）
DateTime refreshtime = DateTime.Now.Subtract(new TimeSpan(1, 0, 0, 0, 0));
where = string.Format("IsDisabled=FALSE AND SubscribeStatus>=0 AND (FollowCountLastClearTime IS NULL OR FollowCountLastClearTime<'{0}')", refreshtime);
result = LoginAccountBusiness.GetAllByWhere(where, null, LoginAccountMySqlDAL.OrderColumn.Default);
foreach (var account in result)
{
account.FollowCountLastClearTime = DateTime.Now;
account.FollowCountToday = 0;
account.FailCount = 0;
LoginAccountBusiness.UpdateByAccountID(account);
}
#endregion
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
}
}