private static void NormalizeAccount()
{
string[] acc = File.ReadAllLines("moreAccount.txt");
foreach (var st in acc)
{
string[] tmp = st.Split(' ');
string username = tmp[0], password = tmp[1];
OAuth o = new OAuth(DefaultSettings.AppKey, DefaultSettings.AppSecret, String.Empty);
o.ClientLogin(username, password);
Client client = new Client(o);
try
{
var s = client.API.Entity.Account.GetUID();
var collection = WeiboAPI.GetFriendsIDs(s, Enums.SampleMethod.All);
foreach (var id in collection)
{
client.API.Entity.Friendships.Destroy(id);
Thread.Sleep(1000);
}
Console.WriteLine(username + " successed");
File.AppendAllText("accountResult.txt", username + " " + password + '\n');
}
catch (WeiboException ex)
{
Console.WriteLine(username + " " + ex.ErrorCode + " " + ex.ErrorMessage);
}
Thread.Sleep(9000);
}
}
/// <summary>
/// 取消对某个用户的关注
/// </summary>
/// <param name="username">关注某个用户的僵尸账号</param>
/// <param name="targetAuthorID">需要被取消的用户ID</param>
/// <returns>是否取消成功</returns>
public static bool UnFollowUser(string targetAuthorID)
{
string username = AuthorDBManager.GetUsernameFollowing(targetAuthorID);
if (username == null)
{
return(false);
}
string where = string.Format("UserName='******'", username);
LoginAccountEntity acc = null;
try
{
acc = LoginAccountBusiness.GetTopByWhere(where, null, LoginAccountMySqlDAL.OrderColumn.Default);
if (acc == null)
{
return(false);
}
try
{
WeiboAPI.UnFollowUser(username, targetAuthorID);
}
catch (WeiboException ex)
{
Logger.Error(ex.ToString());
acc.FailCount++;
if (acc.FailCount > DefaultSettings.MaxFailureCount)
{
acc.IsDisabled = true;
}
return(false);
}
acc.FollowCount--;
AuthorDBManager.UnSubscribeAuthor(targetAuthorID);
return(true);
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
return(false);
}
finally
{
if (acc != null)
{
var param = new LoginAccountMySqlDAL.LoginAccountParameter[3];
param[0] = new LoginAccountMySqlDAL.LoginAccountParameter(Palas.Common.Lib.DTO.LoginAccountDTO.Column.FollowCount, acc.FollowCount);
param[1] = new LoginAccountMySqlDAL.LoginAccountParameter(Palas.Common.Lib.DTO.LoginAccountDTO.Column.FailCount, acc.FailCount);
param[2] = new LoginAccountMySqlDAL.LoginAccountParameter(Palas.Common.Lib.DTO.LoginAccountDTO.Column.IsDisabled, acc.IsDisabled);
LoginAccountBusiness.UpdateWhere(param, where, null);
}
}
}
public static Tuple <string, float, float> AddNewLocation(dynamic place, string url)
{
Location loc = new Location();
loc.PoIDSource = WeiboUtilities.GetPoIDSource(url);
loc.Url = url;
loc.IntervalMins = 15;
loc.LocationSampleMethode = Enums.SampleMethod.All;
loc.PoID = Guid.NewGuid().ToString("N");
loc.Radius = 600;
loc.RefreshStatus = Enums.CrawlStatus.Stop;
try
{
loc.Lat = float.Parse(place.lat);
loc.Lon = float.Parse(place.lon);
}
catch (Exception)
{
try
{
loc.Lat = (float)place.lat;
loc.Lon = (float)place.lon;
}
catch (Exception) { }
}
loc.Title = place.title;
loc.ClientID = place.poiid;
try
{
if (loc.ClientID != null)
{
WeiboAPI.SetPOIInfo(loc, loc.ClientID);
}
}
catch (Exception) { }
LocationDBManager.AddNewLocation(loc);
return(new Tuple <string, float, float>(loc.PoID, loc.Lon, loc.Lat));
}
public string DoOneJob(IPipeline Pipeline)
{
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
Location loc = GetNextJob();
if (loc != null)
{
try
{
DateTime lastTime = loc.LastRefreshTime;
loc.LastRefreshTime = DateTime.Now;
loc.NextRefreshTime = DateTime.Now.AddMinutes(loc.IntervalMins);
//获取CBD周围的人
List <dynamic> users = new List <dynamic>();
List <dynamic> statuses = new List <dynamic>();
try
{
SendMsg(string.Format("正在刷新{0}周围的位置动态", loc.Title));
WeiboAPI.GetUsersNearCBD(loc.Lon, loc.Lat, loc.Radius, Utilities.DateTime2UnixTime(lastTime), Utilities.DateTime2UnixTime(DateTime.Now), loc.LocationSampleMethode, users, statuses);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (WeiboException ex)
{
SendMsg("获取CBD周围动态时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
Logger.Error(ex.ToString());
}
SendMsg(string.Format("{0}的位置动态获取到{1}条，开始插入数据库", loc.Title, users.Count()));
for (int i = 0; i < users.Count; ++i)
{
if (i >= statuses.Count)
{
break;
}
if (statuses[i] == null || users[i] == null)
{
continue;
}
Item item = ItemDBManager.ConvertToItem(Enums.AuthorSource.LocationScan, CrawlID, statuses[i], users[i]);
if (item.PoID == null)
{
item.PoID = item.PoIDSource = loc.CategoryID;
}
ItemDBManager.InsertOrUpdateItem(item);
Author author = AuthorDBManager.ConvertToAuthor(users[i], Enums.AuthorSource.LocationScan);
AuthorDBManager.InsertOrUpdateAuthorInfo(author);
CntData.Tick();
}
SuccCount++;
SendMsg(string.Format("{0}的任务完成", loc.Title));
continue;
}
catch (Exception ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
Logger.Error(ex.ToString());
}
finally
{
loc.RefreshCount++;
loc.RefreshStatus = Enums.CrawlStatus.Normal;
LocationDBManager.PushbackLoationJob(loc);
}
}
}
SendMsg(string.Format("休息{0}秒", IntervalMS / 1000));
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
public string DoOneJob(IPipeline Pipeline)
{
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
Author author = GetNextJob();
if (author != null)
{
try
{
//如果不是红人，那么只刷新一次就结束
if (WeiboUtilities.IsRedSkin(author.AuthorSource))
{
author.Fans_RefreshStatus = Enums.CrawlStatus.Normal;
}
else
{
author.Fans_RefreshStatus = Enums.CrawlStatus.Stop;
}
#region 用户粉丝刷新
List <NetDimension.Weibo.Entities.user.Entity> users = new List <NetDimension.Weibo.Entities.user.Entity>();
try
{
SendMsg(string.Format("正在刷新{0}的粉丝", author.AuthorName));
WeiboAPI.GetFollowers(author.AuthorID, author.FollowerSampleMethode, users);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (Exception ex)
{
SendMsg("获取粉丝列表时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.Fans_RefreshStatus = Enums.CrawlStatus.Normal;
Logger.Error(ex.ToString());
}
SendMsg(string.Format("{0}的粉丝抓取到{1}个，开始插入数据库", author.AuthorName, users.Count));
double avg = 0; //用户粉丝的粉丝平均数
for (int i = 0; i < users.Count; ++i)
{
var user = AuthorDBManager.ConvertToAuthor(users[i], Enums.AuthorSource.FansDiscover);
AuthorDBManager.InsertOrUpdateAuthorInfo(user);
CntData.Tick();
AuthorRelationDBManager.InsertOrUpdateRelation(user.AuthorID, author.AuthorID);
avg += (double)users[i].FollowersCount / (double)users.Count;
}
#endregion
#region 用户关注列表
try
{
IEnumerable <string> friends = null;
SendMsg(string.Format("{0}的粉丝插入完成，开始获取他的关注列表", author.AuthorName));
friends = WeiboAPI.GetFriendsIDs(author.AuthorID, author.FansSampleMethode);
if (friends != null)
{
foreach (var user in friends)
{
AuthorRelationDBManager.InsertOrUpdateRelation(user, author.AuthorID);
}
}
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (Exception ex)
{
SendMsg("获取关注列表时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.Fans_RefreshStatus = Enums.CrawlStatus.Normal;
Logger.Error(ex.ToString());
}
SendMsg(string.Format("{0}的关系刷新任务完成", author.AuthorName));
#endregion
author.AvgFansCountOfFans = (int)avg;
SuccCount++;
continue;
}
catch (Exception ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.Fans_RefreshStatus = Enums.CrawlStatus.Normal;
SendMsg(ex.ToString());
Logger.Error(ex.ToString());
}
finally
{
author.Fans_UpdateCount++;
author.Fans_NextRefreshTime = DateTime.Now.AddDays(author.Fans_IntervalDays);
AuthorDBManager.PushbackRelationshipJob(author);
}
}
}
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
public string DoOneJob(IPipeline Pipeline)
{
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
LoginAccountEntity account = GetNextJob();
if (account != null)
{
try
{
//刷新订阅的微博
List <NetDimension.Weibo.Entities.status.Entity> result = new List <NetDimension.Weibo.Entities.status.Entity>();
try
{
SendMsg(string.Format("正在刷新{0}关注的最新微博", account.UserName));
WeiboAPI.FetchStatus(account, result);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (Exception ex)
{
SendMsg("获取新微博时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
Logger.Error(ex.ToString());
}
SendMsg(string.Format("{0}关注的微博抓取到{1}条，开始插入", account.UserName, result.Count));
for (int i = 0; i < result.Count; ++i)
{
var item = ItemDBManager.ConvertToItem(result[i], Enums.AuthorSource.PublicLeader, CrawlID);
ItemDBManager.InsertOrUpdateItem(item);
CntData.Tick();
}
SendMsg(string.Format("{0}的关注微博刷新任务完成", account.UserName));
SuccCount++;
continue;
}
catch (Exception ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
Logger.Error(ex.ToString());
}
finally
{
AccountDBManager.PushbackSubscribeJob(account);
}
}
}
SendMsg(string.Format("休息{0}秒", IntervalMS / 1000));
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
public string DoOneJob(IPipeline Pipeline)
{
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
Author author = GetNextJob();
if (author != null)
{
try
{
author.RefreshStatus = Enums.CrawlStatus.Stop;
try
{
SendMsg(string.Format("正在更新用户{0}的个人信息", author.AuthorName));
//更新用户个人信息
NetDimension.Weibo.Entities.user.Entity user = WeiboAPI.GetAuthorInfo(author.AuthorID);
var author_new = AuthorDBManager.ConvertToAuthor(user, author.AuthorSource);
AuthorDBManager.InsertOrUpdateAuthorInfo(author);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (Exception ex)
{
SendMsg("获取个人信息时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.RefreshStatus = Enums.CrawlStatus.Normal;
Logger.Error(ex.ToString());
}
SendMsg(string.Format("用户{0}的个人信息更新完毕，开始刷新其最新微博", author.AuthorName));
List <NetDimension.Weibo.Entities.status.Entity> result = new List <NetDimension.Weibo.Entities.status.Entity>();
try
{
//获取最新若干微博
WeiboAPI.GetAuthorLatestStatus(author, result, author.PostSampleMethode);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (Exception ex)
{
SendMsg("获取最新微博时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.RefreshStatus = Enums.CrawlStatus.Normal;
Logger.Error(ex.ToString());
}
SendMsg(string.Format("找到{0}的{1}条最新微博，开始插入数据库并统计相关信息", author.AuthorName, result.Count));
//同时更新平均转发数和评论数
double avgForward = 0, avgReply = 0;
for (int i = 0; i < result.Count; ++i)
{
avgForward += (double)result[i].RepostsCount / (double)result.Count;
avgReply += (double)result[i].CommentsCount / (double)result.Count;
var item = ItemDBManager.ConvertToItem(result[i], author.AuthorSource, CrawlID);
ItemDBManager.InsertOrUpdateItem(item);
CntData.Tick();
}
author.AvgForward = (int)avgForward;
author.AvgReply = (int)avgReply;
SuccCount++;
continue;
}
catch (Exception ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
author.RefreshStatus = Enums.CrawlStatus.Normal;
SendMsg(ex.ToString());
}
finally
{
author.UpdateCount++;
author.UpdateTime = DateTime.Now;
author.NextRefreshTime = author.UpdateTime.AddDays(author.IntervalDays);
AuthorDBManager.PushbackCensusJob(author);
SendMsg(string.Format("用户{0}的普查任务完成", author.AuthorName));
}
}
}
SendMsg(string.Format("休息{0}秒", IntervalMS / 1000));
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
public string DoOneJob(IPipeline Pipeline)
{
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
Author author = GetNextJob();
if (author != null)
{
try
{
//获取最新若干带有地点信息的微博
List <dynamic> result = new List <dynamic>();
try
{
SendMsg(string.Format("正在刷新{0}的位置动态", author.AuthorName));
WeiboAPI.GetUserStatusLocationHistory(author, result, author.LocationSampleMethode);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (Exception ex)
{
SendMsg("获取位置动态时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
Logger.Error(ex.ToString());
}
SendMsg(string.Format("{0}的位置动态获取到{1}条，开始插入数据库", author.AuthorName, result.Count()));
for (int i = 0; i < result.Count; ++i)
{
Item item = ItemDBManager.ConvertToItem(author.AuthorSource, CrawlID, result[i], null, author);
if (item.PoID == null)
{
continue;
}
ItemDBManager.InsertOrUpdateItem(item);
CntData.Tick();
}
SuccCount++;
SendMsg(string.Format("{0}的任务完成", author.AuthorName));
continue;
}
catch (Exception ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
Logger.Error(ex.ToString());
}
finally
{
author.Location_RefreshStatus = Enums.CrawlStatus.Normal;
author.Location_UpdateTime = DateTime.Now;
author.Location_NextRefreshTime = DateTime.Now.AddDays(author.Location_IntervalDays);
author.Location_UpdateCount++;
AuthorDBManager.PushbackLocHistJob(author);
}
}
}
SendMsg(string.Format("休息{0}秒", IntervalMS / 1000));
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
/// <summary>
/// 订阅某个用户
/// </summary>
/// <param name="AuthorID">待订阅的用户ID</param>
/// <returns>订阅结果枚举</returns>
public static FollowStatus FollowUser(string AuthorID)
{
string where;
LoginAccountEntity acc = null;
try
{
where = string.Format("IsDisabled=FALSE AND AuthorID IS NOT NULL AND SubscribeStatus>=0 AND FollowCountToday<{0} AND FollowCount<{1} AND SubscribeLastTime<'{2}'", DefaultSettings.FollowLimitPerDay, DefaultSettings.MaxFriendsCnt, DateTime.Now.Subtract(new TimeSpan(0, 6, 0)));
acc = LoginAccountBusiness.GetTopByWhere(where, null, LoginAccountMySqlDAL.OrderColumn.Default);
if (acc == null)
{
return(FollowStatus.Shortage);
}
try
{
Console.WriteLine(string.Format("使用账号{0}进行关注", acc.UserName));
WeiboAPI.FollowUser(acc.UserName, AuthorID);
}
catch (WeiboException ex)
{
switch (ex.ErrorCode)
{
case "20036":
Console.WriteLine("关注次数IP限制");
break;
case "20003":
AuthorDBManager.UnSubscribeAuthor(AuthorID);
Logger.Error(ex.ToString());
acc.FailCount++;
if (acc.FailCount > DefaultSettings.MaxFailureCount)
{
acc.IsDisabled = true;
}
Console.WriteLine("出现\"用户不存在\"异常，已取消关注该用户，请手工确保账号有效");
break;
default:
acc.FailCount++;
if (acc.FailCount > DefaultSettings.MaxFailureCount)
{
acc.IsDisabled = true;
}
acc.FollowCountToday = DefaultSettings.FollowLimitPerDay;
Logger.Error(ex.ToString());
break;
}
return(FollowStatus.Exception);
}
catch (Exception ex)
{
Logger.Error(ex.ToString()); return(FollowStatus.Exception);
}
acc.FollowCount++;
acc.FollowCountToday++;
acc.SubscribeLastTime = DateTime.Now;
AuthorDBManager.SubscribeAuthor(acc.UserName, AuthorID);
return(FollowStatus.Succ);
}
catch (Exception ex) { Logger.Error(ex.ToString()); return(FollowStatus.Exception); }
finally
{
if (acc != null)
{
where = string.Format("AccountID='{0}'", acc.AccountID);
var param = new LoginAccountMySqlDAL.LoginAccountParameter[5];
param[0] = new LoginAccountMySqlDAL.LoginAccountParameter(Palas.Common.Lib.DTO.LoginAccountDTO.Column.FollowCount, acc.FollowCount);
param[1] = new LoginAccountMySqlDAL.LoginAccountParameter(Palas.Common.Lib.DTO.LoginAccountDTO.Column.FollowCountToday, acc.FollowCountToday);
param[2] = new LoginAccountMySqlDAL.LoginAccountParameter(Palas.Common.Lib.DTO.LoginAccountDTO.Column.FailCount, acc.FailCount);
param[3] = new LoginAccountMySqlDAL.LoginAccountParameter(Palas.Common.Lib.DTO.LoginAccountDTO.Column.IsDisabled, acc.IsDisabled);
param[4] = new LoginAccountMySqlDAL.LoginAccountParameter(Palas.Common.Lib.DTO.LoginAccountDTO.Column.SubscribeLastTime, acc.SubscribeLastTime);
LoginAccountBusiness.UpdateWhere(param, where, null);
}
}
}
/// <summary>
/// 定期检查账号情况
/// </summary>
private static void NormalizeAccount()
{
string where;
try
{
#region 初始化账号
where = "IsDisabled=FALSE AND SubscribeStatus>=0 AND AuthorID IS NULL";
var result = LoginAccountBusiness.GetAllByWhere(where, null, LoginAccountMySqlDAL.OrderColumn.Default);
foreach (var account in result)
{
try
{
account.AuthorID = WeiboAPI.GetAuthorID(account.UserName);
var author = WeiboAPI.GetAuthorInfo(account.AuthorID);
var list = WeiboAPI.GetFriendsIDs(account.AuthorID, Enums.SampleMethod.All);
account.FollowCount = 0;
foreach (var id in list)
{
AuthorDBManager.SubscribeAuthor(account.UserName, id);
account.FollowCount++;
}
}
catch (Exception ex)
{
Logger.Error(string.Format("Err@ Account: {0}, message: {1}", account.UserName, ex.ToString()));
account.FailCount++;
if (account.FailCount > DefaultSettings.MaxFailureCount)
{
account.IsDisabled = true;
}
LoginAccountBusiness.UpdateByAccountID(account);
continue;
}
account.FailCount = 0;
account.SubscribeIntervalMins = 15;
account.SubscribeLastTime = Utilities.Epoch;
account.SubscribeNextTime = Utilities.Epoch;
account.SubscribeSinceID = null;
account.FollowCountLastClearTime = DateTime.Now;
account.FollowCountToday = 0;
LoginAccountBusiness.UpdateByAccountID(account);
}
#endregion
#region 关注次数和失效次数清零（每天一次）
DateTime refreshtime = DateTime.Now.Subtract(new TimeSpan(1, 0, 0, 0, 0));
where = string.Format("IsDisabled=FALSE AND SubscribeStatus>=0 AND (FollowCountLastClearTime IS NULL OR FollowCountLastClearTime<'{0}')", refreshtime);
result = LoginAccountBusiness.GetAllByWhere(where, null, LoginAccountMySqlDAL.OrderColumn.Default);
foreach (var account in result)
{
account.FollowCountLastClearTime = DateTime.Now;
account.FollowCountToday = 0;
account.FailCount = 0;
LoginAccountBusiness.UpdateByAccountID(account);
}
#endregion
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
}
}
public string DoOneJob(IPipeline Pipeline)
{
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
Item item = GetNextJob();
if (item != null)
{
try
{
#region 获取最新转发列表
List <NetDimension.Weibo.Entities.status.Entity> result = new List <NetDimension.Weibo.Entities.status.Entity>();
try
{
WeiboAPI.GetRepostOfStatus(item, result);
}
catch (WeiboException ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
}
for (int i = 0; i < result.Count; ++i)
{
var newItem = ItemDBManager.ConvertToItem(result[i], Enums.AuthorSource.TopicTrack, CrawlID);
ItemDBManager.InsertOrUpdateItem(newItem);
}
#endregion
#region 更新转发评论数的历史记录
try
{
var countData = WeiboAPI.GetRepostAndReplyCount(item.ClientItemID);
item.CurrentCount.FetchTime = DateTime.Now;
item.CurrentCount.ForwardCount = countData.Item1;
item.CurrentCount.ReplyCount = countData.Item2;
}
catch (WeiboException ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
}
List <ItemCountData> count = null;
if (item.CountHistory == null)
{
count = new List <ItemCountData>();
}
else
{
count = new List <ItemCountData>(item.CountHistory);
}
count.Add(item.CurrentCount);
item.CountHistory = count.ToArray();
#endregion
item.Tracking_Forward.FollowCount++;
if (WeiboUtilities.ShouldKeepFollow(item))
{
item.Tracking_Forward.FollowStatus = Enums.CrawlStatus.Normal;
}
else
{
item.Tracking_Forward.FollowStatus = Enums.CrawlStatus.Stop;
}
SuccCount++;
continue;
}
catch (Exception ex)
{
item.Tracking_Forward.FollowStatus = Enums.CrawlStatus.Normal;
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
}
finally
{
item.Tracking_Forward.FollowNextTime = DateTime.Now.AddMinutes(DefaultSettings.RepostTrackingInterval.TotalMinutes);
ItemDBManager.PushbackRepostTrackingJob(item);
}
}
}
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
public string DoOneJob(IPipeline Pipeline)
{
SendMsg("暂时不执行回复跟踪任务");
StopFlag = true;
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
Item item = GetNextJob();
if (item != null)
{
try
{
//最新评论列表
List <NetDimension.Weibo.Entities.comment.Entity> result = new List <NetDimension.Weibo.Entities.comment.Entity>();
try
{
WeiboAPI.GetCommentsOfStatus(item, result);
}
catch (WeiboException ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
}
for (int i = 0; i < result.Count; ++i)
{
var reply = ItemReplyDBManager.ConvertToItemReply(result[i]);
ItemReplyDBManager.InsertItemReply(reply);
}
item.Tracking.ReplyCount += result.Count;
item.Tracking.FollowCount++;
if (WeiboUtilities.ShouldKeepFollow(item))
{
item.Tracking.FollowStatus = Enums.CrawlStatus.Normal;
}
else
{
item.Tracking.FollowStatus = Enums.CrawlStatus.Stop;
}
SuccCount++;
continue;
}
catch (WeiboException ex)
{
item.Tracking.FollowStatus = Enums.CrawlStatus.Normal;
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
}
finally
{
item.Tracking.FollowNextTime = DateTime.Now.AddMinutes(DefaultSettings.ReplyTrackingInterval.TotalMinutes);
ItemDBManager.PushbackReplyTrackingJob(item);
}
}
}
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
