/// <summary>
/// 添加一个CBD
/// </summary>
/// <param name="loc"></param>
public static void AddNewCBD(Location loc)
{
loc.LastRefreshTime = Utilities.Epoch;
loc.NextRefreshTime = Utilities.Epoch;
loc.RefreshCount = 0;
LocationDBManager.AddNewLocation(loc);
}
private Location GetNextJob()
{
lock (jobLock)
{
SendMsg("正在获取下一任务");
Location loc = LocationDBManager.GetNextCBDJob();
if (loc == null)
{
SendMsg("没有获取到任务，休息一会");
return(null);
}
loc.RefreshStatus = Enums.CrawlStatus.Crawling;
LocationDBManager.UpdateDB <Location>(loc, "PoID", new string[] { "RefreshStatus" }, MongoDB.Driver.SafeMode.True);
SendMsg(string.Format("获取到{0}的任务", loc.Title));
return(loc);
}
}
public string DoOneJob(IPipeline Pipeline)
{
int SuccCount = 0, ErrCount = 0;
DateTime nextWorkTime = Utilities.Epoch;
while (!StopFlag)
{
if (DateTime.Now > nextWorkTime)
{
Location loc = GetNextJob();
if (loc != null)
{
try
{
DateTime lastTime = loc.LastRefreshTime;
loc.LastRefreshTime = DateTime.Now;
loc.NextRefreshTime = DateTime.Now.AddMinutes(loc.IntervalMins);
//获取CBD周围的人
List <dynamic> users = new List <dynamic>();
List <dynamic> statuses = new List <dynamic>();
try
{
SendMsg(string.Format("正在刷新{0}周围的位置动态", loc.Title));
WeiboAPI.GetUsersNearCBD(loc.Lon, loc.Lat, loc.Radius, Utilities.DateTime2UnixTime(lastTime), Utilities.DateTime2UnixTime(DateTime.Now), loc.LocationSampleMethode, users, statuses);
}
catch (IOException)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
}
catch (WeiboException ex)
{
SendMsg("获取CBD周围动态时发生错误，见日志");
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
Logger.Error(ex.ToString());
}
SendMsg(string.Format("{0}的位置动态获取到{1}条，开始插入数据库", loc.Title, users.Count()));
for (int i = 0; i < users.Count; ++i)
{
if (i >= statuses.Count)
{
break;
}
if (statuses[i] == null || users[i] == null)
{
continue;
}
Item item = ItemDBManager.ConvertToItem(Enums.AuthorSource.LocationScan, CrawlID, statuses[i], users[i]);
if (item.PoID == null)
{
item.PoID = item.PoIDSource = loc.CategoryID;
}
ItemDBManager.InsertOrUpdateItem(item);
Author author = AuthorDBManager.ConvertToAuthor(users[i], Enums.AuthorSource.LocationScan);
AuthorDBManager.InsertOrUpdateAuthorInfo(author);
CntData.Tick();
}
SuccCount++;
SendMsg(string.Format("{0}的任务完成", loc.Title));
continue;
}
catch (Exception ex)
{
ErrCount++;
nextWorkTime = WeiboAPI.rateLimitStatus.ResetTime;
SendMsg(ex.ToString());
Logger.Error(ex.ToString());
}
finally
{
loc.RefreshCount++;
loc.RefreshStatus = Enums.CrawlStatus.Normal;
LocationDBManager.PushbackLoationJob(loc);
}
}
}
SendMsg(string.Format("休息{0}秒", IntervalMS / 1000));
Thread.Sleep(IntervalMS);
}
StopFlag = false;
return(SuccCount == 0 && ErrCount == 0 ? "Nothing to do" : string.Format("OneJob Done. Succ {0} Err {1}", SuccCount, ErrCount));
}
public static void BackToOrigin()
{
LocationDBManager.InitLocationJob();
}
