/// <summary>
/// 根据uid list 分析用户微博数
/// </summary>
/// <param name="Layer"></param>
private void analyzeUidList()
{
string sql = "";
bool flag = true;
CheckID checkID = new CheckID();
checkID.ThreadNo = threadNo;
for (int i = 0; i < list.Count && flag; ++i)
{
if (i % threadCount == threadNo)
{
checkID.SubThreadNo = i;
string uid = list[i];
#if ContinuousOperation
bool isRunOver = false;
#endif
var SinaClient = client;
#region 采用异步方式
SinaClient.AsyncInvoke<NetDimension.Weibo.Entities.user.Entity>(() =>
{
log.Info("线程" + threadNo.ToString() + "：开始:" + DateTime.Now.ToString() + " 抓取用户ID:" + uid);
return SinaClient.API.Entity.Users.Show(uid);
}, (callback) =>
{
if (callback.IsSuccess)
{
string userID = callback.Data.ID;
if (callback.Data.StatusesCount >= 50 && callback.Data.Status != null)
{
string statusID = callback.Data.Status.ID;
if (callback.Data.StatusesCount >= 50 && checkID.checkIDByStatuses(SinaClient, userID, statusID))
{
sql = "UPDATE sinauser SET needtocrawler='1' WHERE uid = '" + uid + "'";
db_SQLServer.SQLServerUnit.getInstance().insert(sql);
}
else
{
sql = "UPDATE sinauser SET needtocrawler='-1' WHERE uid = '" + uid + "'";
db_SQLServer.SQLServerUnit.getInstance().update(sql);
}
}
else
{
sql = "UPDATE sinauser SET needtocrawler='-1' WHERE uid = '" + uid + "'";
db_SQLServer.SQLServerUnit.getInstance().update(sql);
}
log.Info("线程" + threadNo.ToString() + "：开始:" + DateTime.Now.ToString() + " 更新用户ID:" + uid);
}
else
{
#region 报错处理
int ret = getErrorCode(callback);
switch (ret)
{
case 0:
sql = "UPDATE sinauser SET needtocrawler='-1' WHERE uid = '" + uid + "'";
db_SQLServer.SQLServerUnit.getInstance().update(sql);
log.Info("用户 " + uid + "不存在 " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
break;
case 1:
log.Info("频次超过上限......");
Thread.Sleep(65 * 60 * 1000);
break;
case 2:
log.Info("IP请求超过上限......");
Thread.Sleep(65 * 60 * 1000);
break;
case 3: break;
case 4: break;
case 5: break;
case 6: break;
case 7: break;
case 8: break;
default:
db_SQLServer.SQLServerUnit.getInstance().insertLog(threadNo.ToString(), uid, callback.Error.ToString(), callback.Error.Message, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
log.Info("异常{0} :\n[" + uid + "]\n[" + callback.Error + "]\n[" + callback.Error.Message + "]");
break;
}
#endregion
}
#if ContinuousOperation
isRunOver = true;
#endif
});
#endregion
#if ContinuousOperation
//判断异步回调是否结束
while (!isRunOver)
{
System.Threading.Thread.Sleep(200);
}
#endif
}
}
}
/// <summary>
/// 根据随机uid 分析用户微博数
/// </summary>
/// <param name="Layer"></param>
private void analyzeUidRandom()
{
bool flag = true;
CheckID checkID = new CheckID();
while (flag)
{
string uid =CommonLib.GenerateID.getUid();
while (list.Contains(uid))
{
uid = CommonLib.GenerateID.getUid();
}
list.Add(uid);
string sql1 = "INSERT INTO sinauserid(`uid`,`ischeck`) VALUES('" + uid + "','-1') ";
db_MySQL.MySQLUnit.getInstance().insert(sql1);
log.Info(sql1);
var SinaClient = client;
#region 采用异步方式
SinaClient.AsyncInvoke<NetDimension.Weibo.Entities.user.Entity>(() =>
{
log.Info("线程" + threadNo.ToString() + "：开始:" + DateTime.Now.ToString() + " 抓取用户ID:" + uid);
return SinaClient.API.Entity.Users.Show(uid);
}, (callback) =>
{
if (callback.IsSuccess)
{
string userID = callback.Data.ID;
if (callback.Data.StatusesCount >= 50 && callback.Data.Status != null)
{
string statusID = callback.Data.Status.ID;
if (callback.Data.StatusesCount >= 50 && checkID.checkIDByStatuses(SinaClient, userID, statusID))
{
string sql2 = "INSERT INTO sinauserid50(`uid`,`statuses`) VALUES('" + userID + "','" + callback.Data.StatusesCount + "') ";
db_MySQL.MySQLUnit.getInstance().insert(sql2);
log.Info(sql2);
}
}
string sql3 = "UPDATE sinauserid SET ischeck='1' WHERE uid='" + uid + "'";
db_MySQL.MySQLUnit.getInstance().update(sql3);
log.Info(sql3);
}
else
{
int indexOfNotExist = callback.Error.Message.IndexOf("用户不存在");
int indexOfOutOfVisitTimes = callback.Error.Message.IndexOf("频次超过上限");
int indexOfNotInstantiation = callback.Error.Message.IndexOf("未将对象引用设置到对象的实例");
int indexOfOperateTimeOut = callback.Error.Message.IndexOf("操作超时");
int indexOfUnableToConnectToTheRemoteServer = callback.Error.Message.IndexOf("无法连接到远程服务器");
if (indexOfNotExist >= 0)
{
string sql4 = "UPDATE sinauserid SET ischeck='0' WHERE uid='" + uid + "'";
db_MySQL.MySQLUnit.getInstance().update(sql4);
log.Info(sql4);
}
else if (indexOfOutOfVisitTimes >= 0)
{
flag = false;
}
else if (indexOfNotInstantiation >= 0)
{
}
else if (indexOfOperateTimeOut >= 0)
{
flag = false;
}
else if (indexOfUnableToConnectToTheRemoteServer >= 0)
{
flag = false;
}
else
{
log.Info("异常{0} :\n[" + uid + "]\n[" + callback.Error + "]\n[" + callback.Error.Message + "]");
db_MySQL.MySQLUnitParameter.getInstance().insertLog(threadNo.ToString(), uid, callback.Error + "", callback.Error.Message, DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss"));
}
}
});
#endregion
}
}
