Пример #1
0
        /// <summary>
        /// 根据用户 id list 抓取用户微博
        /// </summary>
        /// <param name="Layer"></param>
        private void crawlerUserByUserID()
        {
            bool flag = true;
            for (int i = 0; i < list.Count && flag; ++i)
            {
                if (i % threadCount == threadNo)
                {

                    string uid = list[i];

#if ContinuousOperation
                    bool isRunOver = false;
#endif

                    var SinaClient = client;
                    #region 采用异步方式
                    SinaClient.AsyncInvoke<NetDimension.Weibo.Entities.user.Entity>(() =>
                    {
                        log.Info("线程" + threadNo.ToString() + ":开始:" + DateTime.Now.ToString() + "  分析用户:" + uid);
                        return SinaClient.API.Entity.Users.Show(uid);
                    }, (callback) =>
                    {
                        if (callback.IsSuccess)
                        {
                            var user = callback.Data;
                            string userID = user.ID;

                            log.Info("线程" + threadNo.ToString() + ":开始:" + DateTime.Now.ToString() + "  抓取用户:" + uid);                           

                            SinaDal.UserAction action = new SinaDal.UserAction();
                            
                            action.ThreadNo = threadNo;
                            action.SubThreadNo = i;
                            action.operation(SinaClient, user);

                        }
                        else
                        {
                            #region 报错处理
                            int ret = getErrorCode(callback);
                            switch (ret)
                            {
                                case 0:
                                    log.Info("用户 " + uid + "不存在    " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                                    break;
                                case 1:
                                    log.Info("频次超过上限......");
                                    Thread.Sleep(65 * 60 * 1000);
                                    break;
                                case 2:
                                    log.Info("IP请求超过上限......");
                                    Thread.Sleep(65 * 60 * 1000);
                                    break;
                                case 3: break;
                                case 4: break;
                                case 5: break;
                                case 6: break;
                                case 7: break;
                                case 8: break;
                                default:
                                    db_SQLServer.SQLServerUnit.getInstance().insertLog(threadNo.ToString(), uid, callback.Error.ToString(), callback.Error.Message, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
                                    log.Info("异常{0} :\n[" + uid + "]\n[" + callback.Error + "]\n[" + callback.Error.Message + "]");
                                    break;
                            }
                            #endregion
                        }

#if ContinuousOperation
                        isRunOver = true;
#endif

                    });
                    #endregion

#if ContinuousOperation
                    //判断异步回调是否结束
                    while (!isRunOver)
                    {
                        System.Threading.Thread.Sleep(200);
                    }
#endif

                }
            }
        }
Пример #2
0
        /// <summary>
        /// 根据用户 昵称list 抓取用户微博
        /// </summary>
        /// <param name="Layer"></param>
        private void crawlerUserByScreenName()
        {
            bool flag = true;
            TopList tp = new TopList();
            for (int i = 0; i < list.Count && flag; ++i)
            {
                if (i % threadCount == threadNo)
                {
                    string[] strs = list[i].Split(' ');
                    string screen_name = strs[0];
                    string uid = strs[1];
                    //根据用户ID获取用户昵称
                    string name = tp.getUserName(uid);
                    if (name != null)
                    {
                        screen_name = name;
                    }

                    #if ContinuousOperation
                    bool isRunOver = false;
                    #endif

                    var SinaClient = client;
                    #region 采用异步方式
                    SinaClient.AsyncInvoke<NetDimension.Weibo.Entities.user.Entity>(() =>
                    {
                        log.Info("线程" + threadNo.ToString() + ":开始:" + DateTime.Now.ToString() + "  分析用户:" + screen_name);
                        return SinaClient.API.Entity.Users.Show("", screen_name);
                    }, (callback) =>
                    {
                        if (callback.IsSuccess)
                        {
                            var user = callback.Data;
                            string userID = user.ID;

                            log.Info("线程" + threadNo.ToString() + ":开始:" + DateTime.Now.ToString() + "  抓取用户:" + screen_name);
                            SinaDal.UserAction action = new SinaDal.UserAction();
                            action.ThreadNo = threadNo;
                            action.SubThreadNo = i;
                            action.operation(SinaClient, user);
                        }
                        else
                        {                        
                            #region 报错处理                  
                            int ret = getErrorCode(callback);
                            switch (ret)
                            {
                                case 0:
                                    log.Info("用户 " + uid + "不存在    " + DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss"));
                                    break;
                                case 1:
                                    log.Info("频次超过上限......");
                                    Thread.Sleep(65 * 60 * 1000);
                                    break;
                                case 2:
                                    log.Info("IP请求超过上限......");
                                    Thread.Sleep(65 * 60 * 1000);
                                    break;
                                case 3: break;
                                case 4: break;
                                case 5: break;
                                default:
                                    log.Info("异常{0} :\n[" + uid + "]\n[" + callback.Error + "]\n[" + callback.Error.Message + "]");
                                    break;
                            }
                            #endregion
                        }

                        #if ContinuousOperation
                        isRunOver = true;
                        #endif

                    });
                    #endregion

                    #if ContinuousOperation
                    //判断异步回调是否结束
                    while (!isRunOver)
                    {
                        System.Threading.Thread.Sleep(200);
                    }
                    #endif

                }
            }
        }