예제 #1
0
        static void Main(string[] args)
        {
            /*string content = "\u64CD\u4F5C\u6210\u529F";
            string result = Uri.UnescapeDataString(content);
            Console.WriteLine(result);//操作成功*/
            //

            TopList tp = new TopList();

            List<String> list = new List<string>();
            string[] types = { "kejiyenei", "ITchengxuyuan", "kejiqiyegaoguan", "kejiqitaqita" };
            //yule_wangluohongren
            //yanchuhuodong
            //yule_yulegaoguan
            //yuleqita
            foreach (string type in types)
            {
                list.AddRange(tp.getITTopUserList(0, type));
            }
            
            foreach (string str in list)
            {
                //log.Info(str);
            }
            Console.ReadLine();
            //string shtml = Util.getJsonPost("http://huati.weibo.com/aj_topic/list", "http://huati.weibo.com/883375?from=home_content_topic&type=ori&filter=mining", "_pv=1&keyword=马航飞机失联&topicName=马航飞机失联&ori=0&hasv=0&atten=0&match_area=0&mining=1&istag=2&is_olympic=0&_t=0&__rnd=1395296116000");
            /*string shtml = Util.getJsonGet("http://huati.weibo.com/aj_topic/list", "http://huati.weibo.com/883375?from=home_content_topic&type=ori&filter=mining", "_pv=1&keyword=马航飞机失联&topicName=马航飞机失联&ori=0&hasv=0&atten=0&match_area=0&mining=1&istag=2&is_olympic=0&_t=0&__rnd=1395296116000");
            JObject jo = JObject.Parse(shtml);
            JToken msg = jo["msg"];
            Console.WriteLine("MSG: " + msg.ToString());
            JToken code = jo["code"];
            //Console.WriteLine(code.ToString());
            JToken data = jo["data"];
            jo = JObject.Parse(data.ToString());
            JToken html = jo["html"];
            string htmlstring = html.ToString();
            Console.Write(htmlstring.Substring(0, 100));*/
        }
예제 #2
0
파일: Task.cs 프로젝트: limingyao/Crawler
        /// <summary>
        /// 根据用户 昵称list 抓取用户微博
        /// </summary>
        /// <param name="Layer"></param>
        private void crawlerUserByScreenName()
        {
            bool flag = true;
            TopList tp = new TopList();
            for (int i = 0; i < list.Count && flag; ++i)
            {
                if (i % threadCount == threadNo)
                {
                    string[] strs = list[i].Split(' ');
                    string screen_name = strs[0];
                    string uid = strs[1];
                    //根据用户ID获取用户昵称
                    string name = tp.getUserName(uid);
                    if (name != null)
                    {
                        screen_name = name;
                    }

                    #if ContinuousOperation
                    bool isRunOver = false;
                    #endif

                    var SinaClient = client;
                    #region 采用异步方式
                    SinaClient.AsyncInvoke<NetDimension.Weibo.Entities.user.Entity>(() =>
                    {
                        log.Info("线程" + threadNo.ToString() + ":开始:" + DateTime.Now.ToString() + "  分析用户:" + screen_name);
                        return SinaClient.API.Entity.Users.Show("", screen_name);
                    }, (callback) =>
                    {
                        if (callback.IsSuccess)
                        {
                            var user = callback.Data;
                            string userID = user.ID;

                            log.Info("线程" + threadNo.ToString() + ":开始:" + DateTime.Now.ToString() + "  抓取用户:" + screen_name);
                            SinaDal.UserAction action = new SinaDal.UserAction();
                            action.ThreadNo = threadNo;
                            action.SubThreadNo = i;
                            action.operation(SinaClient, user);
                        }
                        else
                        {                        
                            #region 报错处理                  
                            int ret = getErrorCode(callback);
                            switch (ret)
                            {
                                case 0:
                                    log.Info("用户 " + uid + "不存在    " + DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss"));
                                    break;
                                case 1:
                                    log.Info("频次超过上限......");
                                    Thread.Sleep(65 * 60 * 1000);
                                    break;
                                case 2:
                                    log.Info("IP请求超过上限......");
                                    Thread.Sleep(65 * 60 * 1000);
                                    break;
                                case 3: break;
                                case 4: break;
                                case 5: break;
                                default:
                                    log.Info("异常{0} :\n[" + uid + "]\n[" + callback.Error + "]\n[" + callback.Error.Message + "]");
                                    break;
                            }
                            #endregion
                        }

                        #if ContinuousOperation
                        isRunOver = true;
                        #endif

                    });
                    #endregion

                    #if ContinuousOperation
                    //判断异步回调是否结束
                    while (!isRunOver)
                    {
                        System.Threading.Thread.Sleep(200);
                    }
                    #endif

                }
            }
        }