Пример #1
0
        private void ReceivedData(IAsyncResult ar)
        {
            RequestState   rs        = ar.AsyncState as RequestState;
            HttpWebRequest req       = rs.Req;
            Stream         resStream = rs.ResStream;
            string         url       = rs.Url;
            int            depth     = rs.Depth;
            string         html      = string.Empty;
            int            read      = 0;

            try
            {
                read = resStream.EndRead(ar);
                if (Stop)
                {
                    rs.ResStream.Close();
                    req.Abort();
                    return;
                }
                if (read > 0)
                {
                    MemoryStream ms     = new MemoryStream(rs.Data, 0, read);                                         //创建内存流
                    StreamReader reader = new StreamReader(ms, mEncoding);
                    string       str    = reader.ReadToEnd();                                                         //读取全部字符串
                    rs.Html.Append(str);
                    var result = resStream.BeginRead(rs.Data, 0, rs.BufferSize, new AsyncCallback(ReceivedData), rs); //再次请求数据
                    return;
                }
                html = rs.Html.ToString();
                bool isRedis = Convert.ToBoolean(RedisCommon.GetConfig("IsRedis"));
                if (!isRedis)
                {
                    SaveContents(html, url);//保存到本地
                }
                else
                {
                    mRedisCommon.SetValue(url, html);
                }
                string[] links = GetLinks(html); //获取页面中的链接
                Read.Add(url, depth);            //添加到已经下载集合
                AddUrl(links, depth + 1);
                Unread.Remove(url);              //移除未下载
                mRedisCommon.SetValue(url, url);
                DispatchWork();
            }
            catch (Exception ex)
            {
                //MessageBox.Show(ex.Message);
            }
        }
Пример #2
0
 private void Init()
 {
     mRedisCommon = new RedisCommon();
     AddUrl(new string[] { RootUrl }, 0);
 }