private void ReceivedData(IAsyncResult ar)
{
RequestState rs = ar.AsyncState as RequestState;
HttpWebRequest req = rs.Req;
Stream resStream = rs.ResStream;
string url = rs.Url;
int depth = rs.Depth;
string html = string.Empty;
int read = 0;
try
{
read = resStream.EndRead(ar);
if (Stop)
{
rs.ResStream.Close();
req.Abort();
return;
}
if (read > 0)
{
MemoryStream ms = new MemoryStream(rs.Data, 0, read); //创建内存流
StreamReader reader = new StreamReader(ms, mEncoding);
string str = reader.ReadToEnd(); //读取全部字符串
rs.Html.Append(str);
var result = resStream.BeginRead(rs.Data, 0, rs.BufferSize, new AsyncCallback(ReceivedData), rs); //再次请求数据
return;
}
html = rs.Html.ToString();
bool isRedis = Convert.ToBoolean(RedisCommon.GetConfig("IsRedis"));
if (!isRedis)
{
SaveContents(html, url);//保存到本地
}
else
{
mRedisCommon.SetValue(url, html);
}
string[] links = GetLinks(html); //获取页面中的链接
Read.Add(url, depth); //添加到已经下载集合
AddUrl(links, depth + 1);
Unread.Remove(url); //移除未下载
mRedisCommon.SetValue(url, url);
DispatchWork();
}
catch (Exception ex)
{
//MessageBox.Show(ex.Message);
}
}
private void Init()
{
mRedisCommon = new RedisCommon();
AddUrl(new string[] { RootUrl }, 0);
}