public void Crawler()
{
while (ExistCrawlerJC())
{
var listJiaocai = GetRandom10CrawlerJC();
var listArea = GetAreas();
Parallel.ForEach(listJiaocai, (jc) =>
{
try
{
Parallel.ForEach(listArea, (area) =>
{
if (ExistGrabPageAreaSource(area.AreaId, jc.JiaoCaiId))
{
return;
}
var url =
$"http://zujuan.xkw.com/{jc.Prefix}/zj{jc.JiaoCaiId}/a{area.AreaId}/";
Console.WriteLine(url);
var html = HttpWebResponseUtility.ExecuteCreateGetHttpResponseProxy(url, 3000, null);
var doc = NSoupClient.Parse(html);
var totalCount = doc.GetElementById("questioncount").Text().NullToInt();
var pageCount = totalCount / 10 + 1;
AddGrabPageSource(area.AreaId, jc.JiaoCaiId, totalCount);
//for (int i = 1; i <= pageCount; i++)
//{
// AddGrabPageList(area.AreaId, jc.JiaoCaiDetailId, grade.GradeId, totalCount, i);
//}
});
UpdateJiaocaiStatus(jc.JiaoCaiId);
}
catch
{
}
});
}
}
