public static T Request <T>(this HttpWebRequestUtility requestUtility)
{
var result = requestUtility.Request();
var data = Newtonsoft.Json.JsonConvert.DeserializeObject <T>(result);
return(data);
}
internal virtual string Request(HttpWebRequestOption option, int requestType)
{
var paramters = CreateHttpRequest(requestType);
HttpWebRequestUtility httpWebRequestUtility =
new HttpWebRequestUtility(option, paramters);
var result = httpWebRequestUtility.Request();
return(result);
}
public void Crawler()
{
foreach (var category in GetSubjectCategories())
{
var res = HttpWebRequestUtility.Post("http://zujuan.xkw.com/xxyw/zj108223/?action=getcategory", $"cid={category.CategoryId}");
var doc = NSoupClient.Parse(res.ToString());
var a = doc.GetElementsByTag("a");
foreach (var item in a)
{
var jiaocaidId = item.Attr("gradeid").NullToInt();
var jiaocaidName = item.Attr("title").NullToString();
var url = $"http://zujuan.xkw.com/{item.Attr("href").NullToString()}";
AddJiaocai(category.CategoryId, jiaocaidId, jiaocaidName, url);
}
}
}
