/// <summary>
/// 返回请求数据
/// </summary>
/// <param name="curUrlObj"></param>
/// <returns></returns>
public SimpleCrawler.HttpResult GetHttpHtml(string url)
{
url = FixUrlSignStr(url);//修饰字符串
// return GetPostDataFix(curUrlObj, accessToken);
SimpleCrawler.HttpResult result = new SimpleCrawler.HttpResult();
try
{
var item = new SimpleCrawler.HttpItem()
{
URL = url,
Method = "get",//URL 可选项 默认为Get
// ContentType = "text/html",//返回类型 可选项有默认值
UserAgent = "okhttp/3.2.0",
ContentType = "application/x-www-form-urlencoded",
};
// item.Header.Add("Content-Type", "application/x-www-form-urlencoded");
// hi.HeaderSet("Content-Length","154");
// hi.HeaderSet("Connection","Keep-Alive");
item.Header.Add("Accept-Encoding", "gzip");
item.Header.Add("Authorization", curDeviceInfo.accessToken);
item.Cookie = globalCookie;
if (USEWEBPROXY)
{
item.WebProxy = webProxy;
}
result = http.GetHtml(item);
}
catch (WebException ex)
{
}
catch (TimeoutException ex)
{
}
catch (Exception ex)
{
}
return(result);
}
public HttpResult GetPostData(UrlInfo curUrlObj)
{
//创建Httphelper参数对象
SimpleCrawler.HttpItem item = new SimpleCrawler.HttpItem()
{
URL = curUrlObj.UrlString, //URL 必需项
ContentType = "application/x-www-form-urlencoded", //返回类型 可选项有默认值
Timeout = 1500,
Accept = "*/*",
// Encoding = null,//编码格式（utf-8,gb2312,gbk） 可选项 默认类会自动识别
//Encoding = Encoding.Default,
Method = "post", //URL 可选项 默认为Get
//Timeout = 100000,//连接超时时间 可选项默认为100000
//ReadWriteTimeout = 30000,//写入Post数据超时时间 可选项默认为30000
//IsToLower = false,//得到的HTML代码是否转成小写 可选项默认转小写
//Cookie = "",//字符串Cookie 可选项
UserAgent = "okhttp/3.2.0", //用户的浏览器类型，版本，操作系统 可选项有默认值
//Referer = "app.qichacha.net",//来源URL 可选项
Postdata = curUrlObj.PostData,
// Allowautoredirect = true,
// Cookie = Settings.SimulateCookies
};
item.Header.Add("Accept-Encoding", "gzip");
// item.Header.Add("Host", "app.qichacha.net");
item.Header.Add("Authorization", curDeviceInfo.accessToken);
//item.Header.Add("Accept-Language", "zh-CN");
item.Header.Add("charset", "UTF-8");
//item.Header.Add("X-Requested-With", "XMLHttpRequest");
//请求的返回值对象
item.WebProxy = GetWebProxy();
var result = http.GetHtml(item);
return(result);
}