private List <string> BaiduSearch(string baseKeyword, string userAgent)
{
string suggestionUrl = "http://suggestion.baidu.com/su?wd=" + baseKeyword + "&cb=window.bdsug.sug&from=superpage&t=1335581987353";
List <string> result = null;
HttpHelper.HttpParam hp = new HttpHelper.HttpParam()
{
URL = suggestionUrl,
UserAgent = userAgent,
ContentType = "application/json;charset=unicode",
Timeout = TimeSpan.FromSeconds(1),
Method = HttpHelper.HttpVerb.Get,
AllowAutoRedirect = false,
Accept = "*/*",
Referer = "http://www.baidu.com/",
};
HttpHelper.HttpResult hr = HttpHelper.Request(hp);
if (hr.Result != null)
{
var sr = new StreamReader(hr.Result, Encoding.Default);
if (sr != null)
{
string html = sr.ReadToEnd();
if (!string.IsNullOrEmpty(html))
{
ParserBaiduSearchResult(html, out result);
}
}
hr.Result.Close();
}
return(result);
}
private TaskGroup_FileCheck_OneData ScanRequestGetBody(string url, string userAgent)
{
Stopwatch st = new Stopwatch();
st.Start();
TaskGroup_FileCheck_OneData result = new TaskGroup_FileCheck_OneData(url);
HttpHelper.HttpParam hp = new HttpHelper.HttpParam()
{
URL = url,
UserAgent = userAgent,
ContentType = "application/json;charset=unicode",
Timeout = TimeSpan.FromSeconds(1),
Method = HttpHelper.HttpVerb.Get,
AllowAutoRedirect = false,
Accept = "*/*",
Referer = "http://www.baidu.com/",
};
HttpHelper.HttpResult hr = HttpHelper.Request(hp);
if (hr.Result != null)
{
result.contentType = hr.ContentType;
result.length = hr.ContentLength;
result.server = hr.Server;
result.code = (int)hr.StatusCode;
result.powerBy = hr.Header.Get("X-Powered-By");
}
result.time = st.ElapsedMilliseconds;
return(result);
}
private List <string> BaiduSearch(string keyword, string userAgent)
{
string url = "http://www.baidu.com/s?ie=utf-8&wd=" + keyword + "&rn=50";
//GlobalVar.Instance.logger.Debug("Ready to search : " + url);
List <string> result = null;
HttpHelper.HttpParam hp = new HttpHelper.HttpParam()
{
URL = url,
UserAgent = userAgent,
ContentType = "application/json;charset=unicode",
Timeout = TimeSpan.FromSeconds(1),
Method = HttpHelper.HttpVerb.Get,
AllowAutoRedirect = false,
Accept = "*/*",
Referer = "http://www.baidu.com/",
};
HttpHelper.HttpResult hr = HttpHelper.Request(hp);
if (hr.Result != null)
{
var sr = new StreamReader(hr.Result, Encoding.Default);
if (sr != null)
{
string html = sr.ReadToEnd();
if (!string.IsNullOrEmpty(html))
{
//GlobalVar.Instance.logger.Debug("HTML length = " + html.Length);
ParserBaiduSearchResult(html, out result);
}
}
hr.Result.Close();
}
return(result);
}
private TaskGroup_FileCheck_OneData ScanRequestGetHeader(string url)
{
Stopwatch st = new Stopwatch();
st.Start();
TaskGroup_FileCheck_OneData result = new TaskGroup_FileCheck_OneData(url);
HttpHelper.HttpResult hr = HttpHelper.HeadInfo(url);
if (hr.Result != null && hr.IsCompleted)
{
result.contentType = hr.ContentType;
result.length = hr.ContentLength;
result.server = hr.Server;
result.code = (int)hr.StatusCode;
result.powerBy = hr.Header.Get("X-Powered-By");
}
result.time = st.ElapsedMilliseconds;
return(result);
}