public string GetWebRequestResponse(ScrapeRequest scrapeRequest)
{
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(scrapeRequest.Url);
SetRequestCookies(request, scrapeRequest);
SetRequestHeaders(request, scrapeRequest);
SetRequestMeta(request, scrapeRequest);
if (scrapeRequest.RequestType.ToUpper() == "POST")
{
//scrapeRequest.PostContent = HttpUtility.UrlEncode(scrapeRequest.PostContent);
request.ContentType = "application/x-www-form-urlencoded";
byte[] contentData = Encoding.ASCII.GetBytes(scrapeRequest.PostContent);
request.ContentLength = contentData.Length;
Stream contentWriter = request.GetRequestStream();
contentWriter.Write(contentData, 0, contentData.Length);
contentWriter.Close();
}
else
{
request.ContentType = "text/html, charset=utf-8"; //application/json; charset=utf-8
}
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response == null)
{
return(null);
}
return(readResponseStream(response));
}
void SetRequestHeaders(HttpWebRequest request, ScrapeRequest sr)
{
if (sr.WebConfig.Headers != null && sr.WebConfig.Headers.Count > 0)
{
WebHeaderCollection whc = request.Headers;
foreach (var h in sr.WebConfig.Headers)
{
if (WebHeaderCollection.IsRestricted(h.Key))
{
continue;
}
else if (!request.Headers.AllKeys.Equals(h.Key))
{
request.Headers.Add(h.Key, h.Value);
}
else
{
request.Headers[h.Key] = h.Value;
}
}
}
else
{
request.Headers.Add("Accept-Language", "en-US, en;q=0.8");
request.Headers.Add("Cache-Control", "no-chche");
request.Headers.Add("Accept-Encoding", "gzip, deflate");
}
}
void SetRequestMeta(HttpWebRequest request, ScrapeRequest sr)
{
request.Timeout = (sr.WebConfig.TimeOut != 0) ? sr.WebConfig.TimeOut : 90000;
request.Method = sr.RequestType;
if (!string.IsNullOrEmpty(sr.WebConfig.Accept))
{
request.Accept = sr.WebConfig.Accept;
}
request.UserAgent = (!string.IsNullOrEmpty(sr.WebConfig.UserAgent)) ? sr.WebConfig.UserAgent : _userAgent;
}
void SetRequestCookies(HttpWebRequest request, ScrapeRequest sr)
{
request.CookieContainer = _cookies;
if (sr.WebConfig.Cookies != null && sr.WebConfig.Cookies.Count > 0)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(sr.WebConfig.Cookies);
_cookies = request.CookieContainer;
}
}
public string GetWebClientResponse(ScrapeRequest Request)
{
string result = string.Empty;
NameValueCollection postContent = new NameValueCollection();
try
{
if (Request.WebConfig.PostParams != null && Request.WebConfig.PostParams.Count > 0)
{
foreach (var param in Request.WebConfig.PostParams)
{
postContent.Add(param.Key, HttpUtility.UrlEncode(param.Value));
}
}
using (WebClient client = new WebClient())
{
if (Request.WebConfig.Headers != null)
{
foreach (var h in Request.WebConfig.Headers)
{
client.Headers.Add(h.Key, h.Value);
}
}
if (!string.IsNullOrEmpty(Request.WebConfig.CustomCookies))
{
client.Headers.Add(HttpRequestHeader.Cookie, Request.WebConfig.CustomCookies);
}
if (Request.RequestType == "POST")
{
byte[] response = client.UploadValues(Request.Url, Request.RequestType, postContent);
result = Encoding.UTF8.GetString(response);
}
else
{
result = client.DownloadString(Request.Url);
result = Encoding.UTF8.GetString(Encoding.Default.GetBytes(result));
}
}
}
catch (Exception)
{
}
return(result);
}