public RequestException(HttpWebResponse response)
: base(string.Format("Request failed: {0} - {1}", response.StatusCode, response.StatusDescription))
{
Response = response;
HttpStatus = response.StatusCode;
HttpStatusDescription = response.StatusDescription;
ResponseText = response.GetResponseText();
}
/// <summary>
/// Post
/// </summary>
/// <typeparam name="TP">自定义类型</typeparam>
/// <param name="url">url</param>
/// <param name="method">请求方式，暂时只支持Post。Get请调用其他</param>
/// <param name="data">数据，类型为UserType，会自动序列化作为json将数据Post出去</param>
/// <param name="contentType">RequestHeader:contentType</param>
/// <returns>返回反序列化后的实例</returns>
public static string PostJsonAndResponse <TP>(string url, TP data, out string jsonContent, IDictionary <string, string> headers = null)
{
string content = string.Empty;
content = JsonUtility.ToJson(data);
jsonContent = content;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
try
{
request.Method = "POST";
request.Headers.Add("Accept-Encoding", "gzip");
request.ContentType = "application/json";
request.AllowAutoRedirect = true;
request.Proxy = null;
if (headers != null)
{
request.SetRequestHeader(headers);
}
byte[] byteData;
byteData = Encoding.GetEncoding("UTF-8").GetBytes(content);
request.ContentType += "; charset=utf-8";
using (Stream stream = request.GetRequestStream())
{
stream.Write(byteData, 0, (int)byteData.Length);
stream.Close();
// 记录请求开始后到获得响应所耗时间(不包括数据传输)并重置计时器
var counter = new Stopwatch();
counter.Start();
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
counter.Stop();
var requestCost = counter.ElapsedMilliseconds;
counter.Reset();
// 记录数据传输与本地处理数据所耗时间
counter.Start();
string text = response.GetResponseText();
counter.Stop();
var processCost = counter.ElapsedMilliseconds;
if (requestCost + processCost >= 1000)
{
LogRequestCostTime(url, requestCost, processCost);
}
return(text);
}
}
}
finally
{
TryCatch(() =>
{
request.Abort();
});
}
}
/// <summary>
/// Post
/// </summary>
/// <typeparam name="TP">自定义类型</typeparam>
/// <param name="url">url</param>
/// <param name="method">请求方式，暂时只支持Post。Get请调用其他</param>
/// <param name="data">数据</param>
/// <returns>返回反序列化后的实例</returns>
public static TR PostFormDataAndResponse <TR, TP>(string url, TP data, IDictionary <string, string> headers = null)
{
TR result = default(TR);
string content = UrlHelper.ObjectToUrlParameter(data);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
try
{
request.Method = "POST";
request.Headers.Add("Accept-Encoding", "gzip");
request.ContentType = "application/x-www-form-urlencoded";
request.AllowAutoRedirect = true;
request.Proxy = null;
if (headers != null)
{
request.SetRequestHeader(headers);
}
byte[] byteData;
byteData = Encoding.GetEncoding("UTF-8").GetBytes(content);
request.ContentType += "; charset=utf-8";
using (Stream stream = request.GetRequestStream())
{
stream.Write(byteData, 0, (int)byteData.Length);
stream.Close();
// 记录请求开始后到获得响应所耗时间(不包括数据传输)并重置计时器
var counter = new Stopwatch();
counter.Start();
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
counter.Stop();
var requestCost = counter.ElapsedMilliseconds;
counter.Reset();
// 记录数据传输与本地处理数据所耗时间
counter.Start();
string json = response.GetResponseText();
counter.Stop();
var processCost = counter.ElapsedMilliseconds;
if (requestCost + processCost >= 1000)
{
LogRequestCostTime(url, requestCost, processCost);
}
result = JsonUtility.FromJsonTo <TR>(json);
return(result);
}
}
}
finally
{
TryCatch(() =>
{
request.Abort();
});
}
}
public static string GetAndResponseText(string url)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
request.Headers.Add("Accept-Encoding", "gzip");
request.AllowAutoRedirect = true;
request.Proxy = null;
HttpWebResponse response = null;
string returnStr = string.Empty;
try
{
// 记录请求开始后到获得响应所耗时间(不包括数据传输)并重置计时器
var counter = new Stopwatch();
counter.Start();
response = (HttpWebResponse)request.GetResponse();
counter.Stop();
var requestCost = counter.ElapsedMilliseconds;
counter.Reset();
// 记录数据传输与本地处理数据所耗时间
counter.Start();
returnStr = response.GetResponseText();
counter.Stop();
var processCost = counter.ElapsedMilliseconds;
if (requestCost + processCost >= 1000)
{
LogRequestCostTime(url, requestCost, processCost);
}
response.Close();
}
catch// (Exception ex)
{
//获取失败
}
finally
{
response = null;
TryCatch(() =>
{
request.Abort();
});
}
return(returnStr);
}
private static string GetResponseMessage(HttpWebResponse response)
{
if (response.IsSuccessful())
{
return(null);
}
int statusCode = (int)response.StatusCode;
string responseText = response.GetResponseText();
string message = statusCode == 404 ? "404 Page not found." : responseText.Length < 500 ? responseText : "";
if (responseText.Trim().StartsWith("{"))
{
try {
var responseJson = JObject.Parse(responseText);
message = responseJson["message"].Value <string>();
} catch { }
}
return(message);
}
/// <summary>
///
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static T GetAndResponse <T>(string url)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
try
{
request.Method = "GET";
request.Headers.Add("Accept-Encoding", "gzip");
request.AllowAutoRedirect = true;
request.Proxy = null;
// 记录请求开始后到获得响应所耗时间(不包括数据传输)并重置计时器
var counter = new Stopwatch();
counter.Start();
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
counter.Stop();
var requestCost = counter.ElapsedMilliseconds;
counter.Reset();
T result = default(T);
// 记录数据传输与本地处理数据所耗时间
counter.Start();
string json = response.GetResponseText();
counter.Stop();
var processCost = counter.ElapsedMilliseconds;
if (requestCost + processCost >= 1000)
{
LogRequestCostTime(url, requestCost, processCost);
}
result = JsonUtility.FromJsonTo <T>(json);
return(result);
}
}
finally
{
TryCatch(() =>
{
request.Abort();
});
}
}
public static HttpResponse ProcessRequest(HttpRequest Request, ICredentials Credentials)
{
if (string.IsNullOrEmpty(Request.URL))
{
throw new Exception("WebScraperComponent Error: URL cannot be null.");
}
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(Request.URL);
if (Credentials != null)
{
webRequest.Credentials = Credentials;
}
CookieContainer cookieContainer = new CookieContainer();
webRequest.CookieContainer = cookieContainer;
if (!string.IsNullOrEmpty(Request.Cookie))
{
webRequest.CookieContainer.SetCookies(new System.Uri(Request.URL), Request.Cookie);
}
if ((Request.Cookies ?? new StringDictionary()).Count > 0)
{
foreach (string key in Request.Cookies.Keys)
{
webRequest.CookieContainer.SetCookies(new System.Uri(Request.URL), string.Format("{0}={1};", key.ToUpper(), Request.Cookies[key]));
}
}
webRequest.Headers.Add("charset", "UTF-8");
Dictionary <string, string> headers = Request.Headers ?? new Dictionary <string, string>();
foreach (string key in headers.Keys)
{
webRequest.Headers.Add(key, headers[key]);
}
if (Request.Method == "POST")
{
webRequest.Method = "POST";
webRequest.ContentType = "application/x-www-form-urlencoded";
byte[] postData = System.Text.Encoding.UTF8.GetBytes(Request.PostData);
webRequest.ContentLength = postData.Length;
Stream dataStream = webRequest.GetRequestStream();
dataStream.Write(postData, 0, postData.Length);
dataStream.Close();
}
else
{
webRequest.Method = "GET";
}
webRequest.KeepAlive = false;
webRequest.Accept = "*/*";
webRequest.AllowAutoRedirect = false;
if (!webRequest.UserAgent.ContainsText())
{
webRequest.UserAgent = "WebScraper 1.0 Mozilla/4.0 (compatible; MSIE 6.0; Windows)";
}
if (Request.ContentType != string.Empty && webRequest.ContentType == null)
{
webRequest.ContentType = Request.ContentType;
}
using (HttpWebResponse response = (HttpWebResponse)webRequest.GetResponse())
{
HttpResponse result = new HttpResponse();
result.HttpScraperUrl = webRequest.RequestUri.AbsoluteUri;
result.ResponseCode = ((int)response.StatusCode).ToString();
result.ResponseText = response.GetResponseText();
result.Headers = new Dictionary <string, string>();
foreach (string key in response.Headers.AllKeys)
{
result.Headers.Add(key, response.Headers.Get(key));
}
result.Cookies = new Dictionary <string, string>();
foreach (Cookie cookie in response.Cookies)
{
result.Cookies.Add(cookie.Name, cookie.Value);
}
response.Close();
return(result);
}
}
public static string ReadResponseText(HttpWebResponse webResponse, Encoding encoding)
{
return(webResponse.GetResponseText(encoding));
}
public static string ReadResponseText(HttpWebResponse webResponse)
{
return(webResponse.GetResponseText());
}
