public HttpState(HttpMode mode, char sourcePort, uint destinationIP)
{
this.HttpTransaction.ConnectionStartTime = DateTime.Now;
this.HttpTransaction.SourcePort = sourcePort;
this.HttpTransaction.DestinationIP = destinationIP;
this.HttpTransaction.Mode = mode;
}
/// <summary>
/// Creates an instance of the HTTP implementation represented by the given <see cref="HttpMode" />
/// </summary>
/// <param name="mode">the HTTP implementation to get</param>
/// <returns>a HTTP implementation</returns>
public static Http Create(HttpMode mode = HttpMode.Dual)
{
if (mode == HttpMode.Legacy)
{
return(new HttpOne());
}
return(new HttpTwo());
}
/// <summary>
/// 将 HttpWebResponse 转为 HttpResult
/// </summary>
/// <param name="response"></param>
/// <param name="mode"></param>
/// <returns></returns>
public static HttpResult GetHttpResult(HttpWebResponse response, HttpMode mode)
{
HttpResult httpResult = new HttpResult();
httpResult.StatusCode = response.StatusCode;
WebHeaderCollection webHeaderCollection = response.Headers;
//获得响应头
ResponseHeaders responseHeaders = new ResponseHeaders()
{
Location = webHeaderCollection.Get("Location"),
Date = webHeaderCollection.Get("Date"),
ContentType = webHeaderCollection.Get("Content-Type"),
Connection = webHeaderCollection.Get("Connection"),
CacheControl = webHeaderCollection.Get("Cache-Control"),
SetCookie = webHeaderCollection.Get("Set-Cookie"),
};
double.TryParse(webHeaderCollection.Get("Content-Length"), out responseHeaders.ContentLength);
httpResult.Headers = responseHeaders;
if (response.StatusCode == HttpStatusCode.OK)
{
if (mode == HttpMode.Normal)
{
using (StreamReader sr = new StreamReader(response.GetResponseStream()))
{
httpResult.SourceCode = sr.ReadToEnd();
}
if (responseHeaders.ContentLength == 0)
{
responseHeaders.ContentLength = httpResult.SourceCode.Length;
}
}
else if (mode == HttpMode.Stream)
{
using (MemoryStream ms = new MemoryStream())
{
response.GetResponseStream().CopyTo(ms);
httpResult.FileByte = ms.ToArray();
}
if (responseHeaders.ContentLength == 0)
{
responseHeaders.ContentLength = httpResult.FileByte.Length;
}
}
}
return(httpResult);
}
/// <summary>
/// Internal Helper Method which executes the HTTP Requests against the Sparql Endpoint.
/// </summary>
/// <param name="target">Uri to make Request to.</param>
/// <param name="postData">Data that is to be POSTed to the Endpoint in <strong>application/x-www-form-urlencoded</strong> format.</param>
/// <param name="accept">The Accept Header that should be used.</param>
/// <returns>HTTP Response.</returns>
private HttpWebResponse ExecuteQuery(Uri target, String postData, String accept)
{
// Expect errors in this function to be handled by the calling function
// Set-up the Request
HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(target);
// Use HTTP GET/POST according to user set preference
httpRequest.Accept = accept;
if (!postData.Equals(string.Empty))
{
httpRequest.Method = "POST";
httpRequest.ContentType = MimeTypesHelper.Utf8WWWFormURLEncoded;
using (var writer = new StreamWriter(httpRequest.GetRequestStream(), new UTF8Encoding(Options.UseBomForUtf8)))
{
writer.Write(postData);
writer.Close();
}
}
else
{
if (HttpMode.Equals("AUTO"))
{
httpRequest.Method = postData.Equals(string.Empty) ? "GET" : "POST";
}
else
{
httpRequest.Method = HttpMode;
}
}
ApplyRequestOptions(httpRequest);
Tools.HttpDebugRequest(httpRequest);
HttpWebResponse httpResponse = (HttpWebResponse)httpRequest.GetResponse();
Tools.HttpDebugResponse(httpResponse);
return(httpResponse);
}
public static async Task <(string, int)> Http(string Url, string Method = "GET", HttpMode Mode = HttpMode.Normal, WebProxy Proxy = null, string Cookie = "", bool allowRedirect = true)
{
string HtmlText = "";
int StatusCode = 404;
int num = 0;
while (num < ATTEMPTNUM & string.IsNullOrEmpty(HtmlText))
{
try
{
HtmlText = await Task.Run(() =>
{
string result = "";
HttpWebRequest Request;
HttpWebResponse Response = default;
try
{
Request = (HttpWebRequest)HttpWebRequest.Create(Url);
if (Cookie != "")
{
Request.Headers.Add("Cookie", Cookie);
}
Request.Accept = "*/*";
Request.Timeout = 50000;
Request.Method = Method;
Request.KeepAlive = false;
Request.AllowAutoRedirect = allowRedirect;
if (Mode == HttpMode.RedirectGet)
{
Request.AllowAutoRedirect = false;
}
Request.Referer = Url;
Request.UserAgent = UserAgent;
Request.Headers.Add("Accept-Language", "zh-CN,zh;q=0.9");
Request.ReadWriteTimeout = READWRITETIMEOUT;
if (Proxy != null)
{
Request.Proxy = Proxy;
}
Response = (HttpWebResponse)Request.GetResponse();
StatusCode = (int)Response.StatusCode;
if (Response.StatusCode == HttpStatusCode.OK)
{
var SR = new StreamReader(Response.GetResponseStream());
result = SR.ReadToEnd();
SR.Close();
StatusCode = 200;
}
else if (Response.StatusCode == HttpStatusCode.Redirect && Mode == HttpMode.RedirectGet)
{
StatusCode = (int)Response.StatusCode;
result = Response.Headers["Location"];// 获得 library 影片 Code
}
else
{
num = 2;
}
Response.Close();
}
catch (WebException e)
{
Logger.LogN($"地址：{Url}，失败原因：{e.Message}");
if (e.Status == WebExceptionStatus.Timeout)
{
num += 1;
}
else
{
num = 2;
}
}
catch (Exception e)
{
Logger.LogN($"地址：{Url}，失败原因：{e.Message}");
num = 2;
}
finally
{
if (Response != null)
{
Response.Close();
}
}
return(result);
}).TimeoutAfter(TimeSpan.FromSeconds(HTTPTIMEOUT));
}
catch (TimeoutException ex) { Logger.LogN($"地址：{Url}，失败原因：{ex.Message}"); num = 2; }
}
return(HtmlText, StatusCode);
}
/// <summary>
/// Http 请求
/// </summary>
/// <param name="Url"></param>
/// <param name="headers"></param>
/// <param name="Mode"></param>
/// <param name="Proxy"></param>
/// <param name="allowRedirect"></param>
/// <param name="poststring"></param>
/// <returns></returns>
public static async Task <HttpResult> Http(string Url, CrawlerHeader headers = null, HttpMode Mode = HttpMode.Normal, WebProxy Proxy = null, bool allowRedirect = true, string postString = "")
{
if (!Url.IsProperUrl())
{
return(null);
}
if (headers == null)
{
headers = new CrawlerHeader();
}
int trynum = 0;
HttpResult httpResult = null;
try
{
while (trynum < ATTEMPTNUM && httpResult == null)
{
httpResult = await Task.Run(() =>
{
HttpWebRequest Request;
HttpWebResponse Response = default;
try
{
Request = (HttpWebRequest)HttpWebRequest.Create(Url);
}
catch (Exception ex)
{
Logger.LogE(ex);
return(null);
}
Uri uri = new Uri(Url);
Request.Host = headers.Host == "" ? uri.Host : headers.Host;
Request.Accept = headers.Accept;
Request.Timeout = HTTPTIMEOUT;
Request.Method = headers.Method;
Request.KeepAlive = true;
Request.AllowAutoRedirect = allowRedirect;
Request.Referer = uri.Scheme + "://" + uri.Host + "/";
Request.UserAgent = headers.UserAgent;
Request.Headers.Add("Accept-Language", headers.AcceptLanguage);
Request.Headers.Add("Upgrade-Insecure-Requests", headers.UpgradeInsecureRequests);
Request.Headers.Add("Sec-Fetch-Site", headers.SecFetchSite);
Request.Headers.Add("Sec-Fetch-Mode", headers.SecFetchMode);
Request.Headers.Add("Sec-Fetch-User", headers.SecFetchUser);
Request.Headers.Add("Sec-Fetch-Dest", headers.SecFetchDest);
Request.ReadWriteTimeout = READWRITETIMEOUT;
if (headers.Cookies != "")
{
Request.Headers.Add("Cookie", headers.Cookies);
}
if (Mode == HttpMode.RedirectGet)
{
Request.AllowAutoRedirect = false;
}
if (Proxy != null)
{
Request.Proxy = Proxy;
}
try
{
if (headers.Method == "POST")
{
Request.Method = "POST";
Request.ContentType = headers.ContentType;
Request.ContentLength = headers.ContentLength;
Request.Headers.Add("Origin", headers.Origin);
byte[] bs = Encoding.UTF8.GetBytes(postString);
using (Stream reqStream = Request.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
}
}
Response = (HttpWebResponse)Request.GetResponse();
httpResult = GetHttpResult(Response, Mode);
Logger.LogN($" {Jvedio.Language.Resources.Url}：{Url} => {httpResult.StatusCode}");
}
catch (WebException e)
{
Logger.LogN($" {Jvedio.Language.Resources.Url}：{Url}， {Jvedio.Language.Resources.Reason}：{e.Message}");
if (e.Status == WebExceptionStatus.Timeout)
{
trynum++;
}
else
{
trynum = 2;
}
}
catch (Exception e)
{
Logger.LogN($" {Jvedio.Language.Resources.Url}：{Url}， {Jvedio.Language.Resources.Reason}：{e.Message}");
trynum = 2;
}
finally
{
if (Response != null)
{
Response.Close();
}
}
return(httpResult);
}).TimeoutAfter(TimeSpan.FromSeconds(HTTPTIMEOUT));
}
}
catch (TimeoutException ex)
{
//任务超时了
Console.WriteLine(ex.Message);
Logger.LogN(ex.Message);
}
return(httpResult);
}
/// <summary>
/// Internal method which builds the Query Uri and executes it via GET/POST as appropriate.
/// </summary>
/// <param name="sparqlQuery">Sparql Query.</param>
/// <param name="acceptHeader">Accept Header to use for the request.</param>
/// <returns></returns>
private HttpWebResponse QueryInternal(String sparqlQuery, String acceptHeader)
{
// Patched by Alexander Zapirov to handle situations where the SPARQL Query is very long
// i.e. would exceed the length limit of the Uri class
// Build the Query Uri
StringBuilder queryUri = new StringBuilder();
queryUri.Append(Uri.AbsoluteUri);
bool longQuery = true;
if (HttpMode.Equals("GET") ||
HttpMode.Equals("AUTO") && sparqlQuery.Length <= LongQueryLength && sparqlQuery.IsAscii())
{
longQuery = false;
try
{
queryUri.Append(!Uri.Query.Equals(string.Empty) ? "&query=" : "?query=");
queryUri.Append(HttpUtility.UrlEncode(sparqlQuery));
// Add the Default Graph URIs
foreach (string defaultGraph in _defaultGraphUris)
{
if (defaultGraph.Equals(string.Empty))
{
continue;
}
queryUri.Append("&default-graph-uri=");
queryUri.Append(HttpUtility.UrlEncode(defaultGraph));
}
// Add the Named Graph URIs
foreach (string namedGraph in _namedGraphUris)
{
if (namedGraph.Equals(string.Empty))
{
continue;
}
queryUri.Append("&named-graph-uri=");
queryUri.Append(HttpUtility.UrlEncode(namedGraph));
}
}
catch (UriFormatException)
{
if (HttpMode.Equals("GET"))
{
throw;
}
longQuery = true;
}
}
// Make the Query via HTTP
HttpWebResponse httpResponse;
if (HttpMode.Equals("AUTO") && (longQuery || queryUri.Length > 2048) || HttpMode == "POST")
{
// Long Uri/HTTP POST Mode so use POST
StringBuilder postData = new StringBuilder();
postData.Append("query=");
postData.Append(HttpUtility.UrlEncode(sparqlQuery));
// Add the Default Graph URI(s)
foreach (String defaultGraph in _defaultGraphUris)
{
if (defaultGraph.Equals(String.Empty))
{
continue;
}
postData.Append("&default-graph-uri=");
postData.Append(HttpUtility.UrlEncode(defaultGraph));
}
// Add the Named Graph URI(s)
foreach (String namedGraph in _namedGraphUris)
{
if (namedGraph.Equals(String.Empty))
{
continue;
}
postData.Append("&named-graph-uri=");
postData.Append(HttpUtility.UrlEncode(namedGraph));
}
httpResponse = ExecuteQuery(Uri, postData.ToString(), acceptHeader);
}
else
{
// Make the query normally via GET
httpResponse = ExecuteQuery(UriFactory.Create(queryUri.ToString()), String.Empty, acceptHeader);
}
return(httpResponse);
}
/// <summary>
/// 将 HttpWebResponse 转为 HttpResult
/// </summary>
/// <param name="response"></param>
/// <param name="mode"></param>
/// <returns></returns>
public HttpResult GetHttpResult(HttpWebResponse response, HttpMode mode)
{
HttpResult httpResult = new HttpResult();
WebHeaderCollection webHeaderCollection = null;
ResponseHeaders responseHeaders = null;
try
{
httpResult.StatusCode = response.StatusCode;
webHeaderCollection = response.Headers;
}
catch (ObjectDisposedException ex)
{
Log(ex.Message);
}
if (webHeaderCollection != null)
{
//获得响应头
responseHeaders = new ResponseHeaders()
{
Location = webHeaderCollection.Get("Location"),
Date = webHeaderCollection.Get("Date"),
ContentType = webHeaderCollection.Get("Content-Type"),
Connection = webHeaderCollection.Get("Connection"),
CacheControl = webHeaderCollection.Get("Cache-Control"),
SetCookie = webHeaderCollection.Get("Set-Cookie"),
};
double.TryParse(webHeaderCollection.Get("Content-Length"), out responseHeaders.ContentLength);
httpResult.Headers = responseHeaders;
}
if (response.StatusCode == HttpStatusCode.OK)
{
if (mode != HttpMode.Stream)
{
try
{
using (StreamReader sr = new StreamReader(response.GetResponseStream()))
{
httpResult.SourceCode = sr.ReadToEnd();
}
}
catch (Exception ex)
{
Log(ex);
}
if (responseHeaders?.ContentLength == 0)
{
responseHeaders.ContentLength = httpResult.SourceCode.Length;
}
}
else
{
try
{
using (MemoryStream ms = new MemoryStream())
{
response.GetResponseStream().CopyTo(ms);
httpResult.FileByte = ms.ToArray();
}
}
catch (Exception ex)
{
Log(ex);
}
if (responseHeaders?.ContentLength == 0)
{
responseHeaders.ContentLength = httpResult.FileByte.Length;
}
}
}
return(httpResult);
}
private async Task <HttpResponse <T> > GetDeserializedObject <T>(string requestUrl, HttpMode mode, string message = null)
{
var result = await _httpService.ExecuteAsync(requestUrl, mode, message);
var response = new HttpResponse <T>
{
HttpStatusCode = result.HttpStatusCode,
Message = result.Message,
Location = result.Location
};
if (result.IsSuccessStatusCode)
{
response.Result = await Task.Run(() => { return(JsonConvert.DeserializeObject <T>(result.Result)); });
}
return(response);
}
