/// <summary>
/// 创建HTTP请求
/// </summary>
/// <returns></returns>
private HttpWebRequest CreateRequest(string url, string method)
{
var uri = new Uri(url);
if (uri.Scheme == "https")
{
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
}
// Set a default policy level for the "http:" and "https" schemes.
var policy = new HttpRequestCachePolicy(HttpRequestCacheLevel.Revalidate);
HttpWebRequest.DefaultCachePolicy = policy;
var request = (HttpWebRequest)WebRequest.Create(uri);
request.AllowAutoRedirect = false;
request.AllowWriteStreamBuffering = false;
request.Method = method;
if (Proxy != null)
{
request.Proxy = Proxy;
}
request.CookieContainer = _cc;
foreach (string key in RequestHeaders.Keys)
{
request.Headers.Add(key, RequestHeaders[key]);
}
RequestHeaders.Clear();
return(request);
}
//
// Summary:
// Called on the CEF IO thread when a resource load has completed.
//
// Parameters:
// frame:
// The frame that is being redirected.
//
// request:
// the request object - cannot be modified in this callback
//
// response:
// the response object - cannot be modified in this callback
//
// status:
// indicates the load completion status
//
// receivedContentLength:
// is the number of response bytes actually read.
public void OnResourceLoadComplete(IWebBrowser browserControl, IBrowser browser, IFrame frame, IRequest request, IResponse response, UrlRequestStatus status, long receivedContentLength)
{
if (request.Url.Contains(".user.js"))
{
var code = browserControl.GetSourceAsync().ToString();
myForm.monyet.InstallScript(request.Url, code);
}
if (oldAddress != browserControl.Address || oldAddress == "")
{
oldAddress = browserControl.Address;
Resources.Clear();
ResponseHeaders.Clear();
RequestHeaders.Clear();
}
Dictionary <string, string> dictionary = response.Headers.AllKeys.ToDictionary((string x) => x, (string x) => response.Headers[x]);
ResponseHeaders.Add(new HeaderWrapper(request.Identifier, dictionary));
Dictionary <string, string> strs = request.Headers.AllKeys.ToDictionary((string x) => x, (string x) => request.Headers[x]);
RequestHeaders.Add(new HeaderWrapper(request.Identifier, strs));
if (responseDictionary.TryGetValue(request.Identifier, out MemoryStreamResponseFilter memoryStreamResponseFilter))
{
byte[] data = memoryStreamResponseFilter.Data;
Resources.Add(new RequestWrapper(request.Url, request.ResourceType, response.MimeType, data, request.Identifier));
}
//var x = response.MimeType;
//scrape all link on current web here
//myForm.X.Appender(request.Url.GetQuery("path"));
//myForm.X.Appender(request.Url + " = " + status);
/*
* if (request.Url.Contains(myForm.ViewsourceURL))
* {
*
* var code = myForm.viewer.View(request.Url.GetQuery("path"));
* MessageBox.Show(code.ToString());
* var t = string.Format("var x = @{0}{1}{0};", '"', code);
* t += string.Format("$({0}#xmpTagId{0}).text(x);", '"');
* myForm.X.ExecuteJs(t);
*
* }
*/
}
/// <summary>
/// Clear the Request Headers
/// </summary>
public void ClearHeaders()
{
RequestHeaders.Clear();
}
private HttpStatusCode Exec(string method, string path, string contentType, byte[] payload, int count)
{
RequestUri = new Uri(_baseUri, path);
Log.Verbose("{0} {1}", method, path);
Stopwatch timer = new Stopwatch();
timer.Start();
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(RequestUri);
req.CookieContainer = _cookies;
req.UserAgent = UserAgent;
req.Method = method;
//Request Configuration
req.Accept = "*/*";
req.KeepAlive = true;
req.SendChunked = false;
req.AllowAutoRedirect = false;
req.UseDefaultCredentials = false;
req.Timeout = (int)Timeout.TotalMilliseconds;
req.ReadWriteTimeout = (int)Timeout.TotalMilliseconds;
req.AutomaticDecompression = DecompressionMethods.None;
req.ServicePoint.UseNagleAlgorithm = false;
req.ServicePoint.Expect100Continue = false;
req.ServicePoint.ConnectionLimit = 25;
HttpWebResponse response = null;
try
{
StatusCode = HttpStatusCode.Continue;
ContentType = String.Empty;
Content = EmptyBytes;
RedirectUri = null;
LastModified = null;
ETag = null;
ResponseHeaders = null;
if (String.IsNullOrEmpty(RequestHeaders["Accept-Encoding"]))
{
req.Headers.Add("Accept-Encoding", "gzip");
}
if (RequestHeaders.Count > 0)
{
foreach (string key in RequestHeaders.Keys)
{
req.Headers.Add(key, RequestHeaders[key]);
}
RequestHeaders.Clear();
}
if (method == "POST")
{
req.ContentType = contentType;
req.ContentLength = count;
if (count > 0)
{
using (Stream io = req.GetRequestStream())
io.Write(payload, 0, count);
}
}
response = GetResponse(req);
}
finally
{
timer.Stop();
Log.Verbose("{0} {1} ({2:n0} ms)", response == null ? 0 : response.StatusCode, path,
timer.ElapsedMilliseconds);
}
using (response)
{
WebHeaderCollection headers = response.Headers;
ResponseHeaders = headers;
ContentType = headers[HttpResponseHeader.ContentType] ?? String.Empty;
StatusCode = response.StatusCode;
ETag = headers["ETag"];
DateTime modified;
if (DateTime.TryParseExact(headers["Last-Modified"], "r", CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out modified))
{
LastModified = modified.ToLocalTime();
}
if (StatusCode == HttpStatusCode.Redirect || StatusCode == HttpStatusCode.Moved || StatusCode == HttpStatusCode.SeeOther)
{
RedirectUri = new Uri(RequestUri, headers[HttpResponseHeader.Location]);
}
if (method == "HEAD")
{
Content = new byte[0];
}
else
{
using (Stream input = response.GetResponseStream())
{
int length;
if (headers[HttpResponseHeader.ContentLength] != null &&
int.TryParse(headers[HttpResponseHeader.ContentLength], out length))
{
Content = IOStream.Read(input, length);
}
else
{
Content = IOStream.ReadAllBytes(input);
}
if ((headers[HttpResponseHeader.ContentEncoding] ?? "").IndexOf("gzip", StringComparison.OrdinalIgnoreCase) >= 0 ||
(headers[HttpResponseHeader.TransferEncoding] ?? "").IndexOf("gzip", StringComparison.OrdinalIgnoreCase) >= 0)
{
Content = Content.Decompress();
}
}
if (StatusCode == HttpStatusCode.OK && Content.Length == 0)
{
Uri uriRedirect;
Match m;
if (!String.IsNullOrEmpty(ResponseHeaders["Refresh"]) &&
(m = Regex.Match(ResponseHeaders["Refresh"], @"^\d+\s*;\s*url\s*=\s*([^\s]+)\s*$")).Success &&
Uri.TryCreate(RequestUri, m.Groups[1].Value, out uriRedirect))
{
RedirectUri = uriRedirect;
StatusCode = HttpStatusCode.Redirect;
}
else
{
Log.Warning("The server returned emtpy content for the url = {0}", RequestUri);
}
}
}
return(StatusCode);
}
}
