/// <summary> /// 创建HTTP请求 /// </summary> /// <returns></returns> private HttpWebRequest CreateRequest(string url, string method) { var uri = new Uri(url); if (uri.Scheme == "https") { ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult; } // Set a default policy level for the "http:" and "https" schemes. var policy = new HttpRequestCachePolicy(HttpRequestCacheLevel.Revalidate); HttpWebRequest.DefaultCachePolicy = policy; var request = (HttpWebRequest)WebRequest.Create(uri); request.AllowAutoRedirect = false; request.AllowWriteStreamBuffering = false; request.Method = method; if (Proxy != null) { request.Proxy = Proxy; } request.CookieContainer = _cc; foreach (string key in RequestHeaders.Keys) { request.Headers.Add(key, RequestHeaders[key]); } RequestHeaders.Clear(); return(request); }
// // Summary: // Called on the CEF IO thread when a resource load has completed. // // Parameters: // frame: // The frame that is being redirected. // // request: // the request object - cannot be modified in this callback // // response: // the response object - cannot be modified in this callback // // status: // indicates the load completion status // // receivedContentLength: // is the number of response bytes actually read. public void OnResourceLoadComplete(IWebBrowser browserControl, IBrowser browser, IFrame frame, IRequest request, IResponse response, UrlRequestStatus status, long receivedContentLength) { if (request.Url.Contains(".user.js")) { var code = browserControl.GetSourceAsync().ToString(); myForm.monyet.InstallScript(request.Url, code); } if (oldAddress != browserControl.Address || oldAddress == "") { oldAddress = browserControl.Address; Resources.Clear(); ResponseHeaders.Clear(); RequestHeaders.Clear(); } Dictionary <string, string> dictionary = response.Headers.AllKeys.ToDictionary((string x) => x, (string x) => response.Headers[x]); ResponseHeaders.Add(new HeaderWrapper(request.Identifier, dictionary)); Dictionary <string, string> strs = request.Headers.AllKeys.ToDictionary((string x) => x, (string x) => request.Headers[x]); RequestHeaders.Add(new HeaderWrapper(request.Identifier, strs)); if (responseDictionary.TryGetValue(request.Identifier, out MemoryStreamResponseFilter memoryStreamResponseFilter)) { byte[] data = memoryStreamResponseFilter.Data; Resources.Add(new RequestWrapper(request.Url, request.ResourceType, response.MimeType, data, request.Identifier)); } //var x = response.MimeType; //scrape all link on current web here //myForm.X.Appender(request.Url.GetQuery("path")); //myForm.X.Appender(request.Url + " = " + status); /* * if (request.Url.Contains(myForm.ViewsourceURL)) * { * * var code = myForm.viewer.View(request.Url.GetQuery("path")); * MessageBox.Show(code.ToString()); * var t = string.Format("var x = @{0}{1}{0};", '"', code); * t += string.Format("$({0}#xmpTagId{0}).text(x);", '"'); * myForm.X.ExecuteJs(t); * * } */ }
/// <summary> /// Clear the Request Headers /// </summary> public void ClearHeaders() { RequestHeaders.Clear(); }
private HttpStatusCode Exec(string method, string path, string contentType, byte[] payload, int count) { RequestUri = new Uri(_baseUri, path); Log.Verbose("{0} {1}", method, path); Stopwatch timer = new Stopwatch(); timer.Start(); HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(RequestUri); req.CookieContainer = _cookies; req.UserAgent = UserAgent; req.Method = method; //Request Configuration req.Accept = "*/*"; req.KeepAlive = true; req.SendChunked = false; req.AllowAutoRedirect = false; req.UseDefaultCredentials = false; req.Timeout = (int)Timeout.TotalMilliseconds; req.ReadWriteTimeout = (int)Timeout.TotalMilliseconds; req.AutomaticDecompression = DecompressionMethods.None; req.ServicePoint.UseNagleAlgorithm = false; req.ServicePoint.Expect100Continue = false; req.ServicePoint.ConnectionLimit = 25; HttpWebResponse response = null; try { StatusCode = HttpStatusCode.Continue; ContentType = String.Empty; Content = EmptyBytes; RedirectUri = null; LastModified = null; ETag = null; ResponseHeaders = null; if (String.IsNullOrEmpty(RequestHeaders["Accept-Encoding"])) { req.Headers.Add("Accept-Encoding", "gzip"); } if (RequestHeaders.Count > 0) { foreach (string key in RequestHeaders.Keys) { req.Headers.Add(key, RequestHeaders[key]); } RequestHeaders.Clear(); } if (method == "POST") { req.ContentType = contentType; req.ContentLength = count; if (count > 0) { using (Stream io = req.GetRequestStream()) io.Write(payload, 0, count); } } response = GetResponse(req); } finally { timer.Stop(); Log.Verbose("{0} {1} ({2:n0} ms)", response == null ? 0 : response.StatusCode, path, timer.ElapsedMilliseconds); } using (response) { WebHeaderCollection headers = response.Headers; ResponseHeaders = headers; ContentType = headers[HttpResponseHeader.ContentType] ?? String.Empty; StatusCode = response.StatusCode; ETag = headers["ETag"]; DateTime modified; if (DateTime.TryParseExact(headers["Last-Modified"], "r", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out modified)) { LastModified = modified.ToLocalTime(); } if (StatusCode == HttpStatusCode.Redirect || StatusCode == HttpStatusCode.Moved || StatusCode == HttpStatusCode.SeeOther) { RedirectUri = new Uri(RequestUri, headers[HttpResponseHeader.Location]); } if (method == "HEAD") { Content = new byte[0]; } else { using (Stream input = response.GetResponseStream()) { int length; if (headers[HttpResponseHeader.ContentLength] != null && int.TryParse(headers[HttpResponseHeader.ContentLength], out length)) { Content = IOStream.Read(input, length); } else { Content = IOStream.ReadAllBytes(input); } if ((headers[HttpResponseHeader.ContentEncoding] ?? "").IndexOf("gzip", StringComparison.OrdinalIgnoreCase) >= 0 || (headers[HttpResponseHeader.TransferEncoding] ?? "").IndexOf("gzip", StringComparison.OrdinalIgnoreCase) >= 0) { Content = Content.Decompress(); } } if (StatusCode == HttpStatusCode.OK && Content.Length == 0) { Uri uriRedirect; Match m; if (!String.IsNullOrEmpty(ResponseHeaders["Refresh"]) && (m = Regex.Match(ResponseHeaders["Refresh"], @"^\d+\s*;\s*url\s*=\s*([^\s]+)\s*$")).Success && Uri.TryCreate(RequestUri, m.Groups[1].Value, out uriRedirect)) { RedirectUri = uriRedirect; StatusCode = HttpStatusCode.Redirect; } else { Log.Warning("The server returned emtpy content for the url = {0}", RequestUri); } } } return(StatusCode); } }