/// <summary> /// Checks an individual site and returns a new HttpRequestData object /// </summary> /// <param name="reqData"></param> /// <returns></returns> public HttpRequestData CheckSite(HttpRequestData reqData) { if (CancelThreads) { return(null); } // create a new instance var result = HttpRequestData.Copy(reqData); result.ErrorMessage = "Request is incomplete"; // assume not going to make it result.IsWarmupRequest = StartTime.AddSeconds(Options.WarmupSeconds) > DateTime.UtcNow; try { var client = new HttpClient(); if (!string.IsNullOrEmpty(Options.ReplaceDomain)) { result.Url = ReplaceDomain(result.Url); } if (!string.IsNullOrEmpty(Options.ReplaceQueryStringValuePairs)) { result.Url = ReplaceQueryStringValuePairs(result.Url, Options.ReplaceQueryStringValuePairs); } foreach (var plugin in App.Plugins) { try { if (!plugin.OnBeforeRequestSent(result)) { return(result); } } catch (Exception ex) { App.Log(plugin.GetType().Name + " failed in OnBeforeRequestSent(): " + ex.Message); } } client.CreateWebRequestObject(result.Url); var webRequest = client.WebRequest; // TODO: Connection Groups might help with sharing connections more efficiently // Initial tests show no improvements - more research required //webRequest.ConnectionGroupName = "_WebSurge_" + Thread.CurrentContext.ContextID; if (!string.IsNullOrEmpty(Options.Username)) { client.Username = Options.Username; webRequest.UnsafeAuthenticatedConnectionSharing = true; } if (!string.IsNullOrEmpty(Options.Password)) { client.Password = Options.Password; } webRequest.Method = reqData.HttpVerb; client.ContentType = reqData.ContentType; client.Timeout = Options.RequestTimeoutMs / 1000; // don't auto-add gzip headers and don't decode by default client.UseGZip = false; if (Options.NoContentDecompression) { webRequest.AutomaticDecompression = DecompressionMethods.None; } else { webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; } if (!string.IsNullOrEmpty(reqData.RequestContent)) { var data = reqData.GetRequestContentBytes(); client.AddPostKey(data); } else { webRequest.ContentLength = 0; } foreach (var header in result.Headers) { var lheader = header.Name.ToLower(); // Header Overrides that fail if you try to set them // directly in HTTP if (lheader == "cookie" && !string.IsNullOrEmpty(Options.ReplaceCookieValue)) { string cookie = Options.ReplaceCookieValue; webRequest.Headers.Add("Cookie", cookie); header.Value = cookie; continue; } if (lheader == "authorization" && !string.IsNullOrEmpty(Options.ReplaceAuthorization)) { webRequest.Headers.Add("Authorization", Options.ReplaceAuthorization); header.Value = Options.ReplaceAuthorization; continue; } if (lheader == "user-agent") { client.UserAgent = header.Value; continue; } if (lheader == "accept") { webRequest.Accept = header.Value; continue; } if (lheader == "referer") { webRequest.Referer = header.Value; continue; } if (lheader == "connection") { if (header.Value.ToLower() == "keep-alive") { webRequest.KeepAlive = true; // this has no effect } else if (header.Value.ToLower() == "close") { webRequest.KeepAlive = false; } continue; } // set above view property if (lheader == "content-type") { continue; } // not handled at the moment if (lheader == "proxy-connection") { continue; // TODO: Figure out what to do with this one } // set explicitly via properties if (lheader == "transfer-encoding") { webRequest.TransferEncoding = header.Value; continue; } if (lheader == "date") { continue; } if (lheader == "expect") { //webRequest.Expect = header.Value; continue; } if (lheader == "if-modified-since") { continue; } webRequest.Headers.Add(header.Name, header.Value); } DateTime dt = DateTime.UtcNow; if (CancelThreads) { return(null); } // using West Wind HttpClient string httpOutput = client.DownloadString(result.Url); if (CancelThreads) { return(null); } result.TimeTakenMs = (int)DateTime.UtcNow.Subtract(dt).TotalMilliseconds; // result.TimeToFirstByteMs = client.Timings.TimeToFirstByteMs; if (client.Error || client.WebResponse == null) { result.ErrorMessage = client.ErrorMessage; return(result); } result.StatusCode = ((int)client.WebResponse.StatusCode).ToString(); result.StatusDescription = client.WebResponse.StatusDescription ?? string.Empty; result.TimeToFirstByteMs = client.HttpTimings.TimeToFirstByteMs; result.ResponseLength = (int)client.WebResponse.ContentLength; if (result.ResponseLength < 1 && !string.IsNullOrEmpty(httpOutput)) { result.ResponseLength = httpOutput.Length; } result.ResponseContent = httpOutput; StringBuilder sb = new StringBuilder(); foreach (string key in client.WebResponse.Headers.Keys) { sb.AppendLine(key + ": " + client.WebResponse.Headers[key]); } result.ResponseHeaders = sb.ToString(); // update to actual Http headers sent result.Headers.Clear(); foreach (string key in webRequest.Headers.Keys) { result.Headers.Add(new HttpRequestHeader() { Name = key, Value = webRequest.Headers[key] }); } char statusCode = result.StatusCode[0]; if (statusCode == '4' || statusCode == '5') { result.IsError = true; result.ErrorMessage = client.WebResponse.StatusDescription; } else { result.IsError = false; result.ErrorMessage = null; if (Options.MaxResponseSize > 0 && result.ResponseContent.Length > Options.MaxResponseSize) { result.ResponseContent = result.ResponseContent.Substring(0, Options.MaxResponseSize); } } //} // using client client.Dispose(); if (!CancelThreads) { OnRequestProcessed(result); } return(result); } // these will occur on shutdown - don't log since they will return // unstable results - just ignore catch (ThreadAbortException) { return(null); } catch (Exception ex) { Console.WriteLine("Exception: " + ex.Message); result.IsError = true; result.ErrorMessage = "CheckSite Error: " + ex.GetBaseException().Message; if (!CancelThreads) { OnRequestProcessed(result); } return(result); } }