/// <summary>
/// Checks an individual site and returns a new HttpRequestData object
/// </summary>
/// <param name="reqData"></param>
/// <returns></returns>
public HttpRequestData CheckSite(HttpRequestData reqData)
{
if (CancelThreads)
{
return(null);
}
// create a new instance
var result = HttpRequestData.Copy(reqData);
result.ErrorMessage = "Request is incomplete"; // assume not going to make it
result.IsWarmupRequest = StartTime.AddSeconds(Options.WarmupSeconds) > DateTime.UtcNow;
try
{
var client = new HttpClient();
if (!string.IsNullOrEmpty(Options.ReplaceDomain))
{
result.Url = ReplaceDomain(result.Url);
}
if (!string.IsNullOrEmpty(Options.ReplaceQueryStringValuePairs))
{
result.Url = ReplaceQueryStringValuePairs(result.Url, Options.ReplaceQueryStringValuePairs);
}
foreach (var plugin in App.Plugins)
{
try
{
if (!plugin.OnBeforeRequestSent(result))
{
return(result);
}
}
catch (Exception ex)
{
App.Log(plugin.GetType().Name + " failed in OnBeforeRequestSent(): " + ex.Message);
}
}
client.CreateWebRequestObject(result.Url);
var webRequest = client.WebRequest;
// TODO: Connection Groups might help with sharing connections more efficiently
// Initial tests show no improvements - more research required
//webRequest.ConnectionGroupName = "_WebSurge_" + Thread.CurrentContext.ContextID;
if (!string.IsNullOrEmpty(Options.Username))
{
client.Username = Options.Username;
webRequest.UnsafeAuthenticatedConnectionSharing = true;
}
if (!string.IsNullOrEmpty(Options.Password))
{
client.Password = Options.Password;
}
webRequest.Method = reqData.HttpVerb;
client.ContentType = reqData.ContentType;
client.Timeout = Options.RequestTimeoutMs / 1000;
// don't auto-add gzip headers and don't decode by default
client.UseGZip = false;
if (Options.NoContentDecompression)
{
webRequest.AutomaticDecompression = DecompressionMethods.None;
}
else
{
webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
}
if (!string.IsNullOrEmpty(reqData.RequestContent))
{
var data = reqData.GetRequestContentBytes();
client.AddPostKey(data);
}
else
{
webRequest.ContentLength = 0;
}
foreach (var header in result.Headers)
{
var lheader = header.Name.ToLower();
// Header Overrides that fail if you try to set them
// directly in HTTP
if (lheader == "cookie" && !string.IsNullOrEmpty(Options.ReplaceCookieValue))
{
string cookie = Options.ReplaceCookieValue;
webRequest.Headers.Add("Cookie", cookie);
header.Value = cookie;
continue;
}
if (lheader == "authorization" && !string.IsNullOrEmpty(Options.ReplaceAuthorization))
{
webRequest.Headers.Add("Authorization", Options.ReplaceAuthorization);
header.Value = Options.ReplaceAuthorization;
continue;
}
if (lheader == "user-agent")
{
client.UserAgent = header.Value;
continue;
}
if (lheader == "accept")
{
webRequest.Accept = header.Value;
continue;
}
if (lheader == "referer")
{
webRequest.Referer = header.Value;
continue;
}
if (lheader == "connection")
{
if (header.Value.ToLower() == "keep-alive")
{
webRequest.KeepAlive = true; // this has no effect
}
else if (header.Value.ToLower() == "close")
{
webRequest.KeepAlive = false;
}
continue;
}
// set above view property
if (lheader == "content-type")
{
continue;
}
// not handled at the moment
if (lheader == "proxy-connection")
{
continue; // TODO: Figure out what to do with this one
}
// set explicitly via properties
if (lheader == "transfer-encoding")
{
webRequest.TransferEncoding = header.Value;
continue;
}
if (lheader == "date")
{
continue;
}
if (lheader == "expect")
{
//webRequest.Expect = header.Value;
continue;
}
if (lheader == "if-modified-since")
{
continue;
}
webRequest.Headers.Add(header.Name, header.Value);
}
DateTime dt = DateTime.UtcNow;
if (CancelThreads)
{
return(null);
}
// using West Wind HttpClient
string httpOutput = client.DownloadString(result.Url);
if (CancelThreads)
{
return(null);
}
result.TimeTakenMs = (int)DateTime.UtcNow.Subtract(dt).TotalMilliseconds;
// result.TimeToFirstByteMs = client.Timings.TimeToFirstByteMs;
if (client.Error || client.WebResponse == null)
{
result.ErrorMessage = client.ErrorMessage;
return(result);
}
result.StatusCode = ((int)client.WebResponse.StatusCode).ToString();
result.StatusDescription = client.WebResponse.StatusDescription ?? string.Empty;
result.TimeToFirstByteMs = client.HttpTimings.TimeToFirstByteMs;
result.ResponseLength = (int)client.WebResponse.ContentLength;
if (result.ResponseLength < 1 && !string.IsNullOrEmpty(httpOutput))
{
result.ResponseLength = httpOutput.Length;
}
result.ResponseContent = httpOutput;
StringBuilder sb = new StringBuilder();
foreach (string key in client.WebResponse.Headers.Keys)
{
sb.AppendLine(key + ": " + client.WebResponse.Headers[key]);
}
result.ResponseHeaders = sb.ToString();
// update to actual Http headers sent
result.Headers.Clear();
foreach (string key in webRequest.Headers.Keys)
{
result.Headers.Add(new HttpRequestHeader()
{
Name = key, Value = webRequest.Headers[key]
});
}
char statusCode = result.StatusCode[0];
if (statusCode == '4' || statusCode == '5')
{
result.IsError = true;
result.ErrorMessage = client.WebResponse.StatusDescription;
}
else
{
result.IsError = false;
result.ErrorMessage = null;
if (Options.MaxResponseSize > 0 && result.ResponseContent.Length > Options.MaxResponseSize)
{
result.ResponseContent = result.ResponseContent.Substring(0, Options.MaxResponseSize);
}
}
//} // using client
client.Dispose();
if (!CancelThreads)
{
OnRequestProcessed(result);
}
return(result);
}
// these will occur on shutdown - don't log since they will return
// unstable results - just ignore
catch (ThreadAbortException)
{
return(null);
}
catch (Exception ex)
{
Console.WriteLine("Exception: " + ex.Message);
result.IsError = true;
result.ErrorMessage = "CheckSite Error: " + ex.GetBaseException().Message;
if (!CancelThreads)
{
OnRequestProcessed(result);
}
return(result);
}
}
