/// <summary>
/// Called when a request comes into the proxy server
/// </summary>
/// <param name="requestInfo"></param>
/// <returns></returns>
public HttpResponseInfo SendRequest(HttpRequestInfo requestInfo)
{
int port;
HttpResponseInfo respInfo = null;
requestInfo.ParseVariables();
if (String.Compare(START_PAGE, requestInfo.Path, true) == 0)
{
port = ExtractPort(requestInfo);
_commandProxy.StartManualExploreProxy(port);
respInfo = GetSuccessResponse(Resources.RecordingStarted);
}
else if (String.Compare(STOP_PAGE, requestInfo.Path, true) == 0)
{
port = ExtractPort(requestInfo);
string fileName = null;
if (requestInfo.QueryVariables.ContainsKey(FILE_PARAM))
{
fileName = Utils.UrlDecode(requestInfo.QueryVariables[FILE_PARAM]);
}
_commandProxy.StopManualExploreProxy(port, fileName);
respInfo = GetSuccessResponse(Resources.RecordingStopped);
}
else
{
_logWriter.Log(TraceLevel.Error, "An invalid url was specified: {0}", requestInfo.FullUrl);
throw new HttpProxyException(HttpStatusCode.NotImplemented, "Not implemented", ServiceCode.CommandProxyNotImplemented);
}
return(respInfo);
}
private void PrepareConnection(HttpRequestInfo requestInfo)
{
WebProxy proxy = null;
if (NetworkSettings != null)
{
if (NetworkSettings.WebProxy != null)
{
proxy = (NetworkSettings.WebProxy as WebProxy);
if (proxy != null)
{
_connection = new HttpClientConnection(proxy.Address.Host, proxy.Address.Port, false);
}
}
}
if (proxy != null)
{
_connection = new HttpClientConnection(proxy.Address.Host, proxy.Address.Port, false);
}
else
{
_connection = new HttpClientConnection(requestInfo.Host, requestInfo.Port, requestInfo.IsSecure);
}
}
/// <summary>
/// Traps a request and triggers the event for the current thread
/// </summary>
/// <param name="httpReqInfo"></param>
/// <param name="tvReqInfo"></param>
/// <returns>True of False if the request was trapped or not</returns>
public bool TrapRequest(TVRequestInfo tvReqInfo, HttpRequestInfo httpReqInfo)
{
if (_trapRequests)
{
//trigger the event,
if (_requestTrapped != null)
{
if (MatchesTrapDefs(httpReqInfo))
{
ManualResetEvent reqLock = new ManualResetEvent(false);
_trapOn.BeginInvoke(this, new EventArgs(), null, null);
_requestTrapped.BeginInvoke(new RequestTrapEventEventArgs(tvReqInfo, httpReqInfo, reqLock), null, null);
//wait for the event to finish
reqLock.WaitOne();
_trapOff.BeginInvoke(this, new EventArgs(), null, null);
//the request was trapped return true
return(true);
}
}
}
return(false);
}
/// <summary>
/// Remove request headers that can cause issues with the recording
/// </summary>
/// <param name="reqInfo">The request info to be processed</param>
/// <param name="isNonEssential">Whether this has deemed to be non-essential</param>
/// <returns></returns>
protected override HttpRequestInfo ProcessHeaders(HttpRequestInfo reqInfo, bool isNonEssential)
{
if (!isNonEssential) //prevent caching
{
reqInfo.Headers.Remove("Accept-Encoding"); //remove accept encoding to prevent gzip responses
reqInfo.Headers.Remove("If-Modified-Since");
reqInfo.Headers.Remove("If-None-Match");
}
return(base.ProcessHeaders(reqInfo, isNonEssential));
}
/// <summary>
/// Overriden Equals
/// </summary>
/// <param name="obj"></param>
/// <returns></returns>
public override bool Equals(object obj)
{
bool equals = false;
HttpRequestInfo other = obj as HttpRequestInfo;
if (other != null)
{
equals = GetHashCode() == other.GetHashCode();
}
return(equals);
}
/// <summary>
/// Updates the request with the current values
/// </summary>
/// <param name="requestInfo"></param>
public HttpRequestInfo UpdateRequest(HttpRequestInfo requestInfo)
{
if (requestInfo != null)
{
bool isSecure = requestInfo.IsSecure;
string updatedRawRequest = UpdateRequest(requestInfo.ToString());
requestInfo = new HttpRequestInfo(updatedRawRequest);
requestInfo.IsSecure = isSecure;
}
return(requestInfo);
}
/// <summary>
/// Checks if the request matches the traps
/// </summary>
/// <param name="info"></param>
/// <returns></returns>
private bool MatchesTrapDefs(HttpRequestInfo info)
{
string rawRequest = info.ToString();
foreach (HttpTrapDef def in _trapDefs)
{
if (def.Location == HttpTrapLocation.Request && def.IsMatch(rawRequest))
{
return(true);
}
}
return(false);
}
/// <summary>
/// Override to change the OnRead behavior
/// </summary>
/// <param name="ar"></param>
protected virtual void OnRead(IAsyncResult ar)
{
try
{
if (_stop || Closed)
{
return;
}
_isBusy = true;
HttpProxyClientStreamWrapper wrapper = (HttpProxyClientStreamWrapper)ar.AsyncState;
int bytesRead = 0;
bytesRead = wrapper.EndRead(ar);
//we are still connected and we read more bytes
if (bytesRead > 0)
{
// Append data to the request
_requestBuilder.AddChunkReference(Buffer, bytesRead);
_requestInfo = new HttpRequestInfo(_requestBuilder.ToArray(), false);
if (!_requestInfo.IsFullRequest)
{
// not finished keep on reading!
wrapper.BeginRead(Buffer, 0, Buffer.Length, new AsyncCallback(OnRead), wrapper);
}
else
{
lock (_proxyLock)
{
// Done reading, process the request
ProcessRequest();
}
}
}
else
{
//we read 0 bytes
_isBusy = _requestBuilder.Length > 0;
}
}
catch (Exception ex)
{
ClientStreamWrapper.Close();
HttpServerConsole.Instance.WriteLine(ex);
}
}
/// <summary>
/// Checks if the specified request requires credentials (an entry exists in the memory)
/// </summary>
/// <param name="reqInfo"></param>
/// <param name="authInfo"></param>
/// <returns></returns>
public bool RequiresCredentials(HttpRequestInfo reqInfo, out HttpAuthenticationInfo authInfo)
{
int reqHash = reqInfo.HostAndPort.GetHashCode();
authInfo = null;
CacheEntry entry = this.GetEntry(reqHash);
if (entry != null)
{
authInfo = entry.GetClone() as HttpAuthenticationInfo;
return(true);
}
return(false);
}
/// <summary>
/// Handles the request
/// </summary>
protected virtual void ProcessRequest()
{
try
{
_requestInfo.IsSecure = ClientStreamWrapper.IsSecure;
_currentRequestTime = DateTime.Now;
HttpServerConsole.Instance.WriteLine("Got request: {0}", _requestInfo.RequestLine);
if (_requestInfo.IsConnect)
{
// Connect requests are handled differently.
// Process it and return.
ProcessConnect();
return;
}
_isNonEssential = IsExcludedRequest(_requestInfo.RequestLine + Environment.NewLine + _requestInfo.Headers.ToString()); //test both request line (for file extensions) and headers (for domains)
_requestInfo = ProcessHeaders(_requestInfo, _isNonEssential);
if (!_isNonEssential)
{
if (_trafficDataStore != null)
{
//save request to the traffic data store before sending it
AddRequestToDataStore();
}
}
_requestInfo = OnBeforeRequestToSite(_requestInfo);
HttpResponseInfo responseInfo = HttpClient.SendRequest(_requestInfo);
if (responseInfo == null) //timeout or connection closed
{
throw new WebException("Send error", WebExceptionStatus.SendFailure);
}
responseInfo = OnBeforeResponseToClient(responseInfo);
ReturnResponse(responseInfo);
}
catch (Exception ex)
{
HandleException(ex);
}
}
/// <summary>
/// Gets the credentials for the specified requests using the credential provider
/// </summary>
/// <param name="reqInfo"></param>
/// <param name="isProxy"></param>
/// <returns></returns>
public HttpAuthenticationInfo GetCredentials(HttpRequestInfo reqInfo, bool isProxy)
{
HttpAuthenticationInfo authInfo = null;
string domain, userName, password;
if (_credentialsProvider != null && _credentialsProvider.Execute(out domain, out userName, out password))
{
//encrypt
string encryptedPassword = Encryptor.EncryptToString(password);
authInfo = new HttpAuthenticationInfo(new NetworkCredential(userName, encryptedPassword, domain), isProxy);
//store the auth info in the cache
this.Add(reqInfo.HostAndPort.GetHashCode(), new CacheEntry((ICloneable)authInfo.Clone()));
}
return(authInfo);
}
/// <summary>
/// Extracts the port value from the request
/// </summary>
/// <param name="requestInfo"></param>
/// <returns></returns>
private int ExtractPort(HttpRequestInfo requestInfo)
{
int port = -1;
string portString = null;
if (requestInfo.QueryVariables.ContainsKey(PORT_PARAM))
{
portString = requestInfo.QueryVariables[PORT_PARAM];
}
if (!Utils.ParsePort(portString, out port))
{
_logWriter.Log(TraceLevel.Error, "An invalid port was specified: {0}", port);
throw new HttpProxyException(HttpStatusCode.BadRequest, "Invalid port value", ServiceCode.CommandProxyStartInvalidPort);
}
return(port);
}
/// <summary>
/// Process Request headers before sending
/// Add keep alive for requests that contain authorization
/// </summary>
/// <param name="reqInfo">The request info to be processed</param>
/// <param name="isNonEssential">Whether the request is deemed non essential</param>
protected virtual HttpRequestInfo ProcessHeaders(HttpRequestInfo reqInfo, bool isNonEssential)
{
//the proxy connection header was intended to us, don't send it to the site
//instead pass the information via the connection header, if we want the connection be kept alive
if (reqInfo.Headers[PROXY_CONNECTION_HEADER] != null)
{
reqInfo.Headers[CONNECTION_HEADER] = reqInfo.Headers[PROXY_CONNECTION_HEADER];
reqInfo.Headers.Remove(PROXY_CONNECTION_HEADER);
}
//ensure the client keeps the connection open for handshake authorization requests
//the connection header may be missing
if (reqInfo.Headers["Authorization"] != null &&
reqInfo.Headers["Authorization"] != "Basic")
{
reqInfo.Headers[CONNECTION_HEADER] = "keep-alive";
}
return(reqInfo);
}
protected void ProcessRequestDelay(HttpRequestInfo requestInfo)
{
//delay the request
if (_requestDelay > 0)
{
bool delay = false;
if (!String.IsNullOrEmpty(_requestDelayFilter) && !_requestDelayFilter.Trim().Equals(".*"))
{
delay = Utils.IsMatch(requestInfo.FullUrl, _requestDelayFilter) ||
Utils.IsMatch(requestInfo.ContentDataString, _requestDelayFilter);
}
else
{
delay = true;
}
if (delay)
{
Thread.Sleep(_requestDelay);
}
}
}
/// <summary>
/// Override to do something with the request before sending it to the site
/// </summary>
/// <param name="requestInfo"></param>
/// <returns></returns>
protected virtual HttpRequestInfo OnBeforeRequestToSite(HttpRequestInfo requestInfo)
{
return(requestInfo);
}
private void CopyHeaders(HttpRequestInfo requestInfo, HttpWebRequest webRequest)
{
//by default set 100Continue to false to prevent 417 errors
//however if the client sends an expect header we will change this back
webRequest.ServicePoint.Expect100Continue = false;
//add headers
foreach (HTTPHeader header in requestInfo.Headers)
{
if (String.Compare(header.Name, "Accept", true) == 0)
{
webRequest.Accept = header.Value;
}
else if (String.Compare(header.Name, "Connection", true) == 0 || String.Compare(header.Name, "Proxy-Connection", true) == 0)
{
if (String.Compare(header.Value, "close", true) == 0)
{
webRequest.KeepAlive = false;
}
else
{
webRequest.KeepAlive = true;
}
}
else if (String.Compare(header.Name, "Content-Length", true) == 0)
{
long length;
if (long.TryParse(header.Value, out length))
{
webRequest.ContentLength = length;
}
}
else if (String.Compare(header.Name, "Content-Type", true) == 0)
{
webRequest.ContentType = header.Value;
}
else if (String.Compare(header.Name, "If-Modified-Since", true) == 0)
{
DateTime date;
if (DateTime.TryParse(header.Value, out date))
{
webRequest.IfModifiedSince = date;
}
}
else if (String.Compare(header.Name, "Referer", true) == 0)
{
webRequest.Referer = header.Value;
}
else if (String.Compare(header.Name, "Transfer-Encoding", true) == 0)
{
webRequest.TransferEncoding = header.Value;
}
else if (String.Compare(header.Name, "User-Agent", true) == 0)
{
webRequest.UserAgent = header.Value;
}
else if (String.Compare(header.Name, "Date", true) == 0)
{
DateTime date;
if (DateTime.TryParse(header.Value, out date))
{
webRequest.Date = date;
}
else
{
//the date could not be parsed but try to set now as the date
webRequest.Date = DateTime.Now;
}
}
else if (String.Compare(header.Name, "Range", true) == 0)
{
string specifier;
int from;
int to;
HttpUtil.ParseRange(header.Value, out specifier, out from, out to);
if (from < to)
{
webRequest.AddRange(specifier, from, to);
}
}
else if (String.Compare(header.Name, "Expect", true) == 0)
{
if (String.Compare(header.Value, "100-Continue", true) == 0)
{
webRequest.ServicePoint.Expect100Continue = true;
}
}
else if (String.Compare(header.Name, "Host", true) == 0)
{
//do nothing for these headers
;
}
else if (String.Compare(header.Name, "Cookie", true) == 0 && ShouldHandleCookies)
{
//add the additional cookies to the jar
//if should handle cookies is on cookie management is done by the jar (we want to be able to add cookies)
HttpVariables vars = new HttpVariables(header.Value, RequestLocation.Cookies);
foreach (var cookie in vars)
{
_cookieJar.Add(new Cookie(cookie.Key, cookie.Value, requestInfo.Path, requestInfo.Host));
}
}
else
{
try
{
//webRequest doesn't support PseudoHeaders
if (!header.Name.StartsWith(":"))
{
webRequest.Headers.Set(header.Name, header.Value);
}
}
catch
{
SdkSettings.Instance.Logger.Log(TraceLevel.Error, "WebRequestClient error. Cannot set header {0}", header.Name);
}
}
}
}
/// <summary>
/// Sends the specified request to the server
/// </summary>
/// <param name="parsedRequest"></param>
/// <param name="https"></param>
public void SendRequest(HttpRequestInfo parsedRequest, bool https)
{
SendRequest(parsedRequest, parsedRequest.Host, parsedRequest.Port, https);
}
/// <summary>
/// Sends the specified request to the server
/// </summary>
/// <param name="rawRequest"></param>
/// <param name="https"></param>
public void SendRequest(string rawRequest, bool https)
{
HttpRequestInfo reqInfo = new HttpRequestInfo(rawRequest);
this.SendRequest(reqInfo, reqInfo.Host, reqInfo.Port, https);
}
/// <summary>
/// Sends the specified request to the server
/// (Proxy not supported)
/// </summary>
/// <param name="rawRequest"></param>
public void SendRequest(string rawRequest)
{
HttpRequestInfo reqInfo = new HttpRequestInfo(rawRequest);
this.SendRequest(reqInfo, reqInfo.IsSecure);
}
/// <summary> /// Sends the requests to site and returns an array of bytes /// </summary> /// <param name="requestInfo"></param> /// <returns></returns> public abstract HttpResponseInfo SendRequest(HttpRequestInfo requestInfo);
/// <summary>
/// Sends the request to the site
/// </summary>
/// <param name="requestInfo"></param>
/// <returns></returns>
public override HttpResponseInfo SendRequest(HttpRequestInfo requestInfo)
{
PrepareConnection(requestInfo);
HttpClientRequest request = new HttpClientRequest(_connection);
//check if the request requires credentials
HttpAuthenticationInfo authInfo;
if (HttpAuthenticationManager.Instance.RequiresCredentials(requestInfo, out authInfo))
{
requestInfo.Headers.Add(HttpAuthenticationManager.Instance.GetBasicAuthHeader(authInfo));
}
//delay the request
ProcessRequestDelay(requestInfo);
request.SendRequest(requestInfo, requestInfo.IsSecure);
bool success = request.RequestCompleteEvent.WaitOne(_timeout);
if (!success || request.Response == null)
{
throw new Exception(Resources.RequestTimeout);
}
string rawResponse = Constants.DefaultEncoding.GetString(request.Response);
string responseStatus = HttpResponseInfo.GetResponseStatus(rawResponse);
bool isPlatformAuth = String.Compare(responseStatus, "401") == 0;
bool isProxyAuth = String.Compare(responseStatus, "407") == 0;
if (isPlatformAuth || isProxyAuth)
{
//check the headers
HttpResponseInfo respInfo = new HttpResponseInfo(rawResponse);
List <HTTPHeader> authHeaders = new List <HTTPHeader>();
if (isPlatformAuth)
{
authHeaders = respInfo.Headers.GetHeaders("WWW-Authenticate");
}
else if (isProxyAuth)
{
authHeaders = respInfo.Headers.GetHeaders("Proxy-Authenticate");
}
bool usesBasic = false;
for (int i = authHeaders.Count - 1; i > -1; i--) //go backwards the basic header is usually last
{
HTTPHeader header = authHeaders[i];
if (header.Value.IndexOf("Basic", StringComparison.OrdinalIgnoreCase) == 0)
{
usesBasic = true;
break;
}
}
if (usesBasic)
{
//try to get the credentials from the user
authInfo = HttpAuthenticationManager.Instance.GetCredentials(requestInfo, isProxyAuth);
if (authInfo != null)
{
//add the basic auth header
requestInfo.Headers.Add(HttpAuthenticationManager.Instance.GetBasicAuthHeader(authInfo));
//create a new request
PrepareConnection(requestInfo);
request = new HttpClientRequest(_connection);
//proceed with new request
request.SendRequest(requestInfo, requestInfo.IsSecure);
success = request.RequestCompleteEvent.WaitOne(_timeout);
if (!success)
{
throw new Exception(Resources.RequestTimeout);
}
}
}
}
if (request.Response != null)
{
return(new HttpResponseInfo(request.Response));
}
return(null);
}
/// <summary>
/// Sends the specified request to the server
/// (Proxy not supported)
/// </summary>
/// <param name="parsedRequest"></param>
/// <param name="host"></param>
/// <param name="port"></param>
/// <param name="https"></param>
public void SendRequest(HttpRequestInfo parsedRequest, string host, int port, bool https)
{
_requestCompleteEvent.Reset();
_dataBuilder = new ByteArrayBuilder();
if (_connection == null)
{
_connection = new HttpClientConnection(host, port, https);
}
try
{
//connect
if (_connection.Connect())
{
bool isProxy = _connection.Host != host;
//add connection closed header only this is supported at the moment
parsedRequest.Headers["Connection"] = "close";
if (isProxy)
{
parsedRequest.Headers["Proxy-Connection"] = "close";
}
// Turn off accepting of gzip/deflate
//parsedRequest.Headers.Remove("Accept-Encoding");
//calculate the content length
if (parsedRequest.ContentData != null)
{
parsedRequest.Headers["Content-Length"] = parsedRequest.ContentData.Length.ToString();
}
parsedRequest.Host = host;
parsedRequest.Port = port;
parsedRequest.IsSecure = https;
if (isProxy && https)
{
//send a connect message to the proxy
SendConnect(host, port);
}
byte[] reqBytes = Constants.DefaultEncoding.GetBytes(parsedRequest.ToString(isProxy && !https));
//write to the stream
_connection.Stream.Write(reqBytes, 0, reqBytes.Length);
//start reading
_buffer = new byte[MAX_BUFFER_SIZE];
_connection.Stream.BeginRead(_buffer, 0, MAX_BUFFER_SIZE, new AsyncCallback(ReadResponse), _connection.Stream);
}
else
{
throw new Exception("Cannot connect to server");
}
}
catch (Exception ex)
{
SdkSettings.Instance.Logger.Log(TraceLevel.Error, "HttpClient error sending request {0}", ex.Message);
//notify the caller that the request was completed with an error
if (RequestComplete != null)
{
RequestComplete.Invoke(
new HttpClientRequestCompleteEventArgs());
RequestCompleteEvent.Set();
}
_connection.Close();
}
}
/// <summary>
/// Sends the request to the site
/// </summary>
/// <param name="requestInfo"></param>
/// <returns></returns>
public override HttpResponseInfo SendRequest(HttpRequestInfo requestInfo)
{
HttpWebRequest webRequest = WebRequest.Create(requestInfo.FullUrl) as HttpWebRequest;
if (ShouldHandleCookies)
{
webRequest.CookieContainer = _cookieJar;
}
ProcessRequestDelay(requestInfo);
webRequest.Timeout = _timeout;
//attempt to reserve a connection group name for the current object
//this will make the web request keep alive work properly
webRequest.ConnectionGroupName = GetHashCode().ToString();
SetNetworkSettings(webRequest);
CopyHeaders(requestInfo, webRequest);
//request Method (GET, POST, etc..)
webRequest.Method = requestInfo.Method;
string version = requestInfo.HttpVersion.Replace("HTTP/", "");
if (!version.Equals("1.0") && !version.Equals("1.1"))
{
version = "1.1";
}
webRequest.ProtocolVersion = Version.Parse(version);
webRequest.AllowAutoRedirect = false;
//add POST data, if any
byte[] postData = requestInfo.ContentData;
if (postData != null && postData.Length > 0)
{
try
{
webRequest.ContentLength = postData.Length;
Stream requestStream = webRequest.GetRequestStream();
requestStream.Write(postData, 0, postData.Length);
requestStream.Close();
}
catch (ProtocolViolationException ex) //ignore malformed request such as AppScan requests
{
SdkSettings.Instance.Logger.Log(TraceLevel.Verbose, "WebRequestClient: Got a malformed request: {0}", ex.Message);
}
}
HttpWebResponse response = null;
//pass the request to the target server
try
{
response = webRequest.GetResponse() as HttpWebResponse;
}
catch (WebException webException)
{
response = webException.Response as HttpWebResponse;
if (response == null)
{
throw webException;
}
}
HttpResponseInfo responseInfo = null;
//read response
if (response != null)
{
string responseLine = String.Format("HTTP/{0} {1} {2}\r\n", response.ProtocolVersion, (int)response.StatusCode, response.StatusDescription);
responseInfo = new HttpResponseInfo(responseLine);
AddResponseHeaders(response, responseInfo);
//add response body
string transferEncoding = response.Headers["Transfer-Encoding"];
responseInfo.ResponseBody.IsChunked = transferEncoding != null && String.Compare("chunked", transferEncoding, true) == 0;
Stream responseStream = response.GetResponseStream();
int read;
byte[] buffer = new byte[MAX_BUFFER_SIZE];
while ((read = responseStream.Read(buffer, 0, MAX_BUFFER_SIZE)) > 0)
{
responseInfo.ResponseBody.AddChunkReference(buffer, read);
}
response.Close();
}
return(responseInfo);
}