private void ExtractForms(string fileContents) { MatchCollection matches = Regex.Matches(fileContents, "(?si)<form.*?</form>"); foreach (Match m in matches) { var form = m.Value; //extract the form uri and method var method = Utils.RegexFirstGroupValue(form, "(?si)method\\s*=\\s*['\"]?(\\w+)").ToLower(); if (String.IsNullOrWhiteSpace(method)) { method = "get"; } var url = Utils.RegexFirstGroupValue(form, "(?si)action\\s*=\\s*['\"]?(\\w+)"); //if the url contains script in it or is malformed skip it if (!Uri.IsWellFormedUriString(url, UriKind.RelativeOrAbsolute)) { continue; } Uri formUri = _rootUri; if (!String.IsNullOrWhiteSpace(url)) { formUri = new Uri(url); if (!formUri.IsAbsoluteUri) { formUri = new Uri(_rootUri, formUri); } } //extract form fields var fieldMatches = Regex.Matches(form, "(?si)<input.*?</input>"); HttpRequestInfo formReqInfo; if (method.Equals("get")) { formReqInfo = new HttpRequestInfo(String.Format("GET {0} HTTP/1.1\r\n{1}\r\n\r\n", formUri.PathAndQuery, _headers), true); } else { formReqInfo = new HttpRequestInfo(String.Format("POST {0} HTTP/1.1\r\n{1}\r\n", formUri.PathAndQuery, _headers), true); formReqInfo.Headers["Content-Type"] = "application/x-www-form-urlencoded"; } formReqInfo.Host = formUri.Host; formReqInfo.Port = formUri.Port; foreach (Match fm in fieldMatches) { //get input name and value var input = fm.Value; var name = Utils.RegexFirstGroupValue(form, "(?si)name\\s*=\\s*['\"]?(\\w+)"); var value = Utils.RegexFirstGroupValue(form, "(?si)value\\s*=\\s*['\"]?(\\w+)"); if (method.Equals("get")) { formReqInfo.QueryVariables.Add(name, value); } else { formReqInfo.BodyVariables.Add(name, value); } } //finally if (method.Equals("get"))//add to the list of urls { url = formReqInfo.FullUrl; if (!_foundUrls.Contains(url)) { _foundUrls.Add(url); } } else { //add the post request on the spot TVRequestInfo reqInfo = _curDataAccessor.GetRequestInfo( _curDataAccessor.AddRequestResponse(formReqInfo.ToString(), "") ); reqInfo.IsHttps = formUri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase); _curDataAccessor.UpdateRequestInfo(reqInfo); } } }
/// <summary> /// If the response obtained is different than original attempt to update the path and resend to obtain a good response /// </summary> /// <param name="responseInfo"></param> /// <returns></returns> protected override HttpResponseInfo OnBeforeResponseToClient(HttpResponseInfo responseInfo) { responseInfo = base.OnBeforeResponseToClient(responseInfo); if (_trackingReqInfo != null) { //this is a tracked request we may need to update it if (IsDifferentThanOriginalResponse(responseInfo)) { //we need to obtain the referer response and update the request if (_trackingReqInfo.RefererId > -1 && !String.IsNullOrWhiteSpace(_trackingReqInfo.RequestContext)) { //load the referer request var refererInfo = _dataStore.GetRequestInfo(_trackingReqInfo.RefererId); var refererReqBytes = _dataStore.LoadRequestData(_trackingReqInfo.RefererId); if (refererReqBytes != null && refererReqBytes.Length > 0) { var refererHttpInfo = new HttpRequestInfo(refererReqBytes); refererHttpInfo.IsSecure = refererInfo.IsHttps; //update request cookie headers from the current request refererHttpInfo.Cookies.Clear(); refererHttpInfo.Headers["Cookie"] = _requestInfo.Headers["Cookie"]; //now that the referer request is ready send it to the tracking proxy where if the response is different than the original it will in turn //get its referer resent IHttpClient trackingProxyClient = GetTrackingProxyHttpClient(); //following lines equivalent to a recursive call var refererUpdatedResponse = trackingProxyClient.SendRequest(refererHttpInfo); string refererRawResponse = refererUpdatedResponse.ToString(); //update autotrack traccking patterns string updatedRequestValue = Utils.RegexFirstGroupValue(refererRawResponse, _trackingReqInfo.RequestContext); if (!String.IsNullOrWhiteSpace(updatedRequestValue) && AutoTrackingPatternList.ContainsKey(_trackingReqInfo.TrackingPattern)) { TrackingPattern curPattern = AutoTrackingPatternList[_trackingReqInfo.TrackingPattern]; //update request _originalRawRequest = Utils.ReplaceGroups(_originalRawRequest, curPattern.RequestPattern, updatedRequestValue); } else { HttpServerConsole.Instance.WriteLine(LogMessageType.Error, "Could not update autodetect parameter for tracked request id {0}. Will resend anyway", _trackingReqInfo.Id); } //update patterns string updatedRawRequest = PatternTracker.Instance.UpdateRequest(_originalRawRequest); updatedRawRequest = UpdateDynamicPatterns(updatedRawRequest); HttpRequestInfo newRequest = new HttpRequestInfo(updatedRawRequest); newRequest.IsSecure = _requestInfo.IsSecure; AddSpecialHeader(newRequest); if (!_trackingReqInfo.UpdatedPath.Equals(newRequest.Path)) { _trackingReqInfo.UpdatedPath = newRequest.Path; //update the path in the ui _dataStore.UpdateRequestInfo(_trackingReqInfo); //now send the request one more time } //send the request with the normal HTTP client this time responseInfo = HttpClient.SendRequest(newRequest); //no matter what we return the last value PatternTracker.Instance.UpdatePatternValues(responseInfo); } else { HttpServerConsole.Instance.WriteLine(LogMessageType.Error, "Missing http traffic for request id {0}", _trackingReqInfo.Id); } } else { HttpServerConsole.Instance.WriteLine(LogMessageType.Error, "Missing referer or request context for request id {0}", _trackingReqInfo.Id); } } else { HttpServerConsole.Instance.WriteLine(LogMessageType.Information, "Response similar for request id {0}", _trackingReqInfo.Id); } } return(responseInfo); }