private WebRequest CreateProperRequestType(string url, IProxyDetails proxyDetails) { WebRequest result = null; if (proxyDetails != null) { if (proxyDetails.ProxyType == ProxyType.Proxy) { result = (HttpWebRequest)WebRequest.Create(url); result.Proxy = new WebProxy(proxyDetails.FullProxyAddress); ((HttpWebRequest)result).UserAgent = _userAgent; } else if (proxyDetails.ProxyType == ProxyType.Socks) { result = SocksHttpWebRequest.Create(url); result.Proxy = new WebProxy(proxyDetails.FullProxyAddress); //TODO: implement user and password } else if (proxyDetails.ProxyType == ProxyType.None) { result = (HttpWebRequest)WebRequest.Create(url); ((HttpWebRequest)result).UserAgent = _userAgent; } } else { result = (HttpWebRequest)WebRequest.Create(url); ((HttpWebRequest)result).UserAgent = _userAgent; } return(result); }
private WebRequest CreateProperRequestType(string url,IProxyDetails proxyDetails) { WebRequest result = null; if (proxyDetails != null) { if (proxyDetails.ProxyType == ProxyType.Proxy) { result = (HttpWebRequest)WebRequest.Create(url); result.Proxy = new WebProxy(proxyDetails.FullProxyAddress); ((HttpWebRequest)result).UserAgent = _userAgent; } else if (proxyDetails.ProxyType == ProxyType.Socks) { result = SocksHttpWebRequest.Create(url); result.Proxy = new WebProxy(proxyDetails.FullProxyAddress); //TODO: implement user and password } else if (proxyDetails.ProxyType == ProxyType.None) { result = (HttpWebRequest)WebRequest.Create(url); ((HttpWebRequest)result).UserAgent = _userAgent; } } else { result = (HttpWebRequest)WebRequest.Create(url); ((HttpWebRequest)result).UserAgent = _userAgent; } return result; }
public string GetPageHtml(string url, IProxyDetails proxyDetails) { string result = string.Empty; result = GetRequestResult(url, proxyDetails); return(result); }
private string GetRequestResult(string url, IProxyDetails proxyDetails) { string result = string.Empty; WebResponse resp = null; WebRequest getRequest = CreateProperRequestType(url, proxyDetails); getRequest.Method = "GET"; try { resp = getRequest.GetResponse(); } catch (WebException wex) { if (wex.Status == WebExceptionStatus.ReceiveFailure) { throw new HtmlObtainingException("Failed to receive html output while trying to obtain page result"); } else if (wex.Status == WebExceptionStatus.Timeout) { throw new HtmlObtainingException("Timeout occured while trying to obtain page result"); } else if (wex.Status == WebExceptionStatus.ProtocolError) { resp = wex.Response; //when using Privoxy this is always triggered but I do get a page back from it //the actual error is server returned 503 //throw new HtmlObtainingException("Protocol error while trying to obtain page result"); } else { throw new HtmlObtainingException("Unknown exception occured while trying to obtain page result"); } } if (resp != null) { Encoding encoding = Encoding.Default; if(resp is HttpWebResponse) { var enc = ((HttpWebResponse)resp).ContentEncoding; if(!string.IsNullOrEmpty(enc)) encoding = Encoding.GetEncoding(enc); } else if (resp is SocksHttpWebResponse) { encoding = ((SocksHttpWebResponse)resp).CorrectEncoding; } // Get the stream associated with the response. Stream receiveStream = resp.GetResponseStream(); // Pipes the stream to a higher level stream reader with the required encoding format. StreamReader readStream = new StreamReader(receiveStream, encoding); result = readStream.ReadToEnd(); resp.Close(); readStream.Close(); } return result; }
public string GetPageHtml(string url,IProxyDetails proxyDetails) { string result = string.Empty; result = GetRequestResult(url, proxyDetails); return result; }
public static IList <string> GoogleSearch(string search_expression, int nrResults, IProxyDetails proxyDetails, ref string googleError) { var wc = new SocksWebClient(); //when I try to use socks here I get a html of the type this document has been moved wc.ProxyDetails = proxyDetails; string query = string.Empty; string html = string.Empty; List <string> results = new List <string>(); IList <string> curResultBatch = new List <string>(); for (int curResultNr = 0; curResultNr < nrResults; curResultNr += GeneralParameters.GoogleResultsPerPage) { query = "http://www.google.com/search?q=" + search_expression + "&start=" + curResultNr; html = wc.DownloadString(query); curResultBatch = GetMultipleAnswersFromHtml(html, query, new ExploitDetails() { ResultStart = "<h3 class=\"r\"><a href=\"/url?q=", ResultEnd = "&" }, true, Uri.UnescapeDataString); //true,true); results.AddRange(curResultBatch); } #region Using Google's API - doesn't work too well //var url_template = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&rsz=large&safe=active&q={0}&start={1}"; //Uri search_url; //var results_list = new List<string>(); //int[] offsets = { 0, 8, 16, 24, 32, 40, 48 }; //foreach (var offset in offsets) //{ // search_url = new Uri(string.Format(url_template, search_expression, offset)); // var page = new WebClient().DownloadString(search_url); // JObject o = (JObject)JsonConvert.DeserializeObject(page); // if (o["responseStatus"].ToString() == "200") // { // var results_query = // from result in o["responseData"]["results"].Children() // select result.Value<string>("unescapedUrl").ToString(); // foreach (var result in results_query) // results_list.Add(result); // } // else // googleError = o["responseDetails"].ToString(); //} #endregion Using Google's API - doesn't work too well return(results); }
public static IList<string> GoogleSearch(string search_expression,int nrResults,IProxyDetails proxyDetails,ref string googleError) { var wc = new SocksWebClient(); //when I try to use socks here I get a html of the type this document has been moved wc.ProxyDetails = proxyDetails; string query = string.Empty; string html = string.Empty; List<string> results = new List<string>(); IList<string> curResultBatch = new List<string>(); for (int curResultNr = 0; curResultNr < nrResults; curResultNr += GeneralParameters.GoogleResultsPerPage) { query = "http://www.google.com/search?q=" + search_expression + "&start=" + curResultNr; html = wc.DownloadString(query); curResultBatch = GetMultipleAnswersFromHtml(html, query, new ExploitDetails() { ResultStart = "<h3 class=\"r\"><a href=\"/url?q=", ResultEnd = "&" }, true, Uri.UnescapeDataString); //true,true); results.AddRange(curResultBatch); } #region Using Google's API - doesn't work too well //var url_template = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&rsz=large&safe=active&q={0}&start={1}"; //Uri search_url; //var results_list = new List<string>(); //int[] offsets = { 0, 8, 16, 24, 32, 40, 48 }; //foreach (var offset in offsets) //{ // search_url = new Uri(string.Format(url_template, search_expression, offset)); // var page = new WebClient().DownloadString(search_url); // JObject o = (JObject)JsonConvert.DeserializeObject(page); // if (o["responseStatus"].ToString() == "200") // { // var results_query = // from result in o["responseData"]["results"].Children() // select result.Value<string>("unescapedUrl").ToString(); // foreach (var result in results_query) // results_list.Add(result); // } // else // googleError = o["responseDetails"].ToString(); //} #endregion Using Google's API - doesn't work too well return results; }
private string GetRequestResult(string url, IProxyDetails proxyDetails) { string result = string.Empty; WebResponse resp = null; WebRequest getRequest = CreateProperRequestType(url, proxyDetails); getRequest.Method = "GET"; try { resp = getRequest.GetResponse(); } catch (WebException wex) { if (wex.Status == WebExceptionStatus.ReceiveFailure) { throw new HtmlObtainingException("Failed to receive html output while trying to obtain page result"); } else if (wex.Status == WebExceptionStatus.Timeout) { throw new HtmlObtainingException("Timeout occured while trying to obtain page result"); } else if (wex.Status == WebExceptionStatus.ProtocolError) { resp = wex.Response; //when using Privoxy this is always triggered but I do get a page back from it //the actual error is server returned 503 //throw new HtmlObtainingException("Protocol error while trying to obtain page result"); } else { throw new HtmlObtainingException("Unknown exception occured while trying to obtain page result"); } } if (resp != null) { Encoding encoding = Encoding.Default; if (resp is HttpWebResponse) { var enc = ((HttpWebResponse)resp).ContentEncoding; if (!string.IsNullOrEmpty(enc)) { encoding = Encoding.GetEncoding(enc); } } else if (resp is SocksHttpWebResponse) { encoding = ((SocksHttpWebResponse)resp).CorrectEncoding; } // Get the stream associated with the response. Stream receiveStream = resp.GetResponseStream(); // Pipes the stream to a higher level stream reader with the required encoding format. StreamReader readStream = new StreamReader(receiveStream, encoding); result = readStream.ReadToEnd(); resp.Close(); readStream.Close(); } return(result); }