Esempio n. 1
0
        private WebRequest CreateProperRequestType(string url, IProxyDetails proxyDetails)
        {
            WebRequest result = null;

            if (proxyDetails != null)
            {
                if (proxyDetails.ProxyType == ProxyType.Proxy)
                {
                    result       = (HttpWebRequest)WebRequest.Create(url);
                    result.Proxy = new WebProxy(proxyDetails.FullProxyAddress);
                    ((HttpWebRequest)result).UserAgent = _userAgent;
                }
                else if (proxyDetails.ProxyType == ProxyType.Socks)
                {
                    result       = SocksHttpWebRequest.Create(url);
                    result.Proxy = new WebProxy(proxyDetails.FullProxyAddress);
                    //TODO: implement user and password
                }
                else if (proxyDetails.ProxyType == ProxyType.None)
                {
                    result = (HttpWebRequest)WebRequest.Create(url);
                    ((HttpWebRequest)result).UserAgent = _userAgent;
                }
            }
            else
            {
                result = (HttpWebRequest)WebRequest.Create(url);
                ((HttpWebRequest)result).UserAgent = _userAgent;
            }


            return(result);
        }
Esempio n. 2
0
        private WebRequest CreateProperRequestType(string url,IProxyDetails proxyDetails)
        {
            WebRequest result = null;

            if (proxyDetails != null)
            {
                if (proxyDetails.ProxyType == ProxyType.Proxy)
                {
                    result = (HttpWebRequest)WebRequest.Create(url);
                    result.Proxy = new WebProxy(proxyDetails.FullProxyAddress);
                    ((HttpWebRequest)result).UserAgent = _userAgent;
                }
                else if (proxyDetails.ProxyType == ProxyType.Socks)
                {
                    result = SocksHttpWebRequest.Create(url);
                    result.Proxy = new WebProxy(proxyDetails.FullProxyAddress);
                    //TODO: implement user and password

                }
                else if (proxyDetails.ProxyType == ProxyType.None)
                {
                    result = (HttpWebRequest)WebRequest.Create(url);
                    ((HttpWebRequest)result).UserAgent = _userAgent;
                }
            }
            else
            {
                result = (HttpWebRequest)WebRequest.Create(url);
                ((HttpWebRequest)result).UserAgent = _userAgent;
            }

            return result;
        }
Esempio n. 3
0
        public string GetPageHtml(string url, IProxyDetails proxyDetails)
        {
            string result = string.Empty;

            result = GetRequestResult(url, proxyDetails);
            return(result);
        }
Esempio n. 4
0
        private string GetRequestResult(string url, IProxyDetails proxyDetails)
        {
            string result  = string.Empty;

            WebResponse resp = null;
            WebRequest getRequest = CreateProperRequestType(url, proxyDetails);
            getRequest.Method = "GET";

            try
            {
                resp = getRequest.GetResponse();
            }
            catch (WebException wex)
            {
                if (wex.Status == WebExceptionStatus.ReceiveFailure)
                {
                    throw new HtmlObtainingException("Failed to receive html output while trying to obtain page result");
                }
                else if (wex.Status == WebExceptionStatus.Timeout)
                {
                    throw new HtmlObtainingException("Timeout occured while trying to obtain page result");
                }
                else if (wex.Status == WebExceptionStatus.ProtocolError)
                {
                    resp = wex.Response;
                    //when using Privoxy this is always triggered but I do get a page back from it
                    //the actual error is server returned 503
                    //throw new HtmlObtainingException("Protocol error while trying to obtain page result");
                }
                else
                {
                    throw new HtmlObtainingException("Unknown exception occured while trying to obtain page result");
                }
            }

            if (resp != null)
            {
                Encoding encoding = Encoding.Default;
                if(resp is HttpWebResponse)
                {
                    var enc = ((HttpWebResponse)resp).ContentEncoding;
                    if(!string.IsNullOrEmpty(enc))
                        encoding = Encoding.GetEncoding(enc);
                }
                else if (resp is SocksHttpWebResponse)
                {
                    encoding = ((SocksHttpWebResponse)resp).CorrectEncoding;
                }
                // Get the stream associated with the response.
                Stream receiveStream = resp.GetResponseStream();

                // Pipes the stream to a higher level stream reader with the required encoding format.
                StreamReader readStream = new StreamReader(receiveStream, encoding);

                result = readStream.ReadToEnd();
                resp.Close();
                readStream.Close();
            }

            return result;
        }
Esempio n. 5
0
 public string GetPageHtml(string url,IProxyDetails proxyDetails)
 {
     string result = string.Empty;
     result = GetRequestResult(url, proxyDetails);
     return result;
 }
Esempio n. 6
0
        public static IList <string> GoogleSearch(string search_expression, int nrResults, IProxyDetails proxyDetails, ref string googleError)
        {
            var wc = new SocksWebClient();

            //when I try to use socks here I get a html of the type this document has been moved
            wc.ProxyDetails = proxyDetails;
            string query = string.Empty;
            string html  = string.Empty;

            List <string>  results        = new List <string>();
            IList <string> curResultBatch = new List <string>();

            for (int curResultNr = 0; curResultNr < nrResults; curResultNr += GeneralParameters.GoogleResultsPerPage)
            {
                query          = "http://www.google.com/search?q=" + search_expression + "&start=" + curResultNr;
                html           = wc.DownloadString(query);
                curResultBatch = GetMultipleAnswersFromHtml(html, query,
                                                            new ExploitDetails()
                {
                    ResultStart = "<h3 class=\"r\"><a href=\"/url?q=",
                    ResultEnd   = "&amp;"
                },
                                                            true, Uri.UnescapeDataString);
                //true,true);

                results.AddRange(curResultBatch);
            }



            #region Using Google's API - doesn't work too well

            //var url_template = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&rsz=large&safe=active&q={0}&start={1}";
            //Uri search_url;
            //var results_list = new List<string>();
            //int[] offsets = { 0, 8, 16, 24, 32, 40, 48 };
            //foreach (var offset in offsets)
            //{
            //    search_url = new Uri(string.Format(url_template, search_expression, offset));

            //    var page = new WebClient().DownloadString(search_url);

            //    JObject o = (JObject)JsonConvert.DeserializeObject(page);

            //    if (o["responseStatus"].ToString() == "200")
            //    {
            //        var results_query =
            //            from result in o["responseData"]["results"].Children()
            //            select result.Value<string>("unescapedUrl").ToString();

            //        foreach (var result in results_query)
            //            results_list.Add(result);
            //    }
            //    else
            //        googleError = o["responseDetails"].ToString();
            //}

            #endregion Using Google's API - doesn't work too well

            return(results);
        }
Esempio n. 7
0
        public static IList<string> GoogleSearch(string search_expression,int nrResults,IProxyDetails proxyDetails,ref string googleError)
        {
            var wc = new SocksWebClient();
            //when I try to use socks here I get a html of the type this document has been moved
            wc.ProxyDetails = proxyDetails;
            string query = string.Empty;
            string html = string.Empty;

            List<string> results = new List<string>();
            IList<string> curResultBatch = new List<string>();

            for (int curResultNr = 0; curResultNr < nrResults; curResultNr += GeneralParameters.GoogleResultsPerPage)
            {
                query = "http://www.google.com/search?q=" + search_expression + "&start=" + curResultNr;
                html = wc.DownloadString(query);
                curResultBatch = GetMultipleAnswersFromHtml(html, query,
                                                                        new ExploitDetails()
                                                                        {
                                                                            ResultStart = "<h3 class=\"r\"><a href=\"/url?q=",
                                                                            ResultEnd = "&amp;"
                                                                        },
                                                                        true, Uri.UnescapeDataString);
                                                                        //true,true);

                results.AddRange(curResultBatch);
            }

            #region Using Google's API - doesn't work too well

            //var url_template = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&rsz=large&safe=active&q={0}&start={1}";
            //Uri search_url;
            //var results_list = new List<string>();
            //int[] offsets = { 0, 8, 16, 24, 32, 40, 48 };
            //foreach (var offset in offsets)
            //{
            //    search_url = new Uri(string.Format(url_template, search_expression, offset));

            //    var page = new WebClient().DownloadString(search_url);

            //    JObject o = (JObject)JsonConvert.DeserializeObject(page);

            //    if (o["responseStatus"].ToString() == "200")
            //    {
            //        var results_query =
            //            from result in o["responseData"]["results"].Children()
            //            select result.Value<string>("unescapedUrl").ToString();

            //        foreach (var result in results_query)
            //            results_list.Add(result);
            //    }
            //    else
            //        googleError = o["responseDetails"].ToString();
            //}

            #endregion Using Google's API - doesn't work too well

            return results;
        }
Esempio n. 8
0
        private string GetRequestResult(string url, IProxyDetails proxyDetails)
        {
            string result = string.Empty;

            WebResponse resp       = null;
            WebRequest  getRequest = CreateProperRequestType(url, proxyDetails);

            getRequest.Method = "GET";

            try
            {
                resp = getRequest.GetResponse();
            }
            catch (WebException wex)
            {
                if (wex.Status == WebExceptionStatus.ReceiveFailure)
                {
                    throw new HtmlObtainingException("Failed to receive html output while trying to obtain page result");
                }
                else if (wex.Status == WebExceptionStatus.Timeout)
                {
                    throw new HtmlObtainingException("Timeout occured while trying to obtain page result");
                }
                else if (wex.Status == WebExceptionStatus.ProtocolError)
                {
                    resp = wex.Response;
                    //when using Privoxy this is always triggered but I do get a page back from it
                    //the actual error is server returned 503
                    //throw new HtmlObtainingException("Protocol error while trying to obtain page result");
                }
                else
                {
                    throw new HtmlObtainingException("Unknown exception occured while trying to obtain page result");
                }
            }

            if (resp != null)
            {
                Encoding encoding = Encoding.Default;
                if (resp is HttpWebResponse)
                {
                    var enc = ((HttpWebResponse)resp).ContentEncoding;
                    if (!string.IsNullOrEmpty(enc))
                    {
                        encoding = Encoding.GetEncoding(enc);
                    }
                }
                else if (resp is SocksHttpWebResponse)
                {
                    encoding = ((SocksHttpWebResponse)resp).CorrectEncoding;
                }
                // Get the stream associated with the response.
                Stream receiveStream = resp.GetResponseStream();

                // Pipes the stream to a higher level stream reader with the required encoding format.
                StreamReader readStream = new StreamReader(receiveStream, encoding);

                result = readStream.ReadToEnd();
                resp.Close();
                readStream.Close();
            }

            return(result);
        }