protected IBrowsingResponse PrepareBrowsingResponse(HttpWebRequest req, IBrowsingSession info)
 {
     Stopwatch stopWatch = new Stopwatch();
     stopWatch.Start();
     try
     {
         var response = GetResponseFromRequest(req);
         stopWatch.Stop();
         return new HttpBrowsingResponse(response, info, stopWatch.Elapsed);
     }
     catch (WebException ex)
     {
         stopWatch.Stop();
         //if (ex.Status == WebExceptionStatus.ProtocolError)
         if (ex.Response != null)
         {
             return new HttpBrowsingResponse((HttpWebResponse)ex.Response, info, stopWatch.Elapsed);
         }
         else
         {
             return new ErrorBrowsingResponse()
             {
                 GenerationTime = stopWatch.Elapsed,
                 Headers = info.Headers,
                 ResponseContent = ex.Message,
                 ResponseUrl = req.RequestUri,
                 StatusCode = HttpStatusCode.ServiceUnavailable
             };
         }
     }
     catch (Exception ex)
     {
         throw new Exception("PrepareBrowsingResponse failed", ex);
     }
 }
        protected virtual Uri GetNextProxyFor(IBrowsingSession session, string descriminator)
        {
            //TODO: optimize by caching the state
            lock (getNextProxyForLocker)
            {
                var ignoreIds = _stateService.Get<List<int>>("proxy_" + descriminator);
                if (ignoreIds == null) ignoreIds = new List<int>();
                var proxy = WorkingProxies.FirstOrDefault(x => !ignoreIds.Contains(x.Id));
                if (proxy == null)
                {
                    _log.Warn("Proxy list ended and restared for " + descriminator);
                    proxy = WorkingProxies.First();
                    ignoreIds = new List<int>();
                }
                ignoreIds.Add(proxy.Id);

                var hostTrigger = RootDescriminatorStateTriggers
                    .GetOrAddChild(descriminator, String.Format(message, descriminator));

                _stateService.Put<List<int>>("proxy_" + descriminator,
                    ignoreIds, hostTrigger, (long) StoringDuration.TotalSeconds);

                //_log.Debug(String.Format("Proxy changed for {0} to {1} at {2}",
                //    descriminator, proxy, Thread.CurrentThread.ManagedThreadId));

                session.ClearCookieContainer();
                return new Uri(proxy.IP);
            }
        }
        public virtual IBrowsingResponse GetResponse(Uri httpUri, IBrowsingSession info)
        {
            var req = CreateRequest(httpUri, info);

            req.Method = "GET";
            return(PrepareBrowsingResponse(req, info));
        }
        protected virtual HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info)
        {
            HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(httpUrl);

            req.Timeout = (int)info.Timeout;

            req.Credentials = System.Net.CredentialCache.DefaultCredentials;
            req.CookieContainer = info.CookieContainer;
            foreach(var hKey in info.Headers.AllKeys) {
                switch (hKey)
                {
                    //case "Date": break;
                    //case "Host": break;
                    //case "Range": break;
                    //case "Proxy-Connection": break;
                    //case "Version": break;
                    //case "Content-Length": req.ContentLength = info.Headers[hKey]; break;
                    //case "If-Modified-Since": req.IfModifiedSince = info.Headers[hKey]; break;
                    case "Accept": req.Accept = info.Headers[hKey]; break;
                    case "Connection": req.Connection = info.Headers[hKey]; break;
                    case "Content-Type": req.ContentType = info.Headers[hKey]; break;
                    case "Expect": req.Expect = info.Headers[hKey]; break;
                    case "Referer": req.Referer = info.Headers[hKey]; ; break;
                    case "Transfer-Encoding": req.TransferEncoding = info.Headers[hKey]; break;
                    case "User-Agent": req.UserAgent = info.Headers[hKey]; break;
                    default:
                        req.Headers.Add(hKey, info.Headers[hKey]);
                        break;
                }
                
            }
            return req;
        }
        protected virtual Uri GetNextProxyFor(IBrowsingSession session, string descriminator)
        {
            //TODO: optimize by caching the state
            lock (getNextProxyForLocker)
            {
                var ignoreIds = _stateService.Get <List <int> >("proxy_" + descriminator);
                if (ignoreIds == null)
                {
                    ignoreIds = new List <int>();
                }
                var proxy = WorkingProxies.FirstOrDefault(x => !ignoreIds.Contains(x.Id));
                if (proxy == null)
                {
                    _log.Warn("Proxy list ended and restared for " + descriminator);
                    proxy     = WorkingProxies.First();
                    ignoreIds = new List <int>();
                }
                ignoreIds.Add(proxy.Id);

                var hostTrigger = RootDescriminatorStateTriggers
                                  .GetOrAddChild(descriminator, String.Format(message, descriminator));

                _stateService.Put <List <int> >("proxy_" + descriminator,
                                                ignoreIds, hostTrigger, (long)StoringDuration.TotalSeconds);

                //_log.Debug(String.Format("Proxy changed for {0} to {1} at {2}",
                //    descriminator, proxy, Thread.CurrentThread.ManagedThreadId));

                session.ClearCookieContainer();
                return(new Uri(proxy.IP));
            }
        }
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     if (String.IsNullOrEmpty(Descriminator)) Descriminator = uri.Host;
     if (session.CurrentProxy == null) session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator));
     retries = 0;
     return base.OnBeforeRequestStop(session, uri);
 }
 public override void OnInit(IBrowsingSession session)
 {
     WorkingProxies = _repo.GetWorking();
     if (WorkingProxies.Count == 0)
     {
         throw new Exception("No working proxies found");
     }
 }
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     if(LastRequest!=null)
     {
         session.Headers[HEADER_REFERRER] = LastRequest.ToString();
         //String.Format(@"{0}://{1}/", uri.Scheme, uri.Host);
     }
     return base.OnBeforeRequestStop(session, uri);
 }
 internal Uri GetCurrentProxyFor(IBrowsingSession session, string descriminator)
 {
     if(!ThreadProxies.Keys.Contains(Thread.CurrentThread.ManagedThreadId))
     {
         if (session == null || descriminator == null) return null;
          ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(session,descriminator);
     }
     return ThreadProxies[Thread.CurrentThread.ManagedThreadId];
 }
 public override void OnInit(IBrowsingSession session)
 {
     session.Headers["User-Agent"]      = @"Mozilla/5.0 (X11; U; Linux i686; de-AT; rv:1.8.0.2) Gecko/20060309 SeaMonkey/1.0";
     session.Headers["Accept"]          = @"image/png,image/*;q=0.8,*/*;q=0.5";
     session.Headers["Accept-Language"] = @"en-us,en;q=0.5";
     //session.Headers["Accept-Encoding"] = @"gzip,deflate";
     session.Headers["Accept-Charset"] = @"utf-8;q=0.7,*;q=0.7";
     session.Headers["Keep-Alive"]     = @"115";
 }
        protected override HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info)
        {
            var req = base.CreateRequest(httpUrl, info);

            var proxy = new WebProxy(proxyAddr);
            req.Proxy = proxy;

            return req;
        }
 public override bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri)
 {
     if (String.IsNullOrEmpty(Descriminator))
     {
         Descriminator = uri.Host;
     }
     browsingSession.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(browsingSession, Descriminator));
     return(true);
 }
 public override bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri)
 {
     if (String.IsNullOrEmpty(Descriminator))
     {
         Descriminator = uri.Host;
     }
     ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(browsingSession, Descriminator);
     return(true);
 }
 public override void OnInit(IBrowsingSession session)
 {
     session.Headers["User-Agent"] = @"Mozilla/5.0 (X11; U; Linux i686; de-AT; rv:1.8.0.2) Gecko/20060309 SeaMonkey/1.0";
     session.Headers["Accept"] = @"image/png,image/*;q=0.8,*/*;q=0.5";
     session.Headers["Accept-Language"] = @"en-us,en;q=0.5";
     //session.Headers["Accept-Encoding"] = @"gzip,deflate";
     session.Headers["Accept-Charset"] = @"utf-8;q=0.7,*;q=0.7";
     session.Headers["Keep-Alive"] = @"115";
 }
Esempio n. 15
0
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     if (LastRequest != null)
     {
         session.Headers[HEADER_REFERRER] = LastRequest.ToString();
         //String.Format(@"{0}://{1}/", uri.Scheme, uri.Host);
     }
     return(base.OnBeforeRequestStop(session, uri));
 }
 public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
 {
     if (retries < Attempts && response.StatusCode == HttpStatusCode.ServiceUnavailable)
     {
         retries++;
         session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator));
         return(true);
     }
     return(base.OnAfterRequestRerun(session, uri, response));
 }
        protected override HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info)
        {
            var req = base.CreateRequest(httpUrl, info);

            var proxy = new WebProxy(proxyAddr);

            req.Proxy = proxy;

            return(req);
        }
 public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
 {
     if (retries < Attempts && response.StatusCode == HttpStatusCode.ServiceUnavailable)
     {
         retries++;
         Thread.Sleep(SleepInterval);
         return(true);
     }
     return(base.OnAfterRequestRerun(session, uri, response));
 }
 public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
 {
     if (retries < Attempts&&response.StatusCode == HttpStatusCode.ServiceUnavailable)
     {
         retries++;
         Thread.Sleep(SleepInterval);
         return true;
     }
     return base.OnAfterRequestRerun(session, uri, response);
 }
        protected override HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info)
        {
            var req = base.CreateRequest(httpUrl, info);
            var proxyIp = parallelHttpProxyDecorator.GetCurrentProxyFor(info, httpUrl.Host);

            var proxy = new WebProxy(proxyIp);
            req.Proxy = proxy;

            return req;
        }
 public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
 {
     if (retries < Attempts&&response.StatusCode == HttpStatusCode.ServiceUnavailable)
     {
         retries++;
         session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator));
         return true;
     }
     return base.OnAfterRequestRerun(session, uri, response);
 }
Esempio n. 22
0
        protected override HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info)
        {
            var req     = base.CreateRequest(httpUrl, info);
            var proxyIp = parallelHttpProxyDecorator.GetCurrentProxyFor(info, httpUrl.Host);

            var proxy = new WebProxy(proxyIp);

            req.Proxy = proxy;

            return(req);
        }
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     if (String.IsNullOrEmpty(Descriminator))
     {
         Descriminator = uri.Host;
     }
     if (session.CurrentProxy == null)
     {
         session.CurrentProxy = proxySwitcher;
     }
     return(base.OnBeforeRequestStop(session, uri));
 }
 internal Uri GetCurrentProxyFor(IBrowsingSession session, string descriminator)
 {
     if (!ThreadProxies.Keys.Contains(Thread.CurrentThread.ManagedThreadId))
     {
         if (session == null || descriminator == null)
         {
             return(null);
         }
         ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(session, descriminator);
     }
     return(ThreadProxies[Thread.CurrentThread.ManagedThreadId]);
 }
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     if (String.IsNullOrEmpty(Descriminator))
     {
         Descriminator = uri.Host;
     }
     if (session.CurrentProxy == null)
     {
         session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator));
     }
     retries = 0;
     return(base.OnBeforeRequestStop(session, uri));
 }
        protected virtual HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info)
        {
            HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(httpUrl);

            req.Timeout = (int)info.Timeout;

            req.Credentials     = System.Net.CredentialCache.DefaultCredentials;
            req.CookieContainer = info.CookieContainer;
            foreach (var hKey in info.Headers.AllKeys)
            {
                switch (hKey)
                {
                //case "Date": break;
                //case "Host": break;
                //case "Range": break;
                //case "Proxy-Connection": break;
                //case "Version": break;
                //case "Content-Length": req.ContentLength = info.Headers[hKey]; break;
                //case "If-Modified-Since": req.IfModifiedSince = info.Headers[hKey]; break;
                case "Accept": req.Accept = info.Headers[hKey]; break;

                case "Connection": req.Connection = info.Headers[hKey]; break;

                case "Content-Type": req.ContentType = info.Headers[hKey]; break;

                case "Expect": req.Expect = info.Headers[hKey]; break;

                case "Referer": req.Referer = info.Headers[hKey];; break;

                case "Transfer-Encoding": req.TransferEncoding = info.Headers[hKey]; break;

                case "User-Agent": req.UserAgent = info.Headers[hKey]; break;

                default:
                    req.Headers.Add(hKey, info.Headers[hKey]);
                    break;
                }
            }
            return(req);
        }
        protected IBrowsingResponse PrepareBrowsingResponse(HttpWebRequest req, IBrowsingSession info)
        {
            Stopwatch stopWatch = new Stopwatch();

            stopWatch.Start();
            try
            {
                var response = GetResponseFromRequest(req);
                stopWatch.Stop();
                return(new HttpBrowsingResponse(response, info, stopWatch.Elapsed));
            }
            catch (WebException ex)
            {
                stopWatch.Stop();
                //if (ex.Status == WebExceptionStatus.ProtocolError)
                if (ex.Response != null)
                {
                    return(new HttpBrowsingResponse((HttpWebResponse)ex.Response, info, stopWatch.Elapsed));
                }
                else
                {
                    return(new ErrorBrowsingResponse()
                    {
                        GenerationTime = stopWatch.Elapsed,
                        Headers = info.Headers,
                        ResponseContent = ex.Message,
                        ResponseUrl = req.RequestUri,
                        StatusCode = HttpStatusCode.ServiceUnavailable
                    });
                }
            }
            catch (Exception ex)
            {
                throw new Exception("PrepareBrowsingResponse failed", ex);
            }
        }
 protected override Uri GetNextProxyFor(IBrowsingSession session, string descriminator)
 {
     ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(session, descriminator);
     return ThreadProxies[Thread.CurrentThread.ManagedThreadId];
 }
 public override void OnInit(IBrowsingSession session)
 {
     WorkingProxies = _repo.GetWorking();
     if(WorkingProxies.Count==0) throw new Exception("No working proxies found");
 }
        public virtual IBrowsingResponse FilePostResponse(Uri httpUri, System.Collections.Specialized.NameValueCollection postParamz, List<Tuple<string, string, System.IO.Stream>> files, IBrowsingSession info)
        {
            string boundary = "----------------------------" + DateTime.Now.Ticks.ToString("x");
            HttpWebRequest req = CreateRequest(httpUri, info);
            req.Timeout = req.Timeout*(files.Count + 1);
            req.ContentType = "multipart/form-data; boundary=" + boundary;
            req.Method = "POST";
            req.KeepAlive = true;
            Stream memStream = new System.IO.MemoryStream();

            byte[] boundarybytes = System.Text.Encoding.ASCII.GetBytes("\r\n--" + boundary + "\r\n");
            string formdataTemplate = "\r\n--" + boundary + "\r\nContent-Disposition: form-data; name=\"{0}\";\r\n\r\n{1}";
            //--- params
            foreach (string key in postParamz.Keys)
            {
                string formitem = string.Format(formdataTemplate, key, postParamz[key]);
                byte[] formitembytes = System.Text.Encoding.UTF8.GetBytes(formitem);
                memStream.Write(formitembytes, 0, formitembytes.Length);
            }
            memStream.Write(boundarybytes, 0, boundarybytes.Length);
            // -- files
            string headerTemplate = "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\n Content-Type: application/octet-stream\r\n\r\n";
            foreach (var fileData in files)
            {
                //string header = string.Format(headerTemplate, "file" + i, fileData.Key);
                string header = string.Format(headerTemplate, fileData.Item1, fileData.Item2);
                byte[] headerbytes = System.Text.Encoding.UTF8.GetBytes(header);
                memStream.Write(headerbytes, 0, headerbytes.Length);

                byte[] buffer = new byte[1024];
                int bytesRead = 0;
                while ((bytesRead = fileData.Item3.Read(buffer, 0, buffer.Length)) != 0)
                {
                    memStream.Write(buffer, 0, bytesRead);
                }
                memStream.Write(boundarybytes, 0, boundarybytes.Length);
            }


            req.ContentLength = memStream.Length;
            Stream requestStream = req.GetRequestStream();

            memStream.Position = 0;
            byte[] tempBuffer = new byte[memStream.Length];
            memStream.Read(tempBuffer, 0, tempBuffer.Length);
            memStream.Close();
            requestStream.Write(tempBuffer, 0, tempBuffer.Length);
            requestStream.Close();

            return PrepareBrowsingResponse(req, info);
        }
 public virtual IBrowsingResponse GetResponse(Uri httpUri, IBrowsingSession info)
 {
     var req = CreateRequest(httpUri, info);
     req.Method = "GET";
     return PrepareBrowsingResponse(req, info);
 }
 public override IBrowsingResponse GetResponse(Uri httpUri, IBrowsingSession info)
 {
     var result = base.GetResponse(httpUri, info);
     return result;
 }
Esempio n. 33
0
 public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
 {
     LastRequest = uri;
     return(base.OnAfterRequestRerun(session, uri, response));
 }
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     Thread.Sleep(SleepTime);
     return(base.OnBeforeRequestStop(session, uri));
 }
 public virtual bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     return false;
 }
 public override string GetCurrentIp(Uri httpUrl, IBrowsingSession info)
 {
     return proxyAddr.Host;
 }
Esempio n. 37
0
        public override string GetCurrentIp(Uri httpUrl, IBrowsingSession info)
        {
            var proxy = parallelHttpProxyDecorator.GetCurrentProxyFor(info, httpUrl.Host);

            return(proxy.Host);
        }
 public virtual void OnInit(IBrowsingSession session)
 {
 }
Esempio n. 39
0
        public override IBrowsingResponse GetResponse(Uri httpUri, IBrowsingSession info)
        {
            var result = base.GetResponse(httpUri, info);

            return(result);
        }
 public HttpBrowsingResponse(HttpWebResponse response, IBrowsingSession info, TimeSpan generationTime)
 {
     this.info = info;
     this.response = response;
     this.generationTime = generationTime;
 }
 public virtual string GetCurrentIp(Uri httpUrl, IBrowsingSession info)
 {
     return(NetworkCommons.GetAllUnicastAddresses()[0].ToString());
 }
 public virtual bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
 {
     return false;
 }
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     Thread.Sleep(SleepTime);
     return base.OnBeforeRequestStop(session, uri);
 }
 public virtual bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri)
 {
     return false;
 }
 public override string GetCurrentIp(Uri httpUrl, IBrowsingSession info)
 {
     var proxy = parallelHttpProxyDecorator.GetCurrentProxyFor(info, httpUrl.Host);
     return proxy.Host;
 }
 public virtual void OnInit(IBrowsingSession session) { }
 public override bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri)
 {
     if (String.IsNullOrEmpty(Descriminator)) Descriminator = uri.Host;
     browsingSession.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(browsingSession, Descriminator));
     return true;
 }
 public virtual void OnRemove(IBrowsingSession session) {}
        public virtual IBrowsingResponse PostResponse(Uri httpUri, System.Collections.Specialized.NameValueCollection postParamz, IBrowsingSession info)
        {
            var req = CreateRequest(httpUri, info);
            req.Method = "POST";

            ASCIIEncoding encoding = new ASCIIEncoding();
            string postData = "";
            if (postParamz.Count > 0)
            {
                postData = string.Join("&", Array.ConvertAll(postParamz.AllKeys, key => string.Format("{0}={1}", HttpUtility.UrlEncode(key), HttpUtility.UrlEncode(postParamz[key]))));
            }
            byte[] data = encoding.GetBytes(postData);

            req.ContentType = "application/x-www-form-urlencoded";
            req.ContentLength = data.Length;
            Stream newStream = req.GetRequestStream();
            newStream.Write(data, 0, data.Length);
            newStream.Close();
            return PrepareBrowsingResponse(req, info);
        }
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     retries = 0;
     return(base.OnBeforeRequestStop(session, uri));
 }
 public virtual string GetCurrentIp(Uri httpUrl, IBrowsingSession info)
 {
      return NetworkCommons.GetAllUnicastAddresses()[0].ToString();
 }
 protected override Uri GetNextProxyFor(IBrowsingSession session, string descriminator)
 {
     ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(session, descriminator);
     return(ThreadProxies[Thread.CurrentThread.ManagedThreadId]);
 }
 public virtual bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response)
 {
     return(false);
 }
Esempio n. 54
0
 public HttpBrowsingResponse(HttpWebResponse response, IBrowsingSession info, TimeSpan generationTime)
 {
     this.info           = info;
     this.response       = response;
     this.generationTime = generationTime;
 }
 public virtual bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     return(false);
 }
 public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri)
 {
     if (String.IsNullOrEmpty(Descriminator)) Descriminator = uri.Host;
     if (session.CurrentProxy == null) session.CurrentProxy = proxySwitcher;
     return base.OnBeforeRequestStop(session, uri);
 }
 public virtual bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri)
 {
     return(false);
 }
 public override bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri)
 {
     if (String.IsNullOrEmpty(Descriminator)) Descriminator = uri.Host;
     ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(browsingSession, Descriminator);
     return true;
 }
 public virtual void OnRemove(IBrowsingSession session)
 {
 }
 public override string GetCurrentIp(Uri httpUrl, IBrowsingSession info)
 {
     return(proxyAddr.Host);
 }