protected IBrowsingResponse PrepareBrowsingResponse(HttpWebRequest req, IBrowsingSession info) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); try { var response = GetResponseFromRequest(req); stopWatch.Stop(); return new HttpBrowsingResponse(response, info, stopWatch.Elapsed); } catch (WebException ex) { stopWatch.Stop(); //if (ex.Status == WebExceptionStatus.ProtocolError) if (ex.Response != null) { return new HttpBrowsingResponse((HttpWebResponse)ex.Response, info, stopWatch.Elapsed); } else { return new ErrorBrowsingResponse() { GenerationTime = stopWatch.Elapsed, Headers = info.Headers, ResponseContent = ex.Message, ResponseUrl = req.RequestUri, StatusCode = HttpStatusCode.ServiceUnavailable }; } } catch (Exception ex) { throw new Exception("PrepareBrowsingResponse failed", ex); } }
protected virtual Uri GetNextProxyFor(IBrowsingSession session, string descriminator) { //TODO: optimize by caching the state lock (getNextProxyForLocker) { var ignoreIds = _stateService.Get<List<int>>("proxy_" + descriminator); if (ignoreIds == null) ignoreIds = new List<int>(); var proxy = WorkingProxies.FirstOrDefault(x => !ignoreIds.Contains(x.Id)); if (proxy == null) { _log.Warn("Proxy list ended and restared for " + descriminator); proxy = WorkingProxies.First(); ignoreIds = new List<int>(); } ignoreIds.Add(proxy.Id); var hostTrigger = RootDescriminatorStateTriggers .GetOrAddChild(descriminator, String.Format(message, descriminator)); _stateService.Put<List<int>>("proxy_" + descriminator, ignoreIds, hostTrigger, (long) StoringDuration.TotalSeconds); //_log.Debug(String.Format("Proxy changed for {0} to {1} at {2}", // descriminator, proxy, Thread.CurrentThread.ManagedThreadId)); session.ClearCookieContainer(); return new Uri(proxy.IP); } }
public virtual IBrowsingResponse GetResponse(Uri httpUri, IBrowsingSession info) { var req = CreateRequest(httpUri, info); req.Method = "GET"; return(PrepareBrowsingResponse(req, info)); }
protected virtual HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info) { HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(httpUrl); req.Timeout = (int)info.Timeout; req.Credentials = System.Net.CredentialCache.DefaultCredentials; req.CookieContainer = info.CookieContainer; foreach(var hKey in info.Headers.AllKeys) { switch (hKey) { //case "Date": break; //case "Host": break; //case "Range": break; //case "Proxy-Connection": break; //case "Version": break; //case "Content-Length": req.ContentLength = info.Headers[hKey]; break; //case "If-Modified-Since": req.IfModifiedSince = info.Headers[hKey]; break; case "Accept": req.Accept = info.Headers[hKey]; break; case "Connection": req.Connection = info.Headers[hKey]; break; case "Content-Type": req.ContentType = info.Headers[hKey]; break; case "Expect": req.Expect = info.Headers[hKey]; break; case "Referer": req.Referer = info.Headers[hKey]; ; break; case "Transfer-Encoding": req.TransferEncoding = info.Headers[hKey]; break; case "User-Agent": req.UserAgent = info.Headers[hKey]; break; default: req.Headers.Add(hKey, info.Headers[hKey]); break; } } return req; }
protected virtual Uri GetNextProxyFor(IBrowsingSession session, string descriminator) { //TODO: optimize by caching the state lock (getNextProxyForLocker) { var ignoreIds = _stateService.Get <List <int> >("proxy_" + descriminator); if (ignoreIds == null) { ignoreIds = new List <int>(); } var proxy = WorkingProxies.FirstOrDefault(x => !ignoreIds.Contains(x.Id)); if (proxy == null) { _log.Warn("Proxy list ended and restared for " + descriminator); proxy = WorkingProxies.First(); ignoreIds = new List <int>(); } ignoreIds.Add(proxy.Id); var hostTrigger = RootDescriminatorStateTriggers .GetOrAddChild(descriminator, String.Format(message, descriminator)); _stateService.Put <List <int> >("proxy_" + descriminator, ignoreIds, hostTrigger, (long)StoringDuration.TotalSeconds); //_log.Debug(String.Format("Proxy changed for {0} to {1} at {2}", // descriminator, proxy, Thread.CurrentThread.ManagedThreadId)); session.ClearCookieContainer(); return(new Uri(proxy.IP)); } }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { if (String.IsNullOrEmpty(Descriminator)) Descriminator = uri.Host; if (session.CurrentProxy == null) session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator)); retries = 0; return base.OnBeforeRequestStop(session, uri); }
public override void OnInit(IBrowsingSession session) { WorkingProxies = _repo.GetWorking(); if (WorkingProxies.Count == 0) { throw new Exception("No working proxies found"); } }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { if(LastRequest!=null) { session.Headers[HEADER_REFERRER] = LastRequest.ToString(); //String.Format(@"{0}://{1}/", uri.Scheme, uri.Host); } return base.OnBeforeRequestStop(session, uri); }
internal Uri GetCurrentProxyFor(IBrowsingSession session, string descriminator) { if(!ThreadProxies.Keys.Contains(Thread.CurrentThread.ManagedThreadId)) { if (session == null || descriminator == null) return null; ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(session,descriminator); } return ThreadProxies[Thread.CurrentThread.ManagedThreadId]; }
public override void OnInit(IBrowsingSession session) { session.Headers["User-Agent"] = @"Mozilla/5.0 (X11; U; Linux i686; de-AT; rv:1.8.0.2) Gecko/20060309 SeaMonkey/1.0"; session.Headers["Accept"] = @"image/png,image/*;q=0.8,*/*;q=0.5"; session.Headers["Accept-Language"] = @"en-us,en;q=0.5"; //session.Headers["Accept-Encoding"] = @"gzip,deflate"; session.Headers["Accept-Charset"] = @"utf-8;q=0.7,*;q=0.7"; session.Headers["Keep-Alive"] = @"115"; }
protected override HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info) { var req = base.CreateRequest(httpUrl, info); var proxy = new WebProxy(proxyAddr); req.Proxy = proxy; return req; }
public override bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri) { if (String.IsNullOrEmpty(Descriminator)) { Descriminator = uri.Host; } browsingSession.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(browsingSession, Descriminator)); return(true); }
public override bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri) { if (String.IsNullOrEmpty(Descriminator)) { Descriminator = uri.Host; } ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(browsingSession, Descriminator); return(true); }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { if (LastRequest != null) { session.Headers[HEADER_REFERRER] = LastRequest.ToString(); //String.Format(@"{0}://{1}/", uri.Scheme, uri.Host); } return(base.OnBeforeRequestStop(session, uri)); }
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response) { if (retries < Attempts && response.StatusCode == HttpStatusCode.ServiceUnavailable) { retries++; session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator)); return(true); } return(base.OnAfterRequestRerun(session, uri, response)); }
protected override HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info) { var req = base.CreateRequest(httpUrl, info); var proxy = new WebProxy(proxyAddr); req.Proxy = proxy; return(req); }
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response) { if (retries < Attempts && response.StatusCode == HttpStatusCode.ServiceUnavailable) { retries++; Thread.Sleep(SleepInterval); return(true); } return(base.OnAfterRequestRerun(session, uri, response)); }
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response) { if (retries < Attempts&&response.StatusCode == HttpStatusCode.ServiceUnavailable) { retries++; Thread.Sleep(SleepInterval); return true; } return base.OnAfterRequestRerun(session, uri, response); }
protected override HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info) { var req = base.CreateRequest(httpUrl, info); var proxyIp = parallelHttpProxyDecorator.GetCurrentProxyFor(info, httpUrl.Host); var proxy = new WebProxy(proxyIp); req.Proxy = proxy; return req; }
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response) { if (retries < Attempts&&response.StatusCode == HttpStatusCode.ServiceUnavailable) { retries++; session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator)); return true; } return base.OnAfterRequestRerun(session, uri, response); }
protected override HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info) { var req = base.CreateRequest(httpUrl, info); var proxyIp = parallelHttpProxyDecorator.GetCurrentProxyFor(info, httpUrl.Host); var proxy = new WebProxy(proxyIp); req.Proxy = proxy; return(req); }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { if (String.IsNullOrEmpty(Descriminator)) { Descriminator = uri.Host; } if (session.CurrentProxy == null) { session.CurrentProxy = proxySwitcher; } return(base.OnBeforeRequestStop(session, uri)); }
internal Uri GetCurrentProxyFor(IBrowsingSession session, string descriminator) { if (!ThreadProxies.Keys.Contains(Thread.CurrentThread.ManagedThreadId)) { if (session == null || descriminator == null) { return(null); } ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(session, descriminator); } return(ThreadProxies[Thread.CurrentThread.ManagedThreadId]); }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { if (String.IsNullOrEmpty(Descriminator)) { Descriminator = uri.Host; } if (session.CurrentProxy == null) { session.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(session, Descriminator)); } retries = 0; return(base.OnBeforeRequestStop(session, uri)); }
protected virtual HttpWebRequest CreateRequest(Uri httpUrl, IBrowsingSession info) { HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(httpUrl); req.Timeout = (int)info.Timeout; req.Credentials = System.Net.CredentialCache.DefaultCredentials; req.CookieContainer = info.CookieContainer; foreach (var hKey in info.Headers.AllKeys) { switch (hKey) { //case "Date": break; //case "Host": break; //case "Range": break; //case "Proxy-Connection": break; //case "Version": break; //case "Content-Length": req.ContentLength = info.Headers[hKey]; break; //case "If-Modified-Since": req.IfModifiedSince = info.Headers[hKey]; break; case "Accept": req.Accept = info.Headers[hKey]; break; case "Connection": req.Connection = info.Headers[hKey]; break; case "Content-Type": req.ContentType = info.Headers[hKey]; break; case "Expect": req.Expect = info.Headers[hKey]; break; case "Referer": req.Referer = info.Headers[hKey];; break; case "Transfer-Encoding": req.TransferEncoding = info.Headers[hKey]; break; case "User-Agent": req.UserAgent = info.Headers[hKey]; break; default: req.Headers.Add(hKey, info.Headers[hKey]); break; } } return(req); }
protected IBrowsingResponse PrepareBrowsingResponse(HttpWebRequest req, IBrowsingSession info) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); try { var response = GetResponseFromRequest(req); stopWatch.Stop(); return(new HttpBrowsingResponse(response, info, stopWatch.Elapsed)); } catch (WebException ex) { stopWatch.Stop(); //if (ex.Status == WebExceptionStatus.ProtocolError) if (ex.Response != null) { return(new HttpBrowsingResponse((HttpWebResponse)ex.Response, info, stopWatch.Elapsed)); } else { return(new ErrorBrowsingResponse() { GenerationTime = stopWatch.Elapsed, Headers = info.Headers, ResponseContent = ex.Message, ResponseUrl = req.RequestUri, StatusCode = HttpStatusCode.ServiceUnavailable }); } } catch (Exception ex) { throw new Exception("PrepareBrowsingResponse failed", ex); } }
protected override Uri GetNextProxyFor(IBrowsingSession session, string descriminator) { ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(session, descriminator); return ThreadProxies[Thread.CurrentThread.ManagedThreadId]; }
public override void OnInit(IBrowsingSession session) { WorkingProxies = _repo.GetWorking(); if(WorkingProxies.Count==0) throw new Exception("No working proxies found"); }
public virtual IBrowsingResponse FilePostResponse(Uri httpUri, System.Collections.Specialized.NameValueCollection postParamz, List<Tuple<string, string, System.IO.Stream>> files, IBrowsingSession info) { string boundary = "----------------------------" + DateTime.Now.Ticks.ToString("x"); HttpWebRequest req = CreateRequest(httpUri, info); req.Timeout = req.Timeout*(files.Count + 1); req.ContentType = "multipart/form-data; boundary=" + boundary; req.Method = "POST"; req.KeepAlive = true; Stream memStream = new System.IO.MemoryStream(); byte[] boundarybytes = System.Text.Encoding.ASCII.GetBytes("\r\n--" + boundary + "\r\n"); string formdataTemplate = "\r\n--" + boundary + "\r\nContent-Disposition: form-data; name=\"{0}\";\r\n\r\n{1}"; //--- params foreach (string key in postParamz.Keys) { string formitem = string.Format(formdataTemplate, key, postParamz[key]); byte[] formitembytes = System.Text.Encoding.UTF8.GetBytes(formitem); memStream.Write(formitembytes, 0, formitembytes.Length); } memStream.Write(boundarybytes, 0, boundarybytes.Length); // -- files string headerTemplate = "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\n Content-Type: application/octet-stream\r\n\r\n"; foreach (var fileData in files) { //string header = string.Format(headerTemplate, "file" + i, fileData.Key); string header = string.Format(headerTemplate, fileData.Item1, fileData.Item2); byte[] headerbytes = System.Text.Encoding.UTF8.GetBytes(header); memStream.Write(headerbytes, 0, headerbytes.Length); byte[] buffer = new byte[1024]; int bytesRead = 0; while ((bytesRead = fileData.Item3.Read(buffer, 0, buffer.Length)) != 0) { memStream.Write(buffer, 0, bytesRead); } memStream.Write(boundarybytes, 0, boundarybytes.Length); } req.ContentLength = memStream.Length; Stream requestStream = req.GetRequestStream(); memStream.Position = 0; byte[] tempBuffer = new byte[memStream.Length]; memStream.Read(tempBuffer, 0, tempBuffer.Length); memStream.Close(); requestStream.Write(tempBuffer, 0, tempBuffer.Length); requestStream.Close(); return PrepareBrowsingResponse(req, info); }
public virtual IBrowsingResponse GetResponse(Uri httpUri, IBrowsingSession info) { var req = CreateRequest(httpUri, info); req.Method = "GET"; return PrepareBrowsingResponse(req, info); }
public override IBrowsingResponse GetResponse(Uri httpUri, IBrowsingSession info) { var result = base.GetResponse(httpUri, info); return result; }
public override bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response) { LastRequest = uri; return(base.OnAfterRequestRerun(session, uri, response)); }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { Thread.Sleep(SleepTime); return(base.OnBeforeRequestStop(session, uri)); }
public virtual bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { return false; }
public override string GetCurrentIp(Uri httpUrl, IBrowsingSession info) { return proxyAddr.Host; }
public override string GetCurrentIp(Uri httpUrl, IBrowsingSession info) { var proxy = parallelHttpProxyDecorator.GetCurrentProxyFor(info, httpUrl.Host); return(proxy.Host); }
public virtual void OnInit(IBrowsingSession session) { }
public override IBrowsingResponse GetResponse(Uri httpUri, IBrowsingSession info) { var result = base.GetResponse(httpUri, info); return(result); }
public HttpBrowsingResponse(HttpWebResponse response, IBrowsingSession info, TimeSpan generationTime) { this.info = info; this.response = response; this.generationTime = generationTime; }
public virtual string GetCurrentIp(Uri httpUrl, IBrowsingSession info) { return(NetworkCommons.GetAllUnicastAddresses()[0].ToString()); }
public virtual bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response) { return false; }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { Thread.Sleep(SleepTime); return base.OnBeforeRequestStop(session, uri); }
public virtual bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri) { return false; }
public override string GetCurrentIp(Uri httpUrl, IBrowsingSession info) { var proxy = parallelHttpProxyDecorator.GetCurrentProxyFor(info, httpUrl.Host); return proxy.Host; }
public override bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri) { if (String.IsNullOrEmpty(Descriminator)) Descriminator = uri.Host; browsingSession.CurrentProxy = new HttpProxyBrowsing(GetNextProxyFor(browsingSession, Descriminator)); return true; }
public virtual void OnRemove(IBrowsingSession session) {}
public virtual IBrowsingResponse PostResponse(Uri httpUri, System.Collections.Specialized.NameValueCollection postParamz, IBrowsingSession info) { var req = CreateRequest(httpUri, info); req.Method = "POST"; ASCIIEncoding encoding = new ASCIIEncoding(); string postData = ""; if (postParamz.Count > 0) { postData = string.Join("&", Array.ConvertAll(postParamz.AllKeys, key => string.Format("{0}={1}", HttpUtility.UrlEncode(key), HttpUtility.UrlEncode(postParamz[key])))); } byte[] data = encoding.GetBytes(postData); req.ContentType = "application/x-www-form-urlencoded"; req.ContentLength = data.Length; Stream newStream = req.GetRequestStream(); newStream.Write(data, 0, data.Length); newStream.Close(); return PrepareBrowsingResponse(req, info); }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { retries = 0; return(base.OnBeforeRequestStop(session, uri)); }
public virtual string GetCurrentIp(Uri httpUrl, IBrowsingSession info) { return NetworkCommons.GetAllUnicastAddresses()[0].ToString(); }
protected override Uri GetNextProxyFor(IBrowsingSession session, string descriminator) { ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(session, descriminator); return(ThreadProxies[Thread.CurrentThread.ManagedThreadId]); }
public virtual bool OnAfterRequestRerun(IBrowsingSession session, Uri uri, IBrowsingResponse response) { return(false); }
public virtual bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { return(false); }
public override bool OnBeforeRequestStop(IBrowsingSession session, Uri uri) { if (String.IsNullOrEmpty(Descriminator)) Descriminator = uri.Host; if (session.CurrentProxy == null) session.CurrentProxy = proxySwitcher; return base.OnBeforeRequestStop(session, uri); }
public virtual bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri) { return(false); }
public override bool OnSwitchProxy(IBrowsingSession browsingSession, Uri uri) { if (String.IsNullOrEmpty(Descriminator)) Descriminator = uri.Host; ThreadProxies[Thread.CurrentThread.ManagedThreadId] = base.GetNextProxyFor(browsingSession, Descriminator); return true; }
public virtual void OnRemove(IBrowsingSession session) { }
public override string GetCurrentIp(Uri httpUrl, IBrowsingSession info) { return(proxyAddr.Host); }