public void CalcMultipleProxieRequests() { string jsonBody = null; string response = null; CalcTaskCostResponseDTO responseDTO; int loopTimes = 20; List <ProxyDTO> listProxies = new List <ProxyDTO>(); ProxyDTO proxyInfo = null; for (int i = 0; i < loopTimes; i++) { jsonBody = Library.ReadStringFromFile(Library.VALID_CALC_TASK_COST_PATH); response = Library.MakeHttpPostJSONRequestWithTime(Library.CALC_TASK_API_URL, jsonBody, ref proxyInfo); responseDTO = JsonConvert.DeserializeObject <CalcTaskCostResponseDTO>(response); listProxies.Add(proxyInfo); } foreach (ProxyDTO proxy in listProxies) { proxy.ToString(); } /* * Assert.AreEqual(Library.ERROR_CODE_SUCCESS, responseDTO.ErrorCode); * Assert.IsNotNull(responseDTO.Response.IncentiveToPay.Name); * Assert.IsNotNull(responseDTO.Response.IncentiveToPay.Quantity); */ }
static int SingleBehance() { BehanceWrapper viewer = new BehanceWrapper(); while (queue.Count > 0) { ProxyDTO proxy = proxy_wrapper.dequeProxy(); if (proxy == null) { Console.WriteLine("Proxy not found. Aborting..."); return(-1); } Console.WriteLine("proxy found - {0}", proxy); var to_delete = new List <int>(); var tasks = new List <Task>(); for (int i = 0; i < queue.Count; i++) { var result = viewer.TaskFunction(queue[i], proxy); if (result == -1) { proxy = proxy_wrapper.dequeProxy(); if (proxy == null) { Console.WriteLine("proxy not found, aborting"); return(-1); } Console.WriteLine("proxy found - {0}", proxy); } else { var to_delete_flag = queue.UpdateItem(i); to_delete.Add(i); } } to_delete.Reverse(); foreach (var item in to_delete) { queue.RemoveAt(item); } to_delete.Clear(); current_period += 1; if (refresh_period >= current_period || queue.Count == 0) { Console.WriteLine("Refershing queue"); var status = RefreshQueue(); if (status != 1) { return(-1); } current_period = 0; } } viewer.Close(); return(1); }
private void SetProxy(ProxyDTO proxy) { driver.Navigate().GoToUrl("about:config"); var setupScript = @"var prefs = Components.classes[""@mozilla.org/preferences-service;1""].getService(Components.interfaces.nsIPrefBranch); prefs.setIntPref(""network.proxy.type"", 1); prefs.setCharPref(""network.proxy.http"", ""{0}""); prefs.setIntPref(""network.proxy.http_port"", ""{1}""); prefs.setCharPref(""network.proxy.ssl"", ""{0}""); prefs.setIntPref(""network.proxy.ssl_port"", ""{1}""); prefs.setCharPref(""network.proxy.ftp"", ""{0}""); prefs.setIntPref(""network.proxy.ftp_port"", ""{1}"");"; (driver as IJavaScriptExecutor).ExecuteScript(string.Format(setupScript, proxy.host, proxy.port)); this.Sleep(1000); }
public ProxyDTO dequeProxy(int max_tries = 100, int sleep_inbetween_tries_ms = 5000) { ProxyDTO proxy = null; int counter = 0; while (counter < max_tries) { if (proxies.Count == 0) { if (counter > 0) { Thread.Sleep(sleep_inbetween_tries_ms); } this.getProxy(); counter++; } if (proxies.Count == 0) { continue; } else { proxy = proxies.Dequeue(); if (used_proxies.ContainsKey(proxy)) { proxy = null; } else { used_proxies.Add(proxy, true); break; } } } return(proxy); }
public static string MakeHttpPostJSONRequestWithTime(string url, string postBody, ref ProxyDTO proxyDTO) { string response = null; WebProxy proxy; int proxyPort = -1; Stopwatch timer = new Stopwatch(); while (response == null) { timer.Start(); proxyDTO = GetRandomProxy(); int.TryParse(proxyDTO.Port, out proxyPort); HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(url); proxy = new WebProxy(proxyDTO.IpAddress, proxyPort); proxy.BypassProxyOnLocal = false; httpWebRequest.Proxy = proxy; httpWebRequest.Method = "POST"; httpWebRequest.ContentType = "application/json"; httpWebRequest.ContentLength = Encoding.UTF8.GetBytes(postBody).Length; try { using (var streamWriter = new StreamWriter(httpWebRequest.GetRequestStream())) { streamWriter.Write(postBody); } response = GetResponseFromRequest(httpWebRequest); timer.Stop(); if (response == null) { timer.Reset(); } else { proxyDTO.ExecTime = timer.Elapsed; if (response.Contains("<html>")) { response = null; } } } catch (Exception ex) { response = null; timer.Reset(); } } return(response); }
static int ThreadedBehance(int max_threads = 10) { var loc_max_threads = max_threads > queue.Count ? queue.Count : max_threads; List <BehanceWrapper> viewers = new List <BehanceWrapper>(loc_max_threads); for (int i = 0; i < loc_max_threads; i++) { viewers.Add(new BehanceWrapper()); } while (queue.Count > 0) { ProxyDTO proxy = proxy_wrapper.dequeProxy(); if (proxy == null) { Console.WriteLine("proxy not found, aborting"); return(-1); } Console.WriteLine("proxy found - {0}", proxy); var to_delete_mutex = new Mutex(); var to_delete = new List <int>(); var tasks = new List <Task>(); var semaphore = new SemaphoreSlim(loc_max_threads); for (int i = 0; i < queue.Count; i++) { var j = i; semaphore.Wait(); tasks.Add(Task.Factory.StartNew(() => { var num = j % max_threads; Console.WriteLine("Task {0} started", Task.CurrentId); var result = viewers[num].TaskFunction(queue[j], proxy); if (result == 1) { var to_delete_flag = queue.UpdateItem(j); if (to_delete_flag) { to_delete_mutex.WaitOne(); to_delete.Add(j); to_delete_mutex.ReleaseMutex(); } } Console.WriteLine("Task {0} finished", Task.CurrentId); }, TaskCreationOptions.LongRunning).ContinueWith((task) => { semaphore.Release(); })); } Task.WaitAll(tasks.ToArray()); tasks.Clear(); to_delete.Sort(); to_delete.Reverse(); foreach (var item in to_delete) { queue.RemoveAt(item); } to_delete.Clear(); current_period += 1; if (refresh_period >= current_period || queue.Count == 0) { Console.WriteLine("Refershing queue"); var status = RefreshQueue(); if (status != 1) { return(-1); } current_period = 0; } } viewers.ForEach(item => item.Close()); return(1); }
public int TaskFunction(QueueDTO item, ProxyDTO proxy = null) { int code = -1; Console.WriteLine("{0} started", item.id); int views = item.views_work; int likes = item.likes_work; var url = item.url; if (likes > views) { views = likes; item.views_work = item.likes_work; } Console.WriteLine("Started"); try { if (proxy != null) { this.SetProxy(proxy); } driver.Navigate().GoToUrl(url); Actions action = new Actions(driver); var elems = driver.FindElements(By.ClassName("Project-projectStat-6Y3")); Console.WriteLine("Waiting for views\\likes\\comments block"); elems = (new WebDriverWait(driver, TimeSpan.FromMinutes(2))). Until(SeleniumExtras.WaitHelpers.ExpectedConditions.VisibilityOfAllElementsLocatedBy(By.ClassName("Project-projectStat-6Y3"))); action.MoveToElement(elems[1]); var visits = elems[1].GetAttribute("innerHTML"); int pFrom = visits.IndexOf(">") + ">".Length; int pTo = visits.LastIndexOf("<"); String result = visits.Substring(pFrom, pTo - pFrom); Console.WriteLine("Waiting for jQuery to finish"); (new WebDriverWait(driver, TimeSpan.FromMinutes(2))).Until( d => (bool)(d as IJavaScriptExecutor).ExecuteScript("return window.jQuery != undefined && jQuery.active === 0")); if (item.likes_work > 0) { var like = driver.FindElement(By.ClassName("Appreciate-wrapper-9hi")); var inner_before = like.GetAttribute("innerHTML"); action.MoveToElement(like); Console.WriteLine("Waiting for like button"); like = (new WebDriverWait(driver, TimeSpan.FromMinutes(2))). Until(SeleniumExtras.WaitHelpers.ExpectedConditions.ElementIsVisible(By.ClassName("Appreciate-wrapper-9hi"))); action.MoveToElement(like); like.Click(); Console.WriteLine("Waiting for jQuery to finish"); (new WebDriverWait(driver, TimeSpan.FromMinutes(2))).Until( d => (bool)(d as IJavaScriptExecutor).ExecuteScript("return window.jQuery != undefined && jQuery.active === 0")); var inner_after = like.GetAttribute("innerHTML"); if (inner_after.Contains("Appreciate-count-") && !inner_after.Equals(inner_before)) { item.likes_work -= 1; } } Console.WriteLine("Done"); this.Sleep((new Random()).Next(5000, 10001)); Console.WriteLine("Done sleeping"); item.views_work -= 1; code = 1; } catch (Exception e) { Console.WriteLine("Error"); System.Console.WriteLine(e.Message); code = -1; } return(code); }