public void CalcMultipleProxieRequests()
{
string jsonBody = null;
string response = null;
CalcTaskCostResponseDTO responseDTO;
int loopTimes = 20;
List <ProxyDTO> listProxies = new List <ProxyDTO>();
ProxyDTO proxyInfo = null;
for (int i = 0; i < loopTimes; i++)
{
jsonBody = Library.ReadStringFromFile(Library.VALID_CALC_TASK_COST_PATH);
response = Library.MakeHttpPostJSONRequestWithTime(Library.CALC_TASK_API_URL, jsonBody, ref proxyInfo);
responseDTO = JsonConvert.DeserializeObject <CalcTaskCostResponseDTO>(response);
listProxies.Add(proxyInfo);
}
foreach (ProxyDTO proxy in listProxies)
{
proxy.ToString();
}
/*
* Assert.AreEqual(Library.ERROR_CODE_SUCCESS, responseDTO.ErrorCode);
* Assert.IsNotNull(responseDTO.Response.IncentiveToPay.Name);
* Assert.IsNotNull(responseDTO.Response.IncentiveToPay.Quantity);
*/
}
static int SingleBehance()
{
BehanceWrapper viewer = new BehanceWrapper();
while (queue.Count > 0)
{
ProxyDTO proxy = proxy_wrapper.dequeProxy();
if (proxy == null)
{
Console.WriteLine("Proxy not found. Aborting...");
return(-1);
}
Console.WriteLine("proxy found - {0}", proxy);
var to_delete = new List <int>();
var tasks = new List <Task>();
for (int i = 0; i < queue.Count; i++)
{
var result = viewer.TaskFunction(queue[i], proxy);
if (result == -1)
{
proxy = proxy_wrapper.dequeProxy();
if (proxy == null)
{
Console.WriteLine("proxy not found, aborting");
return(-1);
}
Console.WriteLine("proxy found - {0}", proxy);
}
else
{
var to_delete_flag = queue.UpdateItem(i);
to_delete.Add(i);
}
}
to_delete.Reverse();
foreach (var item in to_delete)
{
queue.RemoveAt(item);
}
to_delete.Clear();
current_period += 1;
if (refresh_period >= current_period || queue.Count == 0)
{
Console.WriteLine("Refershing queue");
var status = RefreshQueue();
if (status != 1)
{
return(-1);
}
current_period = 0;
}
}
viewer.Close();
return(1);
}
private void SetProxy(ProxyDTO proxy)
{
driver.Navigate().GoToUrl("about:config");
var setupScript = @"var prefs = Components.classes[""@mozilla.org/preferences-service;1""].getService(Components.interfaces.nsIPrefBranch);
prefs.setIntPref(""network.proxy.type"", 1);
prefs.setCharPref(""network.proxy.http"", ""{0}"");
prefs.setIntPref(""network.proxy.http_port"", ""{1}"");
prefs.setCharPref(""network.proxy.ssl"", ""{0}"");
prefs.setIntPref(""network.proxy.ssl_port"", ""{1}"");
prefs.setCharPref(""network.proxy.ftp"", ""{0}"");
prefs.setIntPref(""network.proxy.ftp_port"", ""{1}"");";
(driver as IJavaScriptExecutor).ExecuteScript(string.Format(setupScript, proxy.host, proxy.port));
this.Sleep(1000);
}
public ProxyDTO dequeProxy(int max_tries = 100, int sleep_inbetween_tries_ms = 5000)
{
ProxyDTO proxy = null;
int counter = 0;
while (counter < max_tries)
{
if (proxies.Count == 0)
{
if (counter > 0)
{
Thread.Sleep(sleep_inbetween_tries_ms);
}
this.getProxy();
counter++;
}
if (proxies.Count == 0)
{
continue;
}
else
{
proxy = proxies.Dequeue();
if (used_proxies.ContainsKey(proxy))
{
proxy = null;
}
else
{
used_proxies.Add(proxy, true);
break;
}
}
}
return(proxy);
}
public static string MakeHttpPostJSONRequestWithTime(string url, string postBody, ref ProxyDTO proxyDTO)
{
string response = null;
WebProxy proxy;
int proxyPort = -1;
Stopwatch timer = new Stopwatch();
while (response == null)
{
timer.Start();
proxyDTO = GetRandomProxy();
int.TryParse(proxyDTO.Port, out proxyPort);
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(url);
proxy = new WebProxy(proxyDTO.IpAddress, proxyPort);
proxy.BypassProxyOnLocal = false;
httpWebRequest.Proxy = proxy;
httpWebRequest.Method = "POST";
httpWebRequest.ContentType = "application/json";
httpWebRequest.ContentLength = Encoding.UTF8.GetBytes(postBody).Length;
try
{
using (var streamWriter = new StreamWriter(httpWebRequest.GetRequestStream()))
{
streamWriter.Write(postBody);
}
response = GetResponseFromRequest(httpWebRequest);
timer.Stop();
if (response == null)
{
timer.Reset();
}
else
{
proxyDTO.ExecTime = timer.Elapsed;
if (response.Contains("<html>"))
{
response = null;
}
}
}
catch (Exception ex)
{
response = null;
timer.Reset();
}
}
return(response);
}
static int ThreadedBehance(int max_threads = 10)
{
var loc_max_threads = max_threads > queue.Count ? queue.Count : max_threads;
List <BehanceWrapper> viewers = new List <BehanceWrapper>(loc_max_threads);
for (int i = 0; i < loc_max_threads; i++)
{
viewers.Add(new BehanceWrapper());
}
while (queue.Count > 0)
{
ProxyDTO proxy = proxy_wrapper.dequeProxy();
if (proxy == null)
{
Console.WriteLine("proxy not found, aborting");
return(-1);
}
Console.WriteLine("proxy found - {0}", proxy);
var to_delete_mutex = new Mutex();
var to_delete = new List <int>();
var tasks = new List <Task>();
var semaphore = new SemaphoreSlim(loc_max_threads);
for (int i = 0; i < queue.Count; i++)
{
var j = i;
semaphore.Wait();
tasks.Add(Task.Factory.StartNew(() =>
{
var num = j % max_threads;
Console.WriteLine("Task {0} started", Task.CurrentId);
var result = viewers[num].TaskFunction(queue[j], proxy);
if (result == 1)
{
var to_delete_flag = queue.UpdateItem(j);
if (to_delete_flag)
{
to_delete_mutex.WaitOne();
to_delete.Add(j);
to_delete_mutex.ReleaseMutex();
}
}
Console.WriteLine("Task {0} finished", Task.CurrentId);
}, TaskCreationOptions.LongRunning).ContinueWith((task) =>
{
semaphore.Release();
}));
}
Task.WaitAll(tasks.ToArray());
tasks.Clear();
to_delete.Sort();
to_delete.Reverse();
foreach (var item in to_delete)
{
queue.RemoveAt(item);
}
to_delete.Clear();
current_period += 1;
if (refresh_period >= current_period || queue.Count == 0)
{
Console.WriteLine("Refershing queue");
var status = RefreshQueue();
if (status != 1)
{
return(-1);
}
current_period = 0;
}
}
viewers.ForEach(item => item.Close());
return(1);
}
public int TaskFunction(QueueDTO item, ProxyDTO proxy = null)
{
int code = -1;
Console.WriteLine("{0} started", item.id);
int views = item.views_work;
int likes = item.likes_work;
var url = item.url;
if (likes > views)
{
views = likes;
item.views_work = item.likes_work;
}
Console.WriteLine("Started");
try
{
if (proxy != null)
{
this.SetProxy(proxy);
}
driver.Navigate().GoToUrl(url);
Actions action = new Actions(driver);
var elems = driver.FindElements(By.ClassName("Project-projectStat-6Y3"));
Console.WriteLine("Waiting for views\\likes\\comments block");
elems = (new WebDriverWait(driver, TimeSpan.FromMinutes(2))).
Until(SeleniumExtras.WaitHelpers.ExpectedConditions.VisibilityOfAllElementsLocatedBy(By.ClassName("Project-projectStat-6Y3")));
action.MoveToElement(elems[1]);
var visits = elems[1].GetAttribute("innerHTML");
int pFrom = visits.IndexOf(">") + ">".Length;
int pTo = visits.LastIndexOf("<");
String result = visits.Substring(pFrom, pTo - pFrom);
Console.WriteLine("Waiting for jQuery to finish");
(new WebDriverWait(driver, TimeSpan.FromMinutes(2))).Until(
d => (bool)(d as IJavaScriptExecutor).ExecuteScript("return window.jQuery != undefined && jQuery.active === 0"));
if (item.likes_work > 0)
{
var like = driver.FindElement(By.ClassName("Appreciate-wrapper-9hi"));
var inner_before = like.GetAttribute("innerHTML");
action.MoveToElement(like);
Console.WriteLine("Waiting for like button");
like = (new WebDriverWait(driver, TimeSpan.FromMinutes(2))).
Until(SeleniumExtras.WaitHelpers.ExpectedConditions.ElementIsVisible(By.ClassName("Appreciate-wrapper-9hi")));
action.MoveToElement(like);
like.Click();
Console.WriteLine("Waiting for jQuery to finish");
(new WebDriverWait(driver, TimeSpan.FromMinutes(2))).Until(
d => (bool)(d as IJavaScriptExecutor).ExecuteScript("return window.jQuery != undefined && jQuery.active === 0"));
var inner_after = like.GetAttribute("innerHTML");
if (inner_after.Contains("Appreciate-count-") && !inner_after.Equals(inner_before))
{
item.likes_work -= 1;
}
}
Console.WriteLine("Done");
this.Sleep((new Random()).Next(5000, 10001));
Console.WriteLine("Done sleeping");
item.views_work -= 1;
code = 1;
}
catch (Exception e)
{
Console.WriteLine("Error");
System.Console.WriteLine(e.Message);
code = -1;
}
return(code);
}
