public void GetProxies()
{
if (Mgr == null || Mgr.Count < 1 || WC.Timeout < 1)
{
return;
}
Threads++;
Program.UI.UpdateScrapeBtns();
Thread.Sleep(100);
var url = Mgr.RecommendURL();
var proxiesFound = false;
while (url != string.Empty && !TerminateThreads)
{
while (PauseThreads)
{
Thread.Sleep(333);
if (TerminateThreads)
{
PauseThreads = false;
continue;
}
}
try {
var html = WC.DownloadString(url);
List <MyProxy> proxies = ProxiesFromHtml(html);
for (var p = 0; p < proxies.Count; ++p)
{
if (TerminateThreads) //sometimes looping inside this can take long
{
break;
}
Console.WriteLine("[+] {0} from {1} : {2}", proxies[p], url, Thread.CurrentThread.Name);
Program.UI.AddProxy(proxies[p]);
}
proxiesFound = proxies.Count > 0;
} catch (ArgumentNullException) { BadURLs++; } catch (WebException) { BadURLs++; } catch (NotSupportedException) { BadURLs++; } finally {
//End of URL processing -> next loop
if (!proxiesFound)
{
Console.WriteLine("{0} was void of proxies.", url);
EmptyURLs++;
}
URLScraped++;
url = Mgr.RecommendURL(); //change URL
}
}
Threads--;
Program.UI.UpdateScrapeBtns(); //make sure this is before reset else stats get wiped
Thread.Sleep(100);
if (TerminateThreads) //if forced termination and reset() is invoked -> terminate all is reset to false :( (spaghetti ughh)
{
while (Threads > 0)
{
Thread.Sleep(250);
}
}
Reset();
Console.WriteLine(string.Format("***Thread '{0}' terminated, reason is '{1}'... '{2}' Threads left running...",
Thread.CurrentThread.Name, TerminateThreads ? "STOP PRESSED" : "FINISHED", Threads.ToString())); //Debug
}