private static void workerThread(string sourceIP = "192.168.0.0/24")
{
// Solution copied and edited from: http://stackoverflow.com/questions/32028166/convert-cidr-notation-into-ip-range
ProxyCrawler proxyCrawler = new ProxyCrawler();
string[] parts = sourceIP.Split('.', '/');
uint ipNum = (Convert.ToUInt32(parts[0]) << 24) |
(Convert.ToUInt32(parts[1]) << 16) |
(Convert.ToUInt32(parts[2]) << 8) |
Convert.ToUInt32(parts[3]);
int maskbits = Convert.ToInt32(parts[4]);
uint mask = 0xffffffff;
mask <<= (32 - maskbits);
uint ipStart = ipNum & mask;
uint ipEnd = ipNum | (mask ^ 0xffffffff);
uint ipLength = ipEnd - ipStart;
Console.WriteLine("[INFO] Thread #{0}: Range of IP address: {1} - {2}", Thread.CurrentThread.ManagedThreadId, convertToIp(ipStart), convertToIp(ipEnd));
Console.WriteLine("[INFO] Thread #{0}: IP Start Position: {0}, End position: {1}", ipStart, ipEnd);
// We cannot use unsigned 32 bit int here, and signed int32 isn't long enough.
// So, here we use signed long (int64) instead.
Parallel.For(0, ipLength, new ParallelOptions {
MaxDegreeOfParallelism = 3
}, async(long ipIndex) =>
{
for (int proxyPort = 80; proxyPort <= 9000; proxyPort++)
{
Task <bool> validateProxy = proxyCrawler.IsValidProxy(convertToIp(ipStart + (uint)ipIndex), proxyPort);
bool proxyTestResult = await validateProxy;
if (proxyTestResult)
{
string proxyPath = convertToIp(ipStart + (uint)ipIndex) + ":" + proxyPort.ToString();
streamWriter.WriteLine(proxyPath);
streamWriter.Flush();
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("[SUCCESS] Found HTTP Proxy at {0}", proxyPath);
Console.ResetColor();
}
else
{
string proxyPath = convertToIp(ipStart + (uint)ipIndex) + ":" + proxyPort.ToString();
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("[FAIL] Not an HTTP Proxy at {0}", proxyPath);
Console.ResetColor();
}
}
});
}
public static void Main(string[] args)
{
Task mainTask = new Task(() =>
{
ProxyCrawler crawler = new ProxyCrawler();
string[] ipRangeArray = crawler.LoadChinaList();
Console.WriteLine(ipRangeArray);
Parallel.ForEach(ipRangeArray, new ParallelOptions {
MaxDegreeOfParallelism = 3
}, (string currentIp) =>
{
workerThread(currentIp);
});
});
mainTask.Start();
mainTask.Wait();
Console.ReadLine();
}
