public static void GrabberIp()
{
while (true)
{
Thread.Sleep(1000*20);
using (_client = new WebClient())
{
var db = new DataProviderProxy();
_client.Encoding = Encoding.UTF8;
Dictionary<int, string> dbResault = db.GetSiteWithId();
if (dbResault == null) continue;
try
{
string resault = _client.DownloadString(@"http://" + dbResault.Values.ElementAt(0));
_regex = new Regex(_patternIp);
_matchCol = _regex.Matches(resault);
foreach (Match match in _matchCol)
{
Console.WriteLine("найден ip: " + match.Groups[1].Value);
db.InsertIp(dbResault.Keys.ElementAt(0), match.Groups[1].Value);
}
}
catch
{
Console.WriteLine("errorConnect 404");
}
finally
{
db.UpdateDate(dbResault.Keys.ElementAt(0), DateTime.Now);
}
}
}
}
public static void Checker(object stat)
{
WebRequest webReq = WebRequest.Create(@"http://bitfry.narod.ru/02.htm");
var db = new DataProviderProxy();
string ip = stat.ToString();
try
{
webReq.Proxy = new WebProxy(ip);
webReq.Timeout = 10000;
WebResponse webRes = webReq.GetResponse();
Console.WriteLine("отличная прокся: " + ip);
db.UpdateStatus(ip, StatusIp.Good, "ip");
}
catch
{
db.DeleteIp(ip);
Console.WriteLine("плохая прокся del: " + ip);
}
finally
{
_countdown.Signal();
}
}
public static void MainGrabber()
{
// id="serp_res_(\d+)">\s*<div class="res-wrap">\s*<h3 class="res-head">\s*<a\s*[\S, ]*\s*target="[\w]*"\s*href="http://(?:www.)?([-\w.]+/[\S]*)"
_patternSait =
@"id=""serp_res_(\d+)"">\s*<div class=""res-wrap"">\s*<h3 class=""res-head"">\s*<a\s*[\S, ]*\s*target=""[\w]*""\s*href=""http://(?:www.)?([-\w.]+/[\S]*)""";
// (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})
_patternIp = @"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})";
var th1 = new Thread(SiteGrabber);
//th1.Start();
var th2 = new Thread(GrabberIp);
//th2.Start();
var th4 = new Thread(ReGrabber);
//th4.Start();
var db = new DataProviderProxy();
while (true)
{
List<string> listIp = db.GetUncheckedIp(100);
if (listIp != null)
{
_countdown = new CountdownEvent(listIp.Count);
foreach (string item in listIp)
{
new Thread(Checker).Start(item);
}
_countdown.Wait();
}
Thread.Sleep(50);
}
}
public static void SiteGrabber()
{
while (true)
{
using (_client = new WebClient())
{
var db = new DataProviderProxy();
_client.Encoding = Encoding.UTF8;
string resault =
_client.DownloadString(
@"http://go.mail.ru/search?q=+"":8080"" +"":3128"" +"":80""&num=10&rch=e&sf=" + _page*10);
_regex = new Regex(_patternSait);
_matchCol = _regex.Matches(resault);
foreach (Match match in _matchCol)
{
Console.WriteLine("найден: " + match.Groups[2].Value);
db.InsertSite(match.Groups[2].Value);
}
_page++;
}
Thread.Sleep(1000*60*3);
}
}
public static void ReGrabber()
{
while (true)
{
Thread.Sleep(1000*60*60);
var db = new DataProviderProxy();
Dictionary<int, string> resault = db.GetOldSites();
if (resault == null) continue;
foreach (var item in resault)
{
db.UpdateDate(item.Key, null);
db.UpdateStatus(item.Key.ToString(), StatusIp.Unchecked, "idSite");
}
}
}
