private static Task FindUnactive()
{
return(Task.Run(() => {
while (!_Service.exit)
{
var startWiatDT = DateTime.Now;
while ((DateTime.Now - startWiatDT).TotalMinutes < 3)
{
Task.Delay(100);
}
Sql sql = new Sql();
using (DataTable DT = sql.GetTable("select proxy, dtcreate from [xxxImport].[proxy].Proxys"))
{
var dt = DT.AsEnumerable().OrderBy(m => m.Field <DateTime>("dtcreate")).Select(m => m.Field <string>("proxy").Trim()).Distinct();
foreach (string proxy in dt)
{
if (!WebR.Check(proxy))
{
int n = sql.Exec("delete from [xxxImport].[proxy].Proxys where proxy = '" + proxy + "'");
Console.WriteLine(proxy + " : removed from base : " + n);
}
else
{
Console.WriteLine(proxy + " : still alive");
}
}
}
sql.Dispose();
}
}));
}
/// <summary>
/// Добавляет запись в базу,
/// если прокси прошел проверку и не имеет дублей в базе.
/// Возвращает кол-во затронутых строк
/// </summary>
/// <param name="prx_"></param>
/// <returns></returns>
public static int Add(string prx_, ref Sql sql)
{
prx_ = prx_.Trim();
try
{
if (sql.Exec("select count (proxy) from [xxxImport].[proxy].Proxys where proxy = '" + prx_ + "'") > 0)
{
throw new Exception();
}
if (!WebR.Check(prx_))
{
throw new Exception();
}
lock (_Service.addingSync)
{
int n = sql.Exec(@"insert into [xxxImport].[proxy].Proxys (id, proxy, dtcreate) values
((select isnull(max(id), 0) from [CursorImport].[proxy].Proxys) + 1, '" + prx_ + "', getdate())");
Console.WriteLine(prx_ + " : added to base : " + n);
return(n);
}
}
catch { return(0); }
}
/// <summary>
/// Cтавим в очередь ссылки найденые в поисковиках
/// </summary>
private void Start()
{
int threadCount = 10;
while (!_Service.exit)
{
foreach (string search in searchers)
{
try
{
string[] hrefs = WebR.GetYandexHrefs(search);
int threads = hrefs.Length >= threadCount ? threadCount : hrefs.Length; //своеобразный threadpool
for (int i = 0; i < hrefs.Length; i += threads) //перебераем найденые сайты
{
Sql sql = new Sql();
Task[] Tsks = new Task[threads];
for (int y = 0; y < threads && y + i < hrefs.Length; y++)//разбиваем по потокам
{
int yi = y + i;
Tsks[y] = Task.Run(() =>
{
string href = hrefs[yi];
string[] findedProxy = WebR.GetProxys(href);
Console.WriteLine("created thread - " + href);
foreach (string prx in findedProxy)
{
OnCreate.Add(prx, ref sql);
if (_Service.exit)
{
return;
}
}
Console.WriteLine("dispose thread - " + href);
});
}
Task.WaitAll(Tsks.Where(t => t != null).ToArray());
sql.Dispose();
if (_Service.exit)
{
return;
}
}
}
catch (Exception e) { Console.WriteLine("finder : " + e.Message); }
if (_Service.exit)
{
return;
}
}
GC.Collect();
GC.GetTotalMemory(false);
}
}
private async void Start(int threads)
{
while (!_Service.exit)
{
foreach (string search in searchers)
{
try
{
string[] hrefs = WebR.GetYandexHrefs(search);
List <Task> taskList = new List <Task>();
for (int i = 0; i < hrefs.Length; i++)
{
int yi = i;
taskList.Add(Task.Run(() =>
{
Sql sql = new Sql();
if (sql.connectionIsOpen)
{
try
{
string href = hrefs[yi];
string[] findedProxy = WebR.GetProxys(href);
Console.WriteLine("created thread - " + href);
foreach (string prx in findedProxy)
{
OnCreate.Add(prx, ref sql);
if (_Service.exit)
{
return;
}
}
}
catch (Exception e) { Console.WriteLine("one of main threads : " + e.Message); }
finally
{
sql.Dispose();
}
}
}));
//ограничиваем кол-во потоков за раз
if (taskList.Where(e => e != null).Count() > threads)
{
await Task.WhenAny(taskList.ToArray());
}
if (_Service.exit)
{
return;
}
}
await Task.WhenAll(taskList.Where(e => e != null).ToArray());
GC.Collect();
GC.GetTotalMemory(false);
}
catch (Exception e) { Console.WriteLine("finder : " + e.Message); }
if (_Service.exit)
{
return;
}
}
}
}