Ejemplo n.º 1
0
 private static Task FindUnactive()
 {
     return(Task.Run(() => {
         while (!_Service.exit)
         {
             var startWiatDT = DateTime.Now;
             while ((DateTime.Now - startWiatDT).TotalMinutes < 3)
             {
                 Task.Delay(100);
             }
             Sql sql = new Sql();
             using (DataTable DT = sql.GetTable("select proxy, dtcreate from [xxxImport].[proxy].Proxys"))
             {
                 var dt = DT.AsEnumerable().OrderBy(m => m.Field <DateTime>("dtcreate")).Select(m => m.Field <string>("proxy").Trim()).Distinct();
                 foreach (string proxy in dt)
                 {
                     if (!WebR.Check(proxy))
                     {
                         int n = sql.Exec("delete from [xxxImport].[proxy].Proxys where proxy = '" + proxy + "'");
                         Console.WriteLine(proxy + " : removed from base : " + n);
                     }
                     else
                     {
                         Console.WriteLine(proxy + " : still alive");
                     }
                 }
             }
             sql.Dispose();
         }
     }));
 }
Ejemplo n.º 2
0
        /// <summary>
        /// Добавляет запись в базу,
        /// если прокси прошел проверку и не имеет дублей в базе.
        /// Возвращает кол-во затронутых строк
        /// </summary>
        /// <param name="prx_"></param>
        /// <returns></returns>
        public static int Add(string prx_, ref Sql sql)
        {
            prx_ = prx_.Trim();

            try
            {
                if (sql.Exec("select count (proxy) from [xxxImport].[proxy].Proxys where proxy = '" + prx_ + "'") > 0)
                {
                    throw new Exception();
                }
                if (!WebR.Check(prx_))
                {
                    throw new Exception();
                }

                lock (_Service.addingSync)
                {
                    int n = sql.Exec(@"insert into [xxxImport].[proxy].Proxys (id, proxy, dtcreate) values 
((select isnull(max(id), 0) from [CursorImport].[proxy].Proxys) + 1, '" + prx_ + "', getdate())");
                    Console.WriteLine(prx_ + " : added to base : " + n);
                    return(n);
                }
            }
            catch { return(0); }
        }
Ejemplo n.º 3
0
        /// <summary>
        /// Cтавим в очередь ссылки найденые в поисковиках
        /// </summary>
        private void Start()
        {
            int threadCount = 10;

            while (!_Service.exit)
            {
                foreach (string search in searchers)
                {
                    try
                    {
                        string[] hrefs = WebR.GetYandexHrefs(search);

                        int threads = hrefs.Length >= threadCount ? threadCount : hrefs.Length; //своеобразный threadpool

                        for (int i = 0; i < hrefs.Length; i += threads)                         //перебераем найденые сайты
                        {
                            Sql    sql  = new Sql();
                            Task[] Tsks = new Task[threads];

                            for (int y = 0; y < threads && y + i < hrefs.Length; y++)//разбиваем по потокам
                            {
                                int yi = y + i;
                                Tsks[y] = Task.Run(() =>
                                {
                                    string href          = hrefs[yi];
                                    string[] findedProxy = WebR.GetProxys(href);
                                    Console.WriteLine("created thread - " + href);
                                    foreach (string prx in findedProxy)
                                    {
                                        OnCreate.Add(prx, ref sql);
                                        if (_Service.exit)
                                        {
                                            return;
                                        }
                                    }
                                    Console.WriteLine("dispose thread - " + href);
                                });
                            }

                            Task.WaitAll(Tsks.Where(t => t != null).ToArray());
                            sql.Dispose();
                            if (_Service.exit)
                            {
                                return;
                            }
                        }
                    }
                    catch (Exception e) { Console.WriteLine("finder : " + e.Message); }
                    if (_Service.exit)
                    {
                        return;
                    }
                }
                GC.Collect();
                GC.GetTotalMemory(false);
            }
        }
Ejemplo n.º 4
0
        private async void Start(int threads)
        {
            while (!_Service.exit)
            {
                foreach (string search in searchers)
                {
                    try
                    {
                        string[] hrefs = WebR.GetYandexHrefs(search);

                        List <Task> taskList = new List <Task>();


                        for (int i = 0; i < hrefs.Length; i++)
                        {
                            int yi = i;
                            taskList.Add(Task.Run(() =>
                            {
                                Sql sql = new Sql();
                                if (sql.connectionIsOpen)
                                {
                                    try
                                    {
                                        string href          = hrefs[yi];
                                        string[] findedProxy = WebR.GetProxys(href);
                                        Console.WriteLine("created thread - " + href);
                                        foreach (string prx in findedProxy)
                                        {
                                            OnCreate.Add(prx, ref sql);
                                            if (_Service.exit)
                                            {
                                                return;
                                            }
                                        }
                                    }
                                    catch (Exception e) { Console.WriteLine("one of main threads : " + e.Message); }
                                    finally
                                    {
                                        sql.Dispose();
                                    }
                                }
                            }));

                            //ограничиваем кол-во потоков за раз
                            if (taskList.Where(e => e != null).Count() > threads)
                            {
                                await Task.WhenAny(taskList.ToArray());
                            }



                            if (_Service.exit)
                            {
                                return;
                            }
                        }

                        await Task.WhenAll(taskList.Where(e => e != null).ToArray());

                        GC.Collect();
                        GC.GetTotalMemory(false);
                    }
                    catch (Exception e) { Console.WriteLine("finder : " + e.Message); }
                    if (_Service.exit)
                    {
                        return;
                    }
                }
            }
        }