예제 #1
0
 private void WorkUploadPhoto(object sender, DoWorkEventArgs e)
 {
     int i = (int)e.Argument;
     states[i] = new WOrkingThreadState("photo", DateTime.Now, "photo");
     try
     {
         pq.UploadOnePhoto();
     }
     catch (Exception ex)
     {
         ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
                                     "Вай беда-беда " + ex.Message);
     }
     finally
     {
         states[i] = new WOrkingThreadState("sleep", DateTime.Now, "sleep");
     }
 }
예제 #2
0
 private void Worker(object sender, DoWorkEventArgs e)
 {
     Random r = new Random();
     int i = (int)e.Argument;
     //Stopwatch sw = new Stopwatch();
     while (work)
     {
         if (getted_jobs.Count > 0)
             {
                 DataDBURLJob temp = null;
                 try
                 {
                     lock (getted_jobs)
                     {
                         if (getted_jobs.Count > 0)
                             temp = getted_jobs.Dequeue();
                     }
                     //если есть задание на обработку страницы
                     if (temp != null)
                     {
                         states[i] = new WOrkingThreadState(temp.parserName, DateTime.Now, "process");
                         temp.streamNumber = i;
                         if (temp.status < 2)
                         {
                             RunWithTimeout(() =>
                             {
                                 string URL = temp.URL;
                                 DirtyApartments t = temp.StartProcessPage(ddbs,(mode!=3));
                                 if (temp != null)
                                 {
                                     if (mode == 3)
                                         temp.SetPhones(ddbs, ref t);
                                     if (states.Count(x => x != null && x.State == "sleep")>0)
                                         if (t != null)
                                             if (t.Фото != null && t.Фото !="")
                                                 t.Фото = pq.GetPhotos(t.Фото.Split(';'), temp._ipProxy, temp._portProxy, temp._pathForPhoto);
                                     if (t != null)
                                     {
                                         if (t.Источник != temp._source)
                                             temp.status = 5;
                                         else
                                         {
                                             ap.Add(t);
                                             temp.status = 1;
                                         }
                                     }
                                     else
                                     {
                                         temp.status = 4;
                                     }
                                 }
                             }, 31007);
                         }
                             jobs_made++;
                             //((DataDBURLJob)t_job).SetJobToStartOrStop(false, ddbs);
                             if (temp.status < 1)
                                 temp.status = 4;
                             lock (statuses)
                                 if (!statuses.Keys.Contains(temp.maskID))
                                     statuses.Add(temp.maskID, temp.status);
                                 else
                                     statuses[temp.maskID] = temp.status;
                             
                     }
                 }
                 catch (Exception ex)
                 {
                     if (temp != null)
                     {
                         ddbs.WriteErrorMessage(ddbs.ConnectionString, i, temp.URL,
                                                   "Ошибка в задаче получения страницы " + ex.Message, temp._addressparsing);
                         lock (statuses)
                             if (!statuses.Keys.Contains(temp.maskID))
                                 statuses.Add(temp.maskID, 4);
                             else
                                 statuses[temp.maskID]=4;
                     }
                 }
                 finally
                 {
                     if (temp != null)
                         GC.SuppressFinalize(temp);
                 }
             }
             else if (IsAnyJobInQueue)
             {
                 DataDBURLJob djob = Dequeue();
                 //если есть задание на скачивание страницы
                 try
                 {
                     if (djob != null)
                     {
                         states[i] = new WOrkingThreadState(djob.parserName, DateTime.Now, "get");
                         //ddbs.WriteErrorMessage(ddbs.ConnectionString, i, djob.URL, "скачивание страницы началось");
                         //sw.Start();
                         djob.streamNumber = i;
                         {
                             RunWithTimeout(() =>
                             {
                                 try
                                 {
                                     djob.StartGetPage(ddbs);
                                 }
                                 catch (Exception ex) { ddbs.WriteErrorMessage(ddbs.ConnectionString, i, djob.URL, "ошибка получения страницы очереди "+ex.Message, djob._addressparsing); }
                             
                                 // sw.Stop();
                             }, 45000);
                                     lock (current_parsers)
                                         current_parsers.Remove(djob.parserName);
                             if (djob.status < 1)
                                 djob.status = 4;
                         }
                     }
                 }
                 catch (Exception ex)
                 {
                     if (djob != null)
                     {
                         ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
                                                   "Ошибка в задаче получения страницы " + ex.Message, djob._addressparsing);
                         djob.status = 4;
                     }
                 }
                 finally
                 {
                     if (djob != null)
                     {
                         if (work)
                             lock (getted_jobs)
                                 getted_jobs.Enqueue(djob);
                         lock (statuses)
                             if (!statuses.Keys.Contains(djob.maskID))
                                 statuses.Add(djob.maskID, djob.status);
                             else
                                 statuses[djob.maskID] = djob.status;
                         GC.SuppressFinalize(djob);
                     }
                 }
             }
             else if (pq.Count() > 0)
             {
                 RunWithTimeout(() =>
                 {
                     try
                     {
                         states[i] = new WOrkingThreadState("photo", DateTime.Now, "photo");
                         pq.UploadOnePhoto();
                     }
                     catch (Exception ex)
                     {
                         ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
                                                     "Ошибка в задаче отправки фоток " + ex.Message);
                     }
                 }, 20000);
             }
             
             
             else
             {
                 try
                 {
                     if (states[i] == null)
                         states[i] = new WOrkingThreadState(null, DateTime.Now, "sleep");
                     else if (states[i].State != "sleep")
                         states[i] = new WOrkingThreadState(null, DateTime.Now, "sleep");
                     Thread.Sleep(r.Next(1000));
                 }
                 catch (Exception ex)
                 {
                     ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
                                               "Ошибка ухода треда в сон в случае отсутствия работы " + ex.Message);
                 }
             }
     }
     Console.WriteLine("Поток "+i+" остановлен");
     //work = false;
 }
예제 #3
0
 private void WorkProcessAd(object sender, DoWorkEventArgs e)
 {
     int i = (int)e.Argument;
     DataDBURLJob temp = null;
     try
     {
         lock (getted_jobs)
         {
             if (getted_jobs.Count > 0)
                 temp = getted_jobs.Dequeue();
         }
         //если есть задание на обработку страницы
         if (temp != null)
         {   
             states[i] = new WOrkingThreadState("process", DateTime.Now, temp.parserName);
             temp.streamNumber = i;
             if (temp.status < 2)
             {
                 string URL = temp.URL;
                 DirtyApartments t = temp.StartProcessPage(ddbs,(mode!=3));
                 if (temp != null)
                 {
                     if (mode == 3)
                         temp.SetPhones(ddbs, ref t);
                    // if (t != null)
                    //     if (t.Фото != null && t.Фото !="")
                    //         t.Фото=pq.GetPhotos(t.Фото, temp._ipProxy, temp._portProxy, temp._pathForPhoto);
                     if (t != null)
                     {
                         ap.Add(t);
                         temp.status = 1;
                     }
                     else
                     {
                         temp.status = 4;
                     }
                 }
             }
             jobs_made++;
             //((DataDBURLJob)t_job).SetJobToStartOrStop(false, ddbs);
             if (temp.status < 1)
                 temp.status = 4;
             lock (statuses)
                     if (!statuses.Keys.Contains(temp.maskID))
                         statuses.Add(temp.maskID, temp.status);
                             
         }
     }
     catch (Exception ex)
     {
         if(temp!=null)
         ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
                                     "Вай беда-беда "+ex.Message, temp._addressparsing);
     }
     finally
     {
         GC.SuppressFinalize(temp);
         states[i] = new WOrkingThreadState("sleep", DateTime.Now, "sleep");
     }
 }
예제 #4
0
 private void WorkGetAd(object sender, DoWorkEventArgs e)
 {
     int i = (int)e.Argument;
     DataDBURLJob djob = Dequeue();
     //если есть задание на скачивание страницы
     try
     {
         if (djob != null)
         {   
             states[i] = new WOrkingThreadState("get", DateTime.Now, djob.parserName);
             djob.streamNumber = i;
             {
                 try
                 {
                     djob.StartGetPage(ddbs);
                 }
                 catch (Exception ex) { ddbs.WriteErrorMessage(ddbs.ConnectionString, i, djob.URL, "ошибка получения страницы очереди "+ex.Message, djob._addressparsing); }
                     
                 lock (current_parsers)
                     current_parsers.Remove(djob.parserName);
                 if (djob.status < 1)
                     djob.status = 4;
                 if (work)
                     lock (getted_jobs)
                         getted_jobs.Enqueue(djob);
             }
         }
     }
     catch (Exception ex)
     {
         ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
                                     "Вай беда-беда "+ex.Message, djob._addressparsing);
     }
     finally
     {
         GC.SuppressFinalize(djob);
         states[i] = new WOrkingThreadState("sleep", DateTime.Now, "sleep");
     }
 }
예제 #5
0
 private void NewWorker(object sender, DoWorkEventArgs e)
 {
     tss = new DateTime[bws.Count()];
         while (work)
         {
             try
             {
                 int i = -1;
                 for (int j = 0; j < bws.Count(); j++)
                     if (!bws[j].IsBusy)
                     {
                         i = j;
                         break;
                     }
                 if (DateTime.Now.Second % 10 == 0)
                 {
                     Console.WriteLine(DateTime.Now.ToLongTimeString() + " made " + jobs_made + ", thrds " + bws.Count(x => x.IsBusy2) + ", queue " + jobs.Count() + ", dwnld " + getted_jobs.Count() + "; stat " + statuses.Count + ", ads " + ap.Count() + ", phts " + pq.Count());
                     //Thread.Sleep(500);
                 }
                 if (i >= 0)
                 {
                     if (send_flag != "true" && ((statuses.Count > 0 && jobs.Count() == 0) || statuses.Count > 50))
                     {
                         lock (send_flag)
                             send_flag = "true";
                         bws[i].DoWork += new DoWorkEventHandler(SendStatuses);
                         bws[i].RunWorkerAsync(i);
                         tss[i] = DateTime.Now;
                     }
                     else if (fill_flag != "true" && (jobs.jobs.Count() < 300))
                     {
                         lock (fill_flag)
                             fill_flag = "true";
                         bws[i].DoWork += new DoWorkEventHandler(Fill_procedure);
                         bws[i].RunWorkerAsync(i);
                         tss[i] = DateTime.Now;
                     }
                     else if (pq.Count() > 0)
                     {
                         bws[i].DoWork += new DoWorkEventHandler(WorkUploadPhoto);
                         bws[i].RunWorkerAsync(i);
                         tss[i] = DateTime.Now;
                     }
                     else if (getted_jobs.Count > 0)
                     {
                         bws[i].DoWork += new DoWorkEventHandler(WorkProcessAd);
                         bws[i].RunWorkerAsync(i);
                         tss[i] = DateTime.Now;
                     }
                     else if (IsAnyJobInQueue)
                     {
                         bws[i].DoWork += new DoWorkEventHandler(WorkGetAd);
                         bws[i].RunWorkerAsync(i);
                         tss[i] = DateTime.Now;
                     }
                     else
                     {
                         for (int j = 0; j < bws.Count(); j++)
                             if ((DateTime.Now - tss[j]).TotalSeconds > 45 && bws[j].IsBusy)
                             {
                                 bws[j].Abort();
                                 break;
                             }
                         states[i] = new WOrkingThreadState("sleep", DateTime.Now, "sleep");
                     }
                 }
                 else
                 {
                     for (int j = 0; j < bws.Count(); j++)
                         if ((DateTime.Now - tss[j]).TotalSeconds > 45 && bws[j].IsBusy2)
                         {
                             bws[j].Abort();
                             break;
                         }
                 }
             }
             catch (Exception ex) { string s = ex.Message; }
             finally
             {
                 Thread.Sleep(250);
             }
         }
     work = false;
 }
예제 #6
0
 private void SendStatuses(object sender, DoWorkEventArgs e)
 {
     int i = (int)e.Argument;
     states[i] = new WOrkingThreadState("send", DateTime.Now, "send");
     Console.WriteLine("Посылаем статусы на " + i + "-том воркере");
     try
     {
         lock (send_flag)
         {
             if ((statuses.Count > 0 && jobs.Count() == 0) || statuses.Count > 50)
                 lock (statuses)
                 {
                     /*Console.WriteLine(DateTime.Now.ToLongTimeString() + " Made2: " + jobs_made + ", queue: " + jobs.jobs.Count.ToString() + "; unsent statuses: " + statuses.Count.ToString() + ", ads: " + ap.Count().ToString()
                         + ", photos: " + pq.Count().ToString());*/
                     while (!DataDBURLJobs.SetJobStatuses(statuses, ddbs))
                     {
                         Thread.Sleep(1000);
                     }
                     statuses.Clear();
                     GC.Collect();
                 }
             send_flag = "false";
         }
     }
     catch(Exception ex){string s = ex.Message;}
 }
예제 #7
0
 private void Fill_procedure(object sender, DoWorkEventArgs e)
 {
     int i = (int)e.Argument;
     states[i] = new WOrkingThreadState("fill", DateTime.Now, "fill");
     Console.WriteLine("Получаем новые задачи на "+i+"-том воркере");
     try
     {
         lock (fill_flag)
         {            
             if (jobs.jobs.Count() < 300)
             {
                 /*Console.WriteLine(DateTime.Now.ToLongTimeString() + " Made1: " + jobs_made + ", queue: " + jobs.jobs.Count.ToString() + "; unsent statuses: " + statuses.Count.ToString() + ", ads: " + ap.Count().ToString()
                     + ", photos: " + pq.Count().ToString());*/
                 FillQueue();
                 if (statuses.Count == 0 && jobs.jobs.Count == 0)
                     DataDBURLJobs.ClearAddressParsing(ddbs);
                 GC.Collect();
             }
             fill_flag = "false";
         }
     }
     catch(Exception ex){string s = ex.Message;}
 }