private void WorkUploadPhoto(object sender, DoWorkEventArgs e)
{
int i = (int)e.Argument;
states[i] = new WOrkingThreadState("photo", DateTime.Now, "photo");
try
{
pq.UploadOnePhoto();
}
catch (Exception ex)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
"Вай беда-беда " + ex.Message);
}
finally
{
states[i] = new WOrkingThreadState("sleep", DateTime.Now, "sleep");
}
}
private void Worker(object sender, DoWorkEventArgs e)
{
Random r = new Random();
int i = (int)e.Argument;
//Stopwatch sw = new Stopwatch();
while (work)
{
if (getted_jobs.Count > 0)
{
DataDBURLJob temp = null;
try
{
lock (getted_jobs)
{
if (getted_jobs.Count > 0)
temp = getted_jobs.Dequeue();
}
//если есть задание на обработку страницы
if (temp != null)
{
states[i] = new WOrkingThreadState(temp.parserName, DateTime.Now, "process");
temp.streamNumber = i;
if (temp.status < 2)
{
RunWithTimeout(() =>
{
string URL = temp.URL;
DirtyApartments t = temp.StartProcessPage(ddbs,(mode!=3));
if (temp != null)
{
if (mode == 3)
temp.SetPhones(ddbs, ref t);
if (states.Count(x => x != null && x.State == "sleep")>0)
if (t != null)
if (t.Фото != null && t.Фото !="")
t.Фото = pq.GetPhotos(t.Фото.Split(';'), temp._ipProxy, temp._portProxy, temp._pathForPhoto);
if (t != null)
{
if (t.Источник != temp._source)
temp.status = 5;
else
{
ap.Add(t);
temp.status = 1;
}
}
else
{
temp.status = 4;
}
}
}, 31007);
}
jobs_made++;
//((DataDBURLJob)t_job).SetJobToStartOrStop(false, ddbs);
if (temp.status < 1)
temp.status = 4;
lock (statuses)
if (!statuses.Keys.Contains(temp.maskID))
statuses.Add(temp.maskID, temp.status);
else
statuses[temp.maskID] = temp.status;
}
}
catch (Exception ex)
{
if (temp != null)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, i, temp.URL,
"Ошибка в задаче получения страницы " + ex.Message, temp._addressparsing);
lock (statuses)
if (!statuses.Keys.Contains(temp.maskID))
statuses.Add(temp.maskID, 4);
else
statuses[temp.maskID]=4;
}
}
finally
{
if (temp != null)
GC.SuppressFinalize(temp);
}
}
else if (IsAnyJobInQueue)
{
DataDBURLJob djob = Dequeue();
//если есть задание на скачивание страницы
try
{
if (djob != null)
{
states[i] = new WOrkingThreadState(djob.parserName, DateTime.Now, "get");
//ddbs.WriteErrorMessage(ddbs.ConnectionString, i, djob.URL, "скачивание страницы началось");
//sw.Start();
djob.streamNumber = i;
{
RunWithTimeout(() =>
{
try
{
djob.StartGetPage(ddbs);
}
catch (Exception ex) { ddbs.WriteErrorMessage(ddbs.ConnectionString, i, djob.URL, "ошибка получения страницы очереди "+ex.Message, djob._addressparsing); }
// sw.Stop();
}, 45000);
lock (current_parsers)
current_parsers.Remove(djob.parserName);
if (djob.status < 1)
djob.status = 4;
}
}
}
catch (Exception ex)
{
if (djob != null)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
"Ошибка в задаче получения страницы " + ex.Message, djob._addressparsing);
djob.status = 4;
}
}
finally
{
if (djob != null)
{
if (work)
lock (getted_jobs)
getted_jobs.Enqueue(djob);
lock (statuses)
if (!statuses.Keys.Contains(djob.maskID))
statuses.Add(djob.maskID, djob.status);
else
statuses[djob.maskID] = djob.status;
GC.SuppressFinalize(djob);
}
}
}
else if (pq.Count() > 0)
{
RunWithTimeout(() =>
{
try
{
states[i] = new WOrkingThreadState("photo", DateTime.Now, "photo");
pq.UploadOnePhoto();
}
catch (Exception ex)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
"Ошибка в задаче отправки фоток " + ex.Message);
}
}, 20000);
}
else
{
try
{
if (states[i] == null)
states[i] = new WOrkingThreadState(null, DateTime.Now, "sleep");
else if (states[i].State != "sleep")
states[i] = new WOrkingThreadState(null, DateTime.Now, "sleep");
Thread.Sleep(r.Next(1000));
}
catch (Exception ex)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
"Ошибка ухода треда в сон в случае отсутствия работы " + ex.Message);
}
}
}
Console.WriteLine("Поток "+i+" остановлен");
//work = false;
}
private void WorkProcessAd(object sender, DoWorkEventArgs e)
{
int i = (int)e.Argument;
DataDBURLJob temp = null;
try
{
lock (getted_jobs)
{
if (getted_jobs.Count > 0)
temp = getted_jobs.Dequeue();
}
//если есть задание на обработку страницы
if (temp != null)
{
states[i] = new WOrkingThreadState("process", DateTime.Now, temp.parserName);
temp.streamNumber = i;
if (temp.status < 2)
{
string URL = temp.URL;
DirtyApartments t = temp.StartProcessPage(ddbs,(mode!=3));
if (temp != null)
{
if (mode == 3)
temp.SetPhones(ddbs, ref t);
// if (t != null)
// if (t.Фото != null && t.Фото !="")
// t.Фото=pq.GetPhotos(t.Фото, temp._ipProxy, temp._portProxy, temp._pathForPhoto);
if (t != null)
{
ap.Add(t);
temp.status = 1;
}
else
{
temp.status = 4;
}
}
}
jobs_made++;
//((DataDBURLJob)t_job).SetJobToStartOrStop(false, ddbs);
if (temp.status < 1)
temp.status = 4;
lock (statuses)
if (!statuses.Keys.Contains(temp.maskID))
statuses.Add(temp.maskID, temp.status);
}
}
catch (Exception ex)
{
if(temp!=null)
ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
"Вай беда-беда "+ex.Message, temp._addressparsing);
}
finally
{
GC.SuppressFinalize(temp);
states[i] = new WOrkingThreadState("sleep", DateTime.Now, "sleep");
}
}
private void WorkGetAd(object sender, DoWorkEventArgs e)
{
int i = (int)e.Argument;
DataDBURLJob djob = Dequeue();
//если есть задание на скачивание страницы
try
{
if (djob != null)
{
states[i] = new WOrkingThreadState("get", DateTime.Now, djob.parserName);
djob.streamNumber = i;
{
try
{
djob.StartGetPage(ddbs);
}
catch (Exception ex) { ddbs.WriteErrorMessage(ddbs.ConnectionString, i, djob.URL, "ошибка получения страницы очереди "+ex.Message, djob._addressparsing); }
lock (current_parsers)
current_parsers.Remove(djob.parserName);
if (djob.status < 1)
djob.status = 4;
if (work)
lock (getted_jobs)
getted_jobs.Enqueue(djob);
}
}
}
catch (Exception ex)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, i, null,
"Вай беда-беда "+ex.Message, djob._addressparsing);
}
finally
{
GC.SuppressFinalize(djob);
states[i] = new WOrkingThreadState("sleep", DateTime.Now, "sleep");
}
}
private void NewWorker(object sender, DoWorkEventArgs e)
{
tss = new DateTime[bws.Count()];
while (work)
{
try
{
int i = -1;
for (int j = 0; j < bws.Count(); j++)
if (!bws[j].IsBusy)
{
i = j;
break;
}
if (DateTime.Now.Second % 10 == 0)
{
Console.WriteLine(DateTime.Now.ToLongTimeString() + " made " + jobs_made + ", thrds " + bws.Count(x => x.IsBusy2) + ", queue " + jobs.Count() + ", dwnld " + getted_jobs.Count() + "; stat " + statuses.Count + ", ads " + ap.Count() + ", phts " + pq.Count());
//Thread.Sleep(500);
}
if (i >= 0)
{
if (send_flag != "true" && ((statuses.Count > 0 && jobs.Count() == 0) || statuses.Count > 50))
{
lock (send_flag)
send_flag = "true";
bws[i].DoWork += new DoWorkEventHandler(SendStatuses);
bws[i].RunWorkerAsync(i);
tss[i] = DateTime.Now;
}
else if (fill_flag != "true" && (jobs.jobs.Count() < 300))
{
lock (fill_flag)
fill_flag = "true";
bws[i].DoWork += new DoWorkEventHandler(Fill_procedure);
bws[i].RunWorkerAsync(i);
tss[i] = DateTime.Now;
}
else if (pq.Count() > 0)
{
bws[i].DoWork += new DoWorkEventHandler(WorkUploadPhoto);
bws[i].RunWorkerAsync(i);
tss[i] = DateTime.Now;
}
else if (getted_jobs.Count > 0)
{
bws[i].DoWork += new DoWorkEventHandler(WorkProcessAd);
bws[i].RunWorkerAsync(i);
tss[i] = DateTime.Now;
}
else if (IsAnyJobInQueue)
{
bws[i].DoWork += new DoWorkEventHandler(WorkGetAd);
bws[i].RunWorkerAsync(i);
tss[i] = DateTime.Now;
}
else
{
for (int j = 0; j < bws.Count(); j++)
if ((DateTime.Now - tss[j]).TotalSeconds > 45 && bws[j].IsBusy)
{
bws[j].Abort();
break;
}
states[i] = new WOrkingThreadState("sleep", DateTime.Now, "sleep");
}
}
else
{
for (int j = 0; j < bws.Count(); j++)
if ((DateTime.Now - tss[j]).TotalSeconds > 45 && bws[j].IsBusy2)
{
bws[j].Abort();
break;
}
}
}
catch (Exception ex) { string s = ex.Message; }
finally
{
Thread.Sleep(250);
}
}
work = false;
}
private void SendStatuses(object sender, DoWorkEventArgs e)
{
int i = (int)e.Argument;
states[i] = new WOrkingThreadState("send", DateTime.Now, "send");
Console.WriteLine("Посылаем статусы на " + i + "-том воркере");
try
{
lock (send_flag)
{
if ((statuses.Count > 0 && jobs.Count() == 0) || statuses.Count > 50)
lock (statuses)
{
/*Console.WriteLine(DateTime.Now.ToLongTimeString() + " Made2: " + jobs_made + ", queue: " + jobs.jobs.Count.ToString() + "; unsent statuses: " + statuses.Count.ToString() + ", ads: " + ap.Count().ToString()
+ ", photos: " + pq.Count().ToString());*/
while (!DataDBURLJobs.SetJobStatuses(statuses, ddbs))
{
Thread.Sleep(1000);
}
statuses.Clear();
GC.Collect();
}
send_flag = "false";
}
}
catch(Exception ex){string s = ex.Message;}
}
private void Fill_procedure(object sender, DoWorkEventArgs e)
{
int i = (int)e.Argument;
states[i] = new WOrkingThreadState("fill", DateTime.Now, "fill");
Console.WriteLine("Получаем новые задачи на "+i+"-том воркере");
try
{
lock (fill_flag)
{
if (jobs.jobs.Count() < 300)
{
/*Console.WriteLine(DateTime.Now.ToLongTimeString() + " Made1: " + jobs_made + ", queue: " + jobs.jobs.Count.ToString() + "; unsent statuses: " + statuses.Count.ToString() + ", ads: " + ap.Count().ToString()
+ ", photos: " + pq.Count().ToString());*/
FillQueue();
if (statuses.Count == 0 && jobs.jobs.Count == 0)
DataDBURLJobs.ClearAddressParsing(ddbs);
GC.Collect();
}
fill_flag = "false";
}
}
catch(Exception ex){string s = ex.Message;}
}
