public EventQueue(string dbConnectionString, int mode = 0)
{
needjobs += GetJobs;
getpages += GetPages;
processonepage += ProcessOnePage;
getphotosfromonepage += GetPhotosFromOnePage;
statusesadd += StatusesAdd;
sendapartmentsandstatuses += SendApartmentsAndStatuses;
fillqueue += FillQueue;
jobs_made = 0;
//обновляем ссылку на подключение к базе для всяких асинхронных передач данных в духе ошибок
ddbs = new DataDBAsynchService(dbConnectionString);
ddbs.WriteErrorMessage(dbConnectionString, 0, "", "start");
this.mode = mode;
arw = new AsyncRunWork();
starttime = DateTime.Now;
if (jobs != null)
lock (jobs)
jobs = new DataDBURLJobs(ddbs, mode, false);
else
jobs = new DataDBURLJobs(ddbs, mode, false);
lock (jobs)
if (jobs.Count() < 100)
needjobs();
}
public YADataDBURLQueue(string dbConnectionString, int mode = 0)
{
Console.WriteLine("Создаем очередь");
tick += Tick;
work = false;
jobs_made = 0;
got_jobs = new Queue<DataDBURLJob>(0);
photo_aps = new Queue<DirtyApartments>(0);
disposed = false;
//обновляем ссылку на подключение к базе для всяких асинхронных передач данных в духе ошибок
ddbs = new DataDBAsynchService(dbConnectionString);
ddbs.WriteErrorMessage(dbConnectionString, 0, "", "start");
//получаем задания на парсинг
jobs = new DataDBURLJobs(ddbs, mode, false);
//jobs.SetJobToStartOrStop(true);
t_fill = new System.Timers.Timer();
t_fill.Elapsed += new System.Timers.ElapsedEventHandler(fill_timer);
t_fill.Interval = 30000;
t_restart = new System.Timers.Timer();
t_restart.Elapsed += new System.Timers.ElapsedEventHandler(restart_timer);
//рестартить раз в 12 часов
t_restart.Interval = 12 * 60 * 60 * 1000;
this.mode = mode;
pq = new PhotoQueue(ddbs);
arw = new AsyncRunWork();
ap = new apartments();
work = true;
t_fill.Start();
t_restart.Start();
Console.WriteLine("Создали очередь");
}
private void Initialize()
{
string _dbConnectionString = "Data Source=vparsers.r-slon.com;Initial Catalog=AlcionaParsers;Persist Security Info=True;User ID=parsers08;Password=GjrjKtybt2011;Connection Timeout=120;Network Library=DBMSSOCN;Max Pool Size=1000";
ddbs = new DataDBAsynchService(_dbConnectionString);
jobs = new DataDBURLJobs(ddbs, 14, true);
das = new List<Alciona.AlcionaDAL.DirtyApartments>();
made = new List<DataDBURLJob>();
}
private void Form1_Load(object sender, EventArgs e)
{
/*
NpgsqlConnection conn = new NpgsqlConnection("Server=newnine.r-slon.com;Port=5432;User Id=postgres;Password=dar00kat;Database=AlcionaParsers;");
conn.Open();
conn.Close();
*/
StreamReader _fileRead = new StreamReader("dbconnect.txt");
string _dbConnectionString = _fileRead.ReadLine();
_fileRead.Close();
_dbConnectionString = "Data Source=vparsers.r-slon.com;Initial Catalog=AlcionaParsers;Persist Security Info=True;User ID=Alex;Password=AlexPassword13;Connection Timeout=120;Network Library=DBMSSOCN;Max Pool Size=1000";
ddbs = new DataDBAsynchService(_dbConnectionString);
UpdateSourcesList();
}
public DataDBURLJobs(DataDBAsynchService ddbs, int mode, bool get=true)
{
disposed = false;
ClearAddressParsing(ddbs);
if (get)
{
DataDBURLJob[] temp = null;
while (temp == null)
temp = MakeJobs(ddbs, mode);
if (temp != null)
{
jobs = new List<DataDBURLJob>(temp);
GC.SuppressFinalize(temp);
}
}
else
jobs = new List<DataDBURLJob>(0);
}
public static DataDBURLJob[] MakeJobs(DataDBAsynchService ddbs, int mode)
{
DataDBURLJob[] jobs__ = null;
try
{
string[][] jobs_ = GetJobs(ddbs, mode); //берем данные из строки ParsersJob
if (jobs_ != null)
{
jobs__ = new DataDBURLJob[jobs_.Count()];
for (int i = 0; i < jobs_.Count(); i++)
{
//string[] job = jobs_[i];
Array.Resize(ref jobs_[i], jobs_[i].Count() + 1);
jobs__[i] = new DataDBURLJob(i, jobs_[i]);
GC.SuppressFinalize(jobs_[i]);
}
GC.SuppressFinalize(jobs_);
}
}
catch (Exception e) { Console.WriteLine("Ошибка делания джобов " + e.Message); return null; }
return jobs__;
}
public static string[][] GetJobs(DataDBAsynchService ddbs, int mode=0)
{
string[][] jobs_ = null;
try
{
if (!jobs_in_get)
{
jobs_in_get = true;
try
{// имя хоста
Console.WriteLine(DateTime.Now.ToLongTimeString() + " Получаем новые задания на парсинг");
jobs_ = RedJobsSub(ddbs);
if (jobs_==null || jobs_.Count() < 150)
{
if (jobs_!=null)
GC.SuppressFinalize(jobs_);
string q = GetSQLString(ddbs, mode);
Console.WriteLine("Запускаем скрипт получения новой работы " + q);
string s = ddbs.ExecuteSQLCommand(q, ddbs.ConnectionString, 29);
if (!string.IsNullOrEmpty(s))
{
Console.WriteLine("Скрипт получения новой работы завершился с ошибкой " + s);
//ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Скрипт получения новой работы завершился с ошибкой" + s);
//return null;
}
else
{
Console.WriteLine("Скрипт получения новой работы успешно завершился");
jobs_ = RedJobsSub(ddbs);
}
}
}
catch (SystemException ex)
{
string s = ex.Message;
Console.WriteLine("Ошибка четния задания " + s);
//ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка четния задания " + s);
Console.WriteLine("Ошибка четния задания " + s);
}
finally
{ jobs_in_get = false; }
}
}
catch (Exception e) { Console.WriteLine("Произошла ошибка при получении заданий же же " + e.Message); }
return jobs_;
}
public static void ClearAddressParsingOld(DataDBAsynchService ddbs)
{
//update
string SQLstr = "delete from AddressesParsing where Host_Ref=" + GetHostRef(ddbs) + " and Status = 0";
/*ddbs.BeginExecuteSQLCommand(SQLstr, _dbConnectionString);*/
string s2 = ddbs.ExecuteSQLCommand(SQLstr, ddbs.ConnectionString);
if (!string.IsNullOrEmpty(s2))
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка удаления недоделанных работ " + s2);
}
private static int GetHostRef(DataDBAsynchService ddbs)
{
if (hostref == 0)
{
List<int> r = null;
try
{
r = ddbs.GetIntValues(ddbs.ConnectionString,
"select pt.id from ParsersTuning pt where pt.Host='" + Dns.GetHostName() + "'");
if (r != null)
{
if (r.Any())
{
hostref = r[0];
}
else
hostref = 0;
GC.SuppressFinalize(r);
}
else
hostref = 0;
}
catch (Exception ex) { Console.WriteLine("Ошибка получения хострефа "+ex.Message); }
finally { if (r != null) { GC.SuppressFinalize(r); } }
}
return hostref;
}
public static bool SetJobStatuses(Dictionary<int, int> statuses, DataDBAsynchService ddbs)
{
string s2 = "";
bool ret = false;
if (!statuses_are_send)
try
{
statuses_are_send = true;
if (statuses.Count() == 0)
{
statuses_are_send = false;
ret = true;
}
for (int k = 0; k < statuses.Count() / 1000 + 1; k++)
{
// имя хоста// имя хоста
int host_ref = GetHostRef(ddbs);
string s = "";
string SQLstr = "";
int l = 1000;
if (k == statuses.Count() / 1000)
l = statuses.Count() - (statuses.Count() / 1000) * 1000;
{
//lock (statuses)
for (int i = 0; i < l; i++)
{
if (s.Length > 0)
s += ", ";
int val = statuses[statuses.Keys.ToArray()[i + k * 1000]];
if (val == 0) { val = 4; Console.WriteLine("Попытались добавить нулевой статус"); }
s += "(" + statuses.Keys.ToArray()[i + k * 1000].ToString()
+ ", CURRENT_TIMESTAMP, " + val.ToString() + ")";
}
//insert
SQLstr = "insert into AddressesParsingFinish values " + s;
/*ddbs.BeginExecuteSQLCommand(SQLstr, _dbConnectionString); */
//Console.WriteLine(SQLstr);
s2 = ddbs.ExecuteSQLCommand(SQLstr, ddbs.ConnectionString);
if (!string.IsNullOrEmpty(s2))
{
Console.WriteLine("Ошибка записи изменения состояния задания по дате1 " + s2 + " " + SQLstr);
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка записи изменения состояния задания по дате1 "+ s2 + " " + SQLstr );
ret = false;
}
}
}
if (string.IsNullOrEmpty(s2))
ret = true;
else
ret = false;
}
catch (Exception ex)
{
string s = ex.Message;
Console.WriteLine(ddbs.ConnectionString, 0, null, "Ошибка отправки статусов " + s);
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка отправки статусов " + s);
ret = false;
}
finally
{
statuses_are_send = false;
}
else
ret = false;
return ret;
}
public void StartGetPage(DataDBAsynchService ddbs)
{
try
{
status = 0;
if (_encoding != null)
{
encoding = GetEncoding();
if (!string.IsNullOrEmpty(_parserName.Trim()))
{
if (dcs == null)
dcs = new Dictionary<int, DataDBCollector.DataDBCollector>();
if (!dcs.Keys.Contains(_ID))
AddCollector(ddbs);
if (!string.IsNullOrEmpty(_URL))
{
if (_URL.StartsWith("http"))
{
if (pause > 0)
Thread.Sleep(pause);
if (parsertimes.Keys.Contains(parserName))
parsertimes[parserName] = new TimeSpan(DateTime.Now.Ticks);
else
parsertimes.Add(parserName, new TimeSpan(DateTime.Now.Ticks));
error = GetPage(ddbs);
}
else
status = 9;
}
else
{
status = 1;
}
}
else
status = 13; //Пустое наименование парсера
}
else
status = 14; //Пустая кодировка
}
catch (Exception ex)
{ ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "DataDBURLJob. Ошибка в получении страницы " + ex.Message); }
finally
{ if(page!=null)GC.SuppressFinalize(page); }
}
private static string[][] RedJobsSub(DataDBAsynchService ddbs)
{
//Console.WriteLine(DateTime.Now.ToLongTimeString() + " Начали получать расшифровку заданий");
List<string[]> jobs = null;
string SQLstr = "";
try
{
SQLstr =
//"begin tran tran1; "+
//"SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; "+
"select pj.ParserName, pj.id parser_id, am.[ID], am.[Address],am.[Sticker],[Encoding],[IpProxy],[PortProxy],[WebPreAuthentificationPage] "
+ ",[WebAuthentificationPage],[WebConnectionString],[Site],[Source],[PathForPhoto], DelayFrom, DelayTo, ap.id "
+ "from AddressesMask am with(xlock) "
+ "inner join PjobStat pj "
+ "on pj.ID=am.JobID "
//+ "inner join AddressesParsing ap with(xlock) "
+ "inner join AddressesParsingStart ap "
+ "with(xlock) on ap.Address_ref=am.id "
+ " where ap.id not in(select AddressStart_Ref from AddressesParsingFinish) and ap.Host_Ref= " + GetHostRef(ddbs);
//+"; commit tran tran1;";
//+ " order by pj.ParserName";
lock(DataDBAsynchService.db_lock)
using(SqlConnection sc =ddbs.GetDBCon(ddbs.ConnectionString))
try
{
using(SqlCommand SQLCmd = new SqlCommand(SQLstr, sc))
try
{
SQLCmd.CommandTimeout = 29;
using (SqlDataReader MydataReader2 = SQLCmd.ExecuteReader())
try
{
if (MydataReader2 != null)
if (MydataReader2.HasRows)
try
{
jobs = new List<string[]>();
object o0 = "";
//if (MydataReader2.HasRows)
while (MydataReader2.Read())
{
string[] job = new string[17];
//Console.WriteLine(DateTime.Now.ToLongTimeString() + " Считали строку расшифровки");
try
{
if (MydataReader2.FieldCount == 17)
{
if (MydataReader2.IsDBNull(0))
o0 = "";
else
o0 = MydataReader2.GetSqlString(0).ToString();
/*Array.Resize(ref job, 1);*/
job[0] = (string)o0; //[ID]
///*Array.Resize(ref job, 1);*/ job[0] = "70";
if (MydataReader2.IsDBNull(1))
o0 = "0";
else
o0 = MydataReader2.GetInt32(1).ToString();
/*Array.Resize(ref job, 2);*/
job[1] = (string)o0;//[ParserJobID]
///*Array.Resize(ref job, 2);*/ job[1] = "GilcomGotovoeDataCollector";
if (MydataReader2.IsDBNull(2))
o0 = "0";
else
o0 = MydataReader2.GetInt32(2).ToString();
/*Array.Resize(ref job, 3);*/
job[2] = (string)o0;//[AddressMaskID]
///*Array.Resize(ref job, 2);*/ job[1] = "GilcomGotovoeDataCollector";
if (MydataReader2.IsDBNull(3))
o0 = "";
else
o0 = MydataReader2.GetSqlString(3).ToString();
/*Array.Resize(ref job, 4);*/
job[3] = (string)o0;//[Encoding]
///*Array.Resize(ref job, 3);*/ job[2] = "UTF8";
if (MydataReader2.IsDBNull(4))
o0 = "";
else
o0 = MydataReader2.GetSqlString(4).ToString();
/*Array.Resize(ref job, 5);*/
job[4] = (string)o0;
if (MydataReader2.IsDBNull(5))
o0 = "";
else
o0 = MydataReader2.GetSqlString(5).ToString();
/*Array.Resize(ref job, 6);*/
job[5] = (string)o0;
if (MydataReader2.IsDBNull(6))
o0 = "";
else
o0 = MydataReader2.GetSqlString(6).ToString();
/*Array.Resize(ref job, 7);*/
job[6] = (string)o0;
if (MydataReader2.IsDBNull(7))
o0 = "0";
else
o0 = MydataReader2.GetInt32(7).ToString();
/*Array.Resize(ref job, 8);*/
job[7] = (string)o0;
if (MydataReader2.IsDBNull(8))
o0 = "";
else
o0 = MydataReader2.GetSqlString(8).ToString();
/*Array.Resize(ref job, 9);*/
job[8] = (string)o0;
if (MydataReader2.IsDBNull(9))
o0 = "";
else
o0 = MydataReader2.GetSqlString(9).ToString();
/*Array.Resize(ref job, 10);*/
job[9] = (string)o0;
if (MydataReader2.IsDBNull(10))
o0 = "";
else
o0 = MydataReader2.GetSqlString(10).ToString();
/*Array.Resize(ref job, 11);*/
job[10] = (string)o0;
if (MydataReader2.IsDBNull(11))
o0 = "";
else
o0 = MydataReader2.GetSqlString(11).ToString();
/*Array.Resize(ref job, 12);*/
job[11] = (string)o0;
if (MydataReader2.IsDBNull(12))
o0 = "";
else
o0 = MydataReader2.GetSqlString(12).ToString();
/*Array.Resize(ref job, 13);*/
job[12] = (string)o0;
if (MydataReader2.IsDBNull(13))
o0 = "";
else
o0 = MydataReader2.GetSqlString(13).ToString();
/*Array.Resize(ref job, 14);*/
job[13] = (string)o0;
if (MydataReader2.IsDBNull(14))
o0 = "0";
else
o0 = MydataReader2.GetInt32(14).ToString();
/*Array.Resize(ref job, 15);*/
job[14] = (string)o0;
if (MydataReader2.IsDBNull(15))
o0 = "0";
else
o0 = MydataReader2.GetInt32(15).ToString();
/*Array.Resize(ref job, 16);*/
job[15] = (string)o0;
if (MydataReader2.IsDBNull(16))
o0 = "0";
else
o0 = MydataReader2.GetInt32(16).ToString();
/*Array.Resize(ref job, 17);*/
job[16] = (string)o0;
/*if (MydataReader2.IsDBNull(14))
o0 = "0";
else
o0 = MydataReader2.GetSqlString(14).ToString();
/*Array.Resize(ref job, 15);*/
//job[14] = (string)o0;
//Array.Resize(ref jobs, jobs.Count() + 1);
jobs.Add(job);
}
}
catch (SystemException exx)
{
SQLCmd.Cancel();
if (o0 != null)
{
Console.WriteLine("Ошибка четния задания2. " + exx.Message + " получено " + o0.ToString());
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка четния задания2 " + SQLstr + ". " + exx.Message + " получено " + o0.ToString());
}
else
{
Console.WriteLine("Ошибка четния задания2 " + exx.Message + " получено null");
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка четния задания2 " + exx.Message + " получено null");
}
}
finally
{
GC.SuppressFinalize(job);
}
}
/*else
{
Console.WriteLine("Запрос вернул 0 строк. Вроде. Или HasRows косячит");
}*/
}
catch (Exception ex)
{
SQLCmd.Cancel();
Console.WriteLine("Ошибка четния задания1 " + ex.Message);
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка четния задания1 " + ex.Message);
}
}
catch { }
finally
{
if (MydataReader2 != null)
{
if (!MydataReader2.IsClosed)
MydataReader2.Close();
MydataReader2.Dispose();
GC.SuppressFinalize(MydataReader2);
}
}
}
catch (Exception ex)
{
SQLCmd.Cancel();
Console.WriteLine("Ошибка четния задания3 " + ex.Message);
//ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка четния задания3 " + ex.Message);
//return null;
}
finally
{
SQLCmd.Dispose();
GC.SuppressFinalize(SQLCmd);
}
}
catch (Exception ex)
{
Console.WriteLine("Ошибка четния задания4 " + ex.Message);
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка четния задания4 " + ex.Message);
//return null;
}
finally
{
if (sc != null)
{
if(sc.State!= System.Data.ConnectionState.Closed)
sc.Close();
sc.Dispose();
GC.SuppressFinalize(sc);
}
}
}
catch (SystemException ex)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, null, "Ошибка четния задания " + ". " + ex.Message);
//return null;
}
finally
{
//GC.Collect();
if (jobs!=null)
Console.WriteLine(DateTime.Now.ToLongTimeString() + " Закончили получать задания. Получили: " + jobs.Count());
else
Console.WriteLine(DateTime.Now.ToLongTimeString() + " Закончили получать задания. Но не получили(");
}
/*List<string[]> jobs_ret = new List<string[]>(jobs.Count());
for (int i = 0; i < 10; i++)
for (int j = 0; j <= jobs.Count() / 10;j++ )
if (10 * j + i < jobs.Count())
jobs_ret.Add(jobs[10 * j + i]);*/
string[][] jobs_ = null;
if (jobs != null)
{
if (jobs.Any())
{
try
{
jobs_ = new string[jobs.Count][];
for (int i = 0; i < jobs.Count; i++)
{
jobs_[i] = jobs[i];
GC.SuppressFinalize(jobs[i]);
}
jobs.Clear();
}
catch { Console.WriteLine("Произошла ошибка в финальной обработке заданий"); }
finally
{
GC.SuppressFinalize(jobs);
}
}
}
return jobs_;
}
public static DateTime GetRebootDateTime(DataDBAsynchService ddbs)
{
DateTime dt = DateTime.MinValue;
List<string> t = null;
if(ddbs!=null)
try
{
t = ddbs.GetStrValues(ddbs.ConnectionString, "select value from Globals where name='reboot_datetime'");
if (t != null)
if (t.Count > 0)
DateTime.TryParse(t[0], out dt);
/*if(DateTime.TryParse(t[0],out dt))
dt = DateTime.Parse(t[0]);*/
}
catch (Exception ex) { Console.WriteLine(DateTime.Now.ToLongTimeString() + " Ошибка получения перезагрузки " + ex.Message); }
finally { if (t != null) { t.Clear(); GC.SuppressFinalize(t); } }
return dt;
}
public DirtyApartments StartProcessPage(DataDBAsynchService ddbs, bool ParsePhones)
{
DirtyApartments temp = null;
try
{
temp = ProcessPage(ParsePhones, ddbs);
//page = null;
//dc = null;
if (temp != null)
status = 1;
else
status = 8;
}
catch (Exception ex)
{ Console.WriteLine("Произошла ошибка в процессе обработки страницы " + ex.Message); }
finally
{
if (page != null)
GC.SuppressFinalize(page);
}
return temp;
}
private static string GetSQLString(DataDBAsynchService ddbs, int mode)
{
return "exec SetWork " + mode + ", " + GetHostRef(ddbs);
}
public bool AddJobs(DataDBAsynchService ddbs, int mode)
{
bool r = false;
string[][] jobs_ = null;
try
{
jobs_ = GetJobs(ddbs, mode); //берем данные из строки ParsersJob
if (jobs_ != null)
{
if (jobs_.Any())
lock (jobs)
if (!jobs.Any())
jobs.Clear();
for (int i = 0; i < jobs_.Count(); i++)
{
using (DataDBURLJob temp = new DataDBURLJob(i, jobs_[i]))
try
{
lock (jobs)
if (!jobs.Exists(x => x._addressparsing == temp._addressparsing))
jobs.Add(temp);
}
catch (Exception ex) { ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, "", "ошибка при добавлении заданий " + ex.Message); }
finally { if (temp != null) GC.SuppressFinalize(temp); if (jobs_[i] != null) GC.SuppressFinalize(jobs_[i]); }
}
}
r = true;
}
catch (Exception e) { Console.WriteLine("Ошибка в добавлении объявлений же " + e.Message); }
finally { if (jobs_!=null) GC.SuppressFinalize(jobs_); }
return r;
}
private void AddCollector(DataDBAsynchService ddbs)
{
Assembly assembly = Assembly.Load("DataDBPlugin");
Type TestType = assembly.GetType("DataDBCollector." + _parserName.Trim(), false, true);
//если класс найден
if (TestType != null)
{
System.Reflection.ConstructorInfo ci = TestType.GetConstructor(new Type[]
{
typeof(string), typeof(string), typeof(string), typeof(string), typeof(string), typeof(Encoding), typeof(int), typeof(int)
, typeof(int), typeof(int), typeof(string), typeof(int), typeof(int), typeof(int), typeof(string), typeof(string)//, typeof(string)
});
if (ci != null) // такой парсер есть
{
//вызываем конструтор
object dcReflected = ci.Invoke(new object[]
{
ddbs.ConnectionString, _webConnectionString, _webPreAuthentificationPage, _webAuthentificationPage,"",
encoding,0,0,0,0, _ipProxy, _portProxy, _streamNumber, 0, _pathForPhoto, _source //, _ip
});
//Что нам мешает привести dcReflected к типу dc, и пользоваться методами класса напрямую?
dcs.Add(_ID, (DataDBCollector.DataDBCollector)dcReflected);
}
else
status = 12; //Плагин парсера не найден
ci = null;
}
else
status = 12; //Плагин парсера не найден
}
public void SetPhones(DataDBAsynchService ddbs, ref DirtyApartments da)
{
if (da != null)
{
try
{
string SQLstr =
"select Телефон1, Телефон2, Телефон3, Телефон4 from MasksPhones pm inner join AddressesParsing ap on ap.Address_ref=pm.Address_Ref where ap.id=" + _addressparsing;
using(SqlConnection conn = ddbs.GetDBCon(ddbs.ConnectionString))
try
{
SqlCommand SQLCmd = new SqlCommand(SQLstr, conn);
using (SqlDataReader MydataReader2 = SQLCmd.ExecuteReader())
{
if (MydataReader2.HasRows)
{
while (MydataReader2.Read())
{
if (MydataReader2.IsDBNull(0))
da.Телефон1 = null;
else
da.Телефон1 = MydataReader2.GetSqlString(0).ToString();
if (MydataReader2.IsDBNull(1))
da.Телефон2 = null;
else
da.Телефон2 = MydataReader2.GetSqlString(1).ToString();
if (MydataReader2.IsDBNull(2))
da.Телефон3 = null;
else
da.Телефон3 = MydataReader2.GetSqlString(2).ToString();
if (MydataReader2.IsDBNull(3))
da.Телефон4 = null;
else
da.Телефон4 = MydataReader2.GetSqlString(3).ToString();
}
}
MydataReader2.Close();
}
}
catch (Exception ex) { Console.WriteLine(ex.Message); }
finally
{ conn.Close(); conn.Dispose(); }
}
catch (Exception ex)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, 0, da.Ссылка, "Ошибка получения телефонов по троечке: " + ex.Message, _addressparsing);
}
}
}
public DirtyApartments ProcessPage(bool ParsePhones, DataDBAsynchService ddbs = null)
{
DirtyApartments da = null;
try
{
//if (dc == null)
{
if (dcs == null)
dcs = new Dictionary<int, DataDBCollector.DataDBCollector>();
if (!dcs.Keys.Contains(_ID))
AddCollector(ddbs);
}
if (dcs[_ID] != null)
try
{
dcs[_ID]._directoryInfo = new DirectoryInfo("out");
dcs[_ID]._streamNumber = _streamNumber;
da = dcs[_ID].CardParsing(page, _URL, _stiker, ParsePhones);
}
catch (Exception ex)
{
error = ex.Message;
ddbs.WriteErrorMessage(ddbs.ConnectionString, streamNumber, _URL, "DataDBURLJob. ошибка обработки страницы: " + error, _addressparsing);
da = null;
}
else
status = 12;
//page = new HtmlDocument();
if(page!=null)
GC.SuppressFinalize(page);
//page = null;
if (da != null)
{
try
{
status = 1;
da.AddressParsing = _addressparsing;
if (da.Источник != _source)
{
Console.WriteLine("Установлен левый источник " + da.Источник + "вместо" + _source);
}
}
catch (Exception ex)
{
string s2 = ex.Message;
ddbs.WriteErrorMessage(ddbs.ConnectionString, streamNumber, _URL, "DataDBURLJob. ошибка обработки страницы: " + s2, _addressparsing);
status = 5;
da = null;
}
}
else
{
status = 6;
ddbs.WriteErrorMessage(ddbs.ConnectionString, streamNumber, _URL, "DataDBURLJob. ошибка обработки страницы: пустая dirtyapartments (произошла ошибка при обработке этой страницы)", _addressparsing);
da = null;
}
}
catch (Exception ex)
{
string s = ex.Message;
string s1 = ex.Source;
string s2 = ex.StackTrace;
Exception o1 = ex.InnerException;
object o3 = ex.TargetSite;
object o4 = ex.Data;
Console.WriteLine("Произошла ошибка при обработке страницы " + s); status = 7;
da = null;
}
if (status == 0)
status = 20;
return da;
}
private string GetPage(DataDBAsynchService ddbs)
{
//обработка страницы и запись ее в таблицу
if (!string.IsNullOrEmpty(_URL))
{
page = dcs[_ID].WebClientPage(null, _URL);
}
if (page == null && !string.IsNullOrEmpty(_URL))
{
status = 4;
if (dcs[_ID].GetException != null)
{
ddbs.WriteErrorMessage(ddbs.ConnectionString, streamNumber, _URL, "DataDBURLJob. ошибка получения страницы: " + dcs[_ID].GetException.Message, _addressparsing);
return dcs[_ID].GetException.Message;
}
}
else
status = 1;
return "";
}
public DataDBURLQueue(string dbConnectionString, int maxthreads, int mode=0)
{
//обновляем ссылку на подключение к базе для всяких асинхронных передач данных в духе ошибок
ddbs = new DataDBAsynchService(dbConnectionString);
//получаем задания на парсинг
jobs = new DataDBURLJobs(ddbs, mode,false);
this.max_threads = maxthreads;
//jobs.SetJobToStartOrStop(true);
t_fill = new System.Timers.Timer();
t_fill.Elapsed += new System.Timers.ElapsedEventHandler(fill_timer);
t_fill.Interval = 30000;
t_restart = new System.Timers.Timer();
t_restart.Elapsed += new System.Timers.ElapsedEventHandler(restart_timer);
//рестартить раз в 12 часов
t_restart.Interval = 12*1000*60*60;
this.mode = mode;
bws = new BackgroundWorker[max_threads];
states = new WOrkingThreadState[max_threads];
pq = new DataDBCollector.PhotoQueue(ddbs);
//ThreadPool.SetMaxThreads(40, 40);
}
public NewDataDBURLQueue(string dbConnectionString, int maxthreads, int mode=0)
{
//обновляем ссылку на подключение к базе для всяких асинхронных передач данных в духе ошибок
ddbs = new DataDBAsynchService(dbConnectionString);
//получаем задания на парсинг
jobs = new DataDBURLJobs(ddbs, mode,false);
this.max_threads = maxthreads;
this.mode = mode;
bws = new AbortableBackgroundWorker[max_threads];
states = new WOrkingThreadState[max_threads];
pq = new DataDBCollector.PhotoQueue(ddbs);
//ThreadPool.SetMaxThreads(40, 40);
}
