public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_Stop = false;
time = 0;
#region tigerdirect.ca
_IStigerdirect = true;
_ScrapeUrl = "http://tigerdirect.ca/";
try
{
_Worker1 = new IE();
_Worker2 = new IE();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10);
_Work1doc.LoadHtml(_Worker1.Html);
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//ul[@class=\"mastNav-subCats\"]/li/a");
if (_Collection != null)
{
foreach (HtmlNode node in _Collection)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
try
{
CategoryUrl.Add("http://www.tigerdirect.ca" + (att.Value.Contains("?") ? att.Value + "&recs=30" : att.Value + "?recs=30"), "TGRDRCT" + node.InnerText.Trim());
}
catch
{ }
}
}
}
try
{
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_1565_8718_8718&cm_re=Printers-_-Spot%2001-_-Laser%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_1565_84868_84868&cm_re=Printers-_-Spot%2002-_-Inkjet%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=25&name=scanners&cm_re=Printers-_-Spot%2003-_-Scanners&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/category/category_slc.asp?CatId=243&cm_re=Printers-_-Spot%2004-_-Label%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_36_84863_84863&cm_re=Printers-_-Spot%2005-_-Mobile&pagesize=30", "printer");
}
catch
{ }
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("tigerdirect.ca", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
int i = 0;
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
i++;
//if (i == 3)
// break;
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
DisplayRecordProcessdetails("We are going to read product url from sub-category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
_401index = 0;
_IsSubcat = true;
foreach (var Caturl in subCategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
}
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
MessageBox.Show(exp.Message);
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "tigerdirect.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='tigerdirect.ca'");
_Mail.SendMail("OOPS there is no any product scrapped by app for tigerdirect.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='tigerdirect.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data tigerdirect.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_Stop = false;
time = 0;
#region canadacomputers.com
_IScanadacomputers = true;
_ScrapeUrl = "http://www.canadacomputers.com/asus/notebooks.php";
try
{
//_Worker1 = new IE();
//_Worker2 = new IE();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Work1doc.LoadHtml(_Client1.DownloadString(_ScrapeUrl));
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"hd-nav-prod-dropdn\"]");
if (_Collection != null)
{
HtmlNodeCollection _Collection11 = _Collection[0].SelectNodes(".//a");
if (_Collection11 != null)
{
foreach (HtmlNode node in _Collection11)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
try
{
if (att.Value.Trim() != string.Empty && att.Value.Trim() != "#")
CategoryUrl.Add((att.Value.ToLower().Contains("canadacomputers.com") ? "" : "http://www.canadacomputers.com") + att.Value.Replace("..", ""), "CANCOM" + node.InnerText.Trim());
}
catch
{ }
}
}
}
}
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("canadacomputers.com", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
int i = 0;
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
DisplayRecordProcessdetails("We are going to read product url from sub-category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
_401index = 0;
_IsSubcat = true;
foreach (var Caturl in subCategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
}
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
if (!_IsCategorypaging)
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
MessageBox.Show(exp.Message);
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "canadacomputers.com", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='canadacomputers.com'");
_Mail.SendMail("OOPS there is no any product scrapped by app for canadacomputers.com Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='canadacomputers.com'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data canadacomputers.com Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region 401Games
_IS401games = true;
_ScrapeUrl = "http://store.401games.ca/";
try
{
_Worker1 = new IE();
_Worker2 = new IE();
// _Worker1.Visible = false;
// _Worker2.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10000);
_Work1doc.LoadHtml(_Worker1.Html);
//HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"col left\"]");
//CategoryUrl = CommanFunction.GetCategoryUrl(_Collection, "ul", "//li/a", "http://store.401games.ca", "#st=&begin=1&nhit=40");
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"sub-menu\"]");
if (_Collection != null)
{
HtmlNodeCollection menu = _Collection[0].SelectNodes("..//ul[@class=\"submenu\"]//li//a");
foreach (HtmlNode node in menu)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
CategoryUrl.Add(att.Value, node.InnerText.Trim());
}
}
}
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/shipping/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/returns/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.cahttp://payd.moneris.com/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/privacy/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/catalog/93370C/pre-orders#st=&begin=1&nhit=40");
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("store.401games", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = "http://store.401games.ca" + PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = "http://store.401games.ca" + PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch
{
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "store.401games", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='store.401games'");
_Mail.SendMail("OOPS there is no any product scrapped by app for store.401games Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='store.401games'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data store.401games Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
