private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
_Worker1 = new IE();
_Worker2 = new IE();
#region Factory.ca
_ScrapeUrl = "http://www.factorydirect.ca/SearchResults.aspx";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for factorydirect.ca Website";
int counterReload = 0;
bool isError = false;
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10000);
_Work1doc.LoadHtml(_Worker1.Html);
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//b[@class=\"nxt-result-total\"]");
if (_CollectionCatLink != null)
{
try
{
_TotalRecords = Convert.ToInt32(_CollectionCatLink[0].InnerText.Trim());
if ((_TotalRecords % 36) == 0)
{
_Pages = Convert.ToInt32(_TotalRecords / 36);
}
else
{
_Pages = Convert.ToInt32(_TotalRecords / 36) + 1;
}
}
catch
{ }
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (_TotalRecords > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from search page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
for (int Page = 1; Page <= _Pages; Page++)
{
CategoryUrl.Add("http://www.factorydirect.ca/SearchResults.aspx#/?search_return=all&res_per_page=36&page=" + Page);
}
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (string url in CategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url;
_Work.RunWorkerAsync();
}
else
{
Url2 = url;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + Producturl.Count.ToString();
foreach (var url in Producturl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for factorydirect.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data factorydirect.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion Factory.CA
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_Stop = false;
time = 0;
#region canadacomputers.com
_IScanadacomputers = true;
_ScrapeUrl = "http://www.canadacomputers.com/asus/notebooks.php";
try
{
//_Worker1 = new IE();
//_Worker2 = new IE();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Work1doc.LoadHtml(_Client1.DownloadString(_ScrapeUrl));
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"hd-nav-prod-dropdn\"]");
if (_Collection != null)
{
HtmlNodeCollection _Collection11 = _Collection[0].SelectNodes(".//a");
if (_Collection11 != null)
{
foreach (HtmlNode node in _Collection11)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
try
{
if (att.Value.Trim() != string.Empty && att.Value.Trim() != "#")
CategoryUrl.Add((att.Value.ToLower().Contains("canadacomputers.com") ? "" : "http://www.canadacomputers.com") + att.Value.Replace("..", ""), "CANCOM" + node.InnerText.Trim());
}
catch
{ }
}
}
}
}
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("canadacomputers.com", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
int i = 0;
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
DisplayRecordProcessdetails("We are going to read product url from sub-category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
_401index = 0;
_IsSubcat = true;
foreach (var Caturl in subCategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
}
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
if (!_IsCategorypaging)
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
MessageBox.Show(exp.Message);
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "canadacomputers.com", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='canadacomputers.com'");
_Mail.SendMail("OOPS there is no any product scrapped by app for canadacomputers.com Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='canadacomputers.com'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data canadacomputers.com Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region 401Games
_IS401games = true;
_ScrapeUrl = "http://store.401games.ca/";
try
{
_Worker1 = new IE();
_Worker2 = new IE();
// _Worker1.Visible = false;
// _Worker2.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10000);
_Work1doc.LoadHtml(_Worker1.Html);
//HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"col left\"]");
//CategoryUrl = CommanFunction.GetCategoryUrl(_Collection, "ul", "//li/a", "http://store.401games.ca", "#st=&begin=1&nhit=40");
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"sub-menu\"]");
if (_Collection != null)
{
HtmlNodeCollection menu = _Collection[0].SelectNodes("..//ul[@class=\"submenu\"]//li//a");
foreach (HtmlNode node in menu)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
CategoryUrl.Add(att.Value, node.InnerText.Trim());
}
}
}
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/shipping/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/returns/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.cahttp://payd.moneris.com/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/privacy/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/catalog/93370C/pre-orders#st=&begin=1&nhit=40");
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("store.401games", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = "http://store.401games.ca" + PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = "http://store.401games.ca" + PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch
{
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "store.401games", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='store.401games'");
_Mail.SendMail("OOPS there is no any product scrapped by app for store.401games Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='store.401games'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data store.401games Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_Stop = false;
time = 0;
#region tigerdirect.ca
_IStigerdirect = true;
_ScrapeUrl = "http://tigerdirect.ca/";
try
{
_Worker1 = new IE();
_Worker2 = new IE();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10);
_Work1doc.LoadHtml(_Worker1.Html);
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//ul[@class=\"mastNav-subCats\"]/li/a");
if (_Collection != null)
{
foreach (HtmlNode node in _Collection)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
try
{
CategoryUrl.Add("http://www.tigerdirect.ca" + (att.Value.Contains("?") ? att.Value + "&recs=30" : att.Value + "?recs=30"), "TGRDRCT" + node.InnerText.Trim());
}
catch
{ }
}
}
}
try
{
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_1565_8718_8718&cm_re=Printers-_-Spot%2001-_-Laser%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_1565_84868_84868&cm_re=Printers-_-Spot%2002-_-Inkjet%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=25&name=scanners&cm_re=Printers-_-Spot%2003-_-Scanners&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/category/category_slc.asp?CatId=243&cm_re=Printers-_-Spot%2004-_-Label%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_36_84863_84863&cm_re=Printers-_-Spot%2005-_-Mobile&pagesize=30", "printer");
}
catch
{ }
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("tigerdirect.ca", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
int i = 0;
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
i++;
//if (i == 3)
// break;
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
DisplayRecordProcessdetails("We are going to read product url from sub-category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
_401index = 0;
_IsSubcat = true;
foreach (var Caturl in subCategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
}
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
MessageBox.Show(exp.Message);
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "tigerdirect.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='tigerdirect.ca'");
_Mail.SendMail("OOPS there is no any product scrapped by app for tigerdirect.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='tigerdirect.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data tigerdirect.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region bargainballoons.ca
_ScrapeUrl = "http://www.bargainballoons.ca";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for bargainballoons.ca Website";
int counterReload = 0;
bool isError = false;
_Work1doc.LoadHtml(_Client1.DownloadString(_ScrapeUrl));
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//ul[@class=\"subMenuList\"]");
if (_CollectionCatLink != null)
{
try
{
foreach (HtmlNode node in _CollectionCatLink)
{
foreach (HtmlNode node1 in node.SelectNodes(".//a"))
{
foreach (HtmlAttribute attr in node1.Attributes)
{
if (attr.Name == "href")
{
try
{
CategoryUrl.Add(_ScrapeUrl + "/" + attr.Value, "BBALCA" + node1.InnerText.Trim());
}
catch
{
}
}
}
}
}
}
catch
{ }
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (CategoryUrl.Count > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read paging from category page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (var url in CategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Category Pages :" + allCategoryUrl.Count.ToString();
foreach (var url in allCategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "bargainballoons.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='bargainballoons.ca'");
_Prd.ProductDatabaseIntegration(Products, "bargainballoons.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for bargainballoons.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "bargainballoons.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='bargainballoons.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for bargainballoons.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for bargainballoons.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "bargainballoons.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='bargainballoons.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for bargainballoons.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for bargainballoons.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='bargainballoons.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data bargainballoons.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion bargainballoons.ca
