private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
_Worker1 = new IE();
_Worker2 = new IE();
#region Factory.ca
_ScrapeUrl = "http://www.factorydirect.ca/SearchResults.aspx";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for factorydirect.ca Website";
int counterReload = 0;
bool isError = false;
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10000);
_Work1doc.LoadHtml(_Worker1.Html);
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//b[@class=\"nxt-result-total\"]");
if (_CollectionCatLink != null)
{
try
{
_TotalRecords = Convert.ToInt32(_CollectionCatLink[0].InnerText.Trim());
if ((_TotalRecords % 36) == 0)
{
_Pages = Convert.ToInt32(_TotalRecords / 36);
}
else
{
_Pages = Convert.ToInt32(_TotalRecords / 36) + 1;
}
}
catch
{ }
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (_TotalRecords > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from search page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
for (int Page = 1; Page <= _Pages; Page++)
{
CategoryUrl.Add("http://www.factorydirect.ca/SearchResults.aspx#/?search_return=all&res_per_page=36&page=" + Page);
}
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (string url in CategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url;
_Work.RunWorkerAsync();
}
else
{
Url2 = url;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + Producturl.Count.ToString();
foreach (var url in Producturl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for factorydirect.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data factorydirect.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion Factory.CA
writer.Close();
try { _Worker1.Close(); _Worker2.Close(); }
catch
{
}
this.Close();
}
private void Go_Click(object sender, EventArgs e)
{
//string ss="=202{203}202{567}";
//string[] cominations = ss.Split(new string[] { "202"}, StringSplitOptions.None);
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region hockeysupremacy.com
_ISBuy = true;
_ScrapeUrl = "https://hockeysupremacy.com/";
try
{
_Work1doc.LoadHtml(_Client1.DownloadString(_ScrapeUrl));
if (_Work1doc.DocumentNode.SelectNodes("//a[@class=\"level1 view-all\"]") != null)
{
foreach (HtmlNode node in _Work1doc.DocumentNode.SelectNodes("//a[@class=\"level1 view-all\"]"))
{
foreach (HtmlAttribute _attr in node.Attributes)
{
if (_attr.Name == "href")
{
try
{
CategoryUrl.Add(_attr.Value.Contains("?") ? _attr.Value + "&limit=all" : _attr.Value + "?limit=all", "HCKYSUP" + node.InnerText.Trim().ToLower().Replace("view all", ""));
}
catch
{
}
}
}
}
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for hockeysupremacy.com Website";
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (CategoryUrl.Count() > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from category page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (var url in CategoryUrl)
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + _ProductUrl.Count.ToString();
foreach (var url in _ProductUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "hockeysupremacy.com", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='hockeysupremacy.com'");
_Prd.ProductDatabaseIntegration(Products, "hockeysupremacy.com", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for hockeysupremacy.com Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "hockeysupremacy.com", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='hockeysupremacy.com'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for hockeysupremacy.com Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for hockeysupremacy.com Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='hockeysupremacy.com'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data hockeysupremacy.com Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion hockeysupremacy.com
writer.Close();
this.Close();
}
private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region hitgames.ca
_ISBuy = true;
_ScrapeUrl = "http://www.Hitgames.ca/";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for Hitgames.ca Website";
_Work1doc.LoadHtml(_Client1.DownloadString(_ScrapeUrl));
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"menuleft\"]");
if (_CollectionCatLink != null)
{
try
{
HtmlNodeCollection _CollectionCatLink1 = _CollectionCatLink[0].SelectNodes(".//li/a");
if (_CollectionCatLink1 != null)
{
foreach (HtmlNode node in _CollectionCatLink1)
{
foreach (HtmlAttribute _attr in node.Attributes)
{
if (_attr.Name == "href")
{
try
{
CategoryUrl.Add((_attr.Value.Contains("?") ? _attr.Value + "&limit=24" : _attr.Value + "?limit=24"), "HTGM" + node.InnerText.Trim());
}
catch
{ }
}
}
}
}
}
catch
{ }
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (CategoryUrl.Count() > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from category page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (var url in CategoryUrl)
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + _ProductUrl.Count.ToString();
foreach (var url in _ProductUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "Hitgames.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='Hitgames.ca'");
_Prd.ProductDatabaseIntegration(Products, "Hitgames.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for Hitgames.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "Hitgames.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='Hitgames.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for Hitgames.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for Hitgames.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "Hitgames.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='Hitgames.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for Hitgames.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for Hitgames.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='Hitgames.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data Hitgames.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion hitgames.ca
writer.Close();
this.Close();
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_Stop = false;
time = 0;
#region canadacomputers.com
_IScanadacomputers = true;
_ScrapeUrl = "http://www.canadacomputers.com/asus/notebooks.php";
try
{
//_Worker1 = new IE();
//_Worker2 = new IE();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Work1doc.LoadHtml(_Client1.DownloadString(_ScrapeUrl));
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"hd-nav-prod-dropdn\"]");
if (_Collection != null)
{
HtmlNodeCollection _Collection11 = _Collection[0].SelectNodes(".//a");
if (_Collection11 != null)
{
foreach (HtmlNode node in _Collection11)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
try
{
if (att.Value.Trim() != string.Empty && att.Value.Trim() != "#")
CategoryUrl.Add((att.Value.ToLower().Contains("canadacomputers.com") ? "" : "http://www.canadacomputers.com") + att.Value.Replace("..", ""), "CANCOM" + node.InnerText.Trim());
}
catch
{ }
}
}
}
}
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("canadacomputers.com", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
int i = 0;
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
DisplayRecordProcessdetails("We are going to read product url from sub-category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
_401index = 0;
_IsSubcat = true;
foreach (var Caturl in subCategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
}
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
if (!_IsCategorypaging)
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
MessageBox.Show(exp.Message);
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "canadacomputers.com", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='canadacomputers.com'");
_Mail.SendMail("OOPS there is no any product scrapped by app for canadacomputers.com Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='canadacomputers.com'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data canadacomputers.com Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
_Worker1 = new IE();
_Worker2 = new IE();
#region Factory.ca
_ScrapeUrl = "http://www.factorydirect.ca/SearchResults.aspx";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for factorydirect.ca Website";
int counterReload = 0;
bool isError = false;
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10000);
_Work1doc.LoadHtml(_Worker1.Html);
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//b[@class=\"nxt-result-total\"]");
if (_CollectionCatLink != null)
{
try
{
_TotalRecords = Convert.ToInt32(_CollectionCatLink[0].InnerText.Trim());
if ((_TotalRecords % 36) == 0)
{
_Pages = Convert.ToInt32(_TotalRecords / 36);
}
else
{
_Pages = Convert.ToInt32(_TotalRecords / 36) + 1;
}
}
catch
{ }
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (_TotalRecords > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from search page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
for (int Page = 1; Page <= _Pages; Page++)
{
CategoryUrl.Add("http://www.factorydirect.ca/SearchResults.aspx#/?search_return=all&res_per_page=36&page=" + Page);
}
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (string url in CategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url;
_Work.RunWorkerAsync();
}
else
{
Url2 = url;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + Producturl.Count.ToString();
foreach (var url in Producturl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for factorydirect.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data factorydirect.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion Factory.CA
private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region openaparty.com
_ISBuy = true;
_ScrapeUrl = "http://www.openaparty.com/applications/Category/guidedSearch.asp?CatId=12&cm_re=Homepage-_--_-CatId_12";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for openaparty.com Website";
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-all-foil-and-latex-balloons-colour-c-879_1688_999#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-balloon-weights-c-879_1688_1173#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-balloon-bouquets-c-879_1688_1160#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-foil-and-latex-french-balloons-c-879_1688_1169#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-happy-birthday-foil-balloons-c-879_1688_1182#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-happy-birthday-latex-balloons-c-879_1688_1166#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-jumbo-letter-foils-c-879_1688_1181#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-jumbo-number-foil-balloons-c-879_1688_1180#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-latex-and-foil-age-balloons-c-879_1688_1175#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-patterned-latex-balloons-c-879_1688_1172#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/all-balloons-curling-ribbon-c-879_1688_1191#bc", "OPAR200 balloons");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-crayons-c-943_1270_1357#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-pens-c-943_1270_1355#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-erasers-sharpeners-toppers-grips-c-943_1270_1586#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-pencils-pencil-crayons-c-943_1270_1356#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-stationary-sets-c-943_1270_1739#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-stencils-c-943_1270_1732#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-stickers-sticker-scenes-c-943_1270_1361#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-paper-c-943_1270_1595#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/school-supplies-stationary-products-dry-erase-products-c-943_1270_1360#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/teaching-category-classroom-pocket-charts-c-943_1666_1261#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/teaching-category-dry-erase-supplies-and-tools-c-943_1666_1260#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/teaching-category-reading-corner-supplies-c-943_1666_1596#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/teaching-subject-language-arts-and-literacy-supplies-c-943_1667_1669#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/teaching-resources-crafts-and-supplies-classroom-themes-c-943_1587#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/teaching-resources-crafts-and-supplies-classroom-crafts-c-943_1364#bc", "OPAR200 Crafts and supplies");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/200-filled-loot-boxes-c-70#bc", "OPAR200 Filled Loot Boxes");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/5000-novelty-toys-c-474#bc", "OPAR200 Filled Loot Boxes");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/200-filled-loot-boxes-100s-empty-loot-boxes-c-70_948", "OPAR200 Filled Loot Boxes");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/200-filled-loot-boxes-100s-favor-pails-and-buckets-c-70_950", "OPARBuckets and Pails");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/200-filled-loot-boxes-100s-gift-and-favor-bags-c-70_947", "OPARGift And Favor Bags");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/200-filled-loot-boxes-cool-popcorn-treat-boxes-c-70_949", "OPARPopcorn Treat Boxes");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/party-decorations-and-balloons-designer-straws-c-879_1255#bc", "OPARDesigner Straws");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/solid-colour-and-patterned-partyware-c-879_880#bc", "OPARSolid Colour and Patterned Partyware");
CategoryUrl.Add("http://openaparty.com/open-a-party-shop/index.php/party-decorations-and-balloons-cake-candles-c-879_881#bc", "OPARCandles");
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (CategoryUrl.Count() > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from category page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (var url in CategoryUrl)
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + _ProductUrl.Count.ToString();
foreach (var url in _ProductUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "openaparty.com", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='openaparty.com'");
_Prd.ProductDatabaseIntegration(Products, "openaparty.com", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for openaparty.com Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "openaparty.com", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='openaparty.com'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for openaparty.com Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for openaparty.com Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='openaparty.com'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data openaparty.com Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion openaparty.com
writer.Close();
this.Close();
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_Stop = false;
time = 0;
#region tigerdirect.ca
_IStigerdirect = true;
_ScrapeUrl = "http://tigerdirect.ca/";
try
{
_Worker1 = new IE();
_Worker2 = new IE();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10);
_Work1doc.LoadHtml(_Worker1.Html);
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//ul[@class=\"mastNav-subCats\"]/li/a");
if (_Collection != null)
{
foreach (HtmlNode node in _Collection)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
try
{
CategoryUrl.Add("http://www.tigerdirect.ca" + (att.Value.Contains("?") ? att.Value + "&recs=30" : att.Value + "?recs=30"), "TGRDRCT" + node.InnerText.Trim());
}
catch
{ }
}
}
}
try
{
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_1565_8718_8718&cm_re=Printers-_-Spot%2001-_-Laser%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_1565_84868_84868&cm_re=Printers-_-Spot%2002-_-Inkjet%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=25&name=scanners&cm_re=Printers-_-Spot%2003-_-Scanners&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/category/category_slc.asp?CatId=243&cm_re=Printers-_-Spot%2004-_-Label%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_36_84863_84863&cm_re=Printers-_-Spot%2005-_-Mobile&pagesize=30", "printer");
}
catch
{ }
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("tigerdirect.ca", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
int i = 0;
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
i++;
//if (i == 3)
// break;
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
DisplayRecordProcessdetails("We are going to read product url from sub-category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
_401index = 0;
_IsSubcat = true;
foreach (var Caturl in subCategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
}
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
MessageBox.Show(exp.Message);
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "tigerdirect.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='tigerdirect.ca'");
_Mail.SendMail("OOPS there is no any product scrapped by app for tigerdirect.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='tigerdirect.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data tigerdirect.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region 401Games
_IS401games = true;
_ScrapeUrl = "http://store.401games.ca/";
try
{
_Worker1 = new IE();
_Worker2 = new IE();
// _Worker1.Visible = false;
// _Worker2.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10000);
_Work1doc.LoadHtml(_Worker1.Html);
//HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"col left\"]");
//CategoryUrl = CommanFunction.GetCategoryUrl(_Collection, "ul", "//li/a", "http://store.401games.ca", "#st=&begin=1&nhit=40");
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"sub-menu\"]");
if (_Collection != null)
{
HtmlNodeCollection menu = _Collection[0].SelectNodes("..//ul[@class=\"submenu\"]//li//a");
foreach (HtmlNode node in menu)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
CategoryUrl.Add(att.Value, node.InnerText.Trim());
}
}
}
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/shipping/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/returns/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.cahttp://payd.moneris.com/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/privacy/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/catalog/93370C/pre-orders#st=&begin=1&nhit=40");
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("store.401games", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = "http://store.401games.ca" + PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = "http://store.401games.ca" + PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch
{
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "store.401games", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='store.401games'");
_Mail.SendMail("OOPS there is no any product scrapped by app for store.401games Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='store.401games'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data store.401games Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
private void Go_Click(object sender, EventArgs e)
{
//string ss="=202{203}202{567}";
//string[] cominations = ss.Split(new string[] { "202"}, StringSplitOptions.None);
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region poolsuppliescanada.ca
#region GetBrands
_Work1doc2.LoadHtml(_Client2.DownloadString("https://www.poolsuppliescanada.ca/brands/"));
HtmlNodeCollection brandColl = _Work1doc2.DocumentNode.SelectNodes("//div[@class=\"subcategories\"]/ul/li/a");
if (brandColl != null)
{
foreach (HtmlNode node in brandColl)
{
foreach (HtmlAttribute attr in node.Attributes)
{
if (attr.Name == "href")
{
try
{
CategoryUrl.Add(attr.Value, Removeunsuaalcharcterfromstring(StripHTML(node.InnerText)));
}
catch
{
}
}
}
}
}
#region Category
HtmlNodeCollection catColl = _Work1doc2.DocumentNode.SelectNodes("//div[@class=\"ty-menu__submenu\"]");
if (catColl != null)
{
foreach (HtmlNode node in catColl)
{
HtmlNodeCollection anchorColl = node.SelectNodes(".//a");
if (anchorColl != null)
{
foreach (HtmlNode node1 in anchorColl)
{
foreach (HtmlAttribute attr in node1.Attributes)
{
if (attr.Name == "href")
{
try
{
CategoryUrl.Add(attr.Value, "JZ HOLDINGS");
}
catch
{
}
}
}
}
}
}
}
#endregion Category
#endregion GetBrands
try
{
CategoryUrl.Add("https://www.poolsuppliescanada.ca/?subcats=Y&pcode_from_q=Y&pshort=Y&pfull=Y&pname=Y&pkeywords=Y&search_performed=Y&search_id=&q=&dispatch=products.search", "JZ HOLDINGS");
}
catch
{
}
try
{
CategoryUrl.Add("https://www.poolsuppliescanada.ca/toys/", "JZ HOLDINGS");
}
catch
{
}
_ISBuy = true;
if (CategoryUrl.Count > 0)
{
try
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from category page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
totalrecord.Visible = false;
_Bar1.Visible = false;
foreach (var caturl in CategoryUrl)
{
PagingExist = true;
for (int i = 1; i < 1000; i++)
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (!PagingExist)
break;
if (!_Work.IsBusy)
{
Url1 = caturl.Key.Contains("?") ? caturl.Key + "&page=" + i : caturl.Key + "?page=" + i;
Category1 = caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = caturl.Key.Contains("?") ? caturl.Key + "&page=" + i : caturl.Key + "?page=" + i;
Category2 = caturl.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
}
Dictionary<string, string> SubCategoryUrl1 = new Dictionary<string, string>();
foreach (var dic in SubCategoryUrl)
{
SubCategoryUrl1.Add(dic.Key, dic.Value);
}
while (SubCategoryUrl1.Count() > 0)
{
SubCategoryUrl.Clear();
foreach (var caturl in SubCategoryUrl1)
{
PagingExist = true;
for (int i = 1; i < 1000; i++)
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (!PagingExist)
break;
if (!_Work.IsBusy)
{
Url1 = caturl.Key.Contains("?") ? caturl.Key + "&page=" + i : caturl.Key + "?page=" + i;
Category1 = caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = caturl.Key.Contains("?") ? caturl.Key + "&page=" + i : caturl.Key + "?page=" + i;
Category2 = caturl.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
}
SubCategoryUrl1.Clear();
foreach (var dic in SubCategoryUrl)
{
SubCategoryUrl1.Add(dic.Key, dic.Value);
}
}
#region GetProductUrlFromDB
try
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
DataSet ds = _Db.GetDataset("select url,[Brand Name] from product prd join productstore store on store.productid=prd.productid where StoreID=19", CommandType.Text, "");
if (ds.Tables.Count > 0)
{
foreach (DataRow row in ds.Tables[0].Rows)
{
if (!_ProductUrl.Keys.Contains(row[0].ToString().ToLower()))
{
try
{
_ProductUrl.Add(row[0].ToString(), row[1].ToString());
}
catch
{ }
}
}
}
}
catch
{ }
#endregion GetProductUrlFromDB
_Bar1.Visible = true;
totalrecord.Visible = true;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + _ProductUrl.Count.ToString();
foreach (var url in _ProductUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "poolsuppliescanada.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='poolsuppliescanada.ca'");
_Prd.ProductDatabaseIntegration(Products, "poolsuppliescanada.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for poolsuppliescanada.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='poolsuppliescanada.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data poolsuppliescanada.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='poolsuppliescanada.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in getting brands for poolsuppliescanada.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion poolsuppliescanada.ca
writer.Close();
this.Close();
}
private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region bargainballoons.ca
_ScrapeUrl = "http://www.bargainballoons.ca";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for bargainballoons.ca Website";
int counterReload = 0;
bool isError = false;
_Work1doc.LoadHtml(_Client1.DownloadString(_ScrapeUrl));
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//ul[@class=\"subMenuList\"]");
if (_CollectionCatLink != null)
{
try
{
foreach (HtmlNode node in _CollectionCatLink)
{
foreach (HtmlNode node1 in node.SelectNodes(".//a"))
{
foreach (HtmlAttribute attr in node1.Attributes)
{
if (attr.Name == "href")
{
try
{
CategoryUrl.Add(_ScrapeUrl + "/" + attr.Value, "BBALCA" + node1.InnerText.Trim());
}
catch
{
}
}
}
}
}
}
catch
{ }
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (CategoryUrl.Count > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read paging from category page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (var url in CategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Category Pages :" + allCategoryUrl.Count.ToString();
foreach (var url in allCategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
Category1 = url.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
Category2 = url.Value;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "bargainballoons.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='bargainballoons.ca'");
_Prd.ProductDatabaseIntegration(Products, "bargainballoons.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for bargainballoons.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "bargainballoons.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='bargainballoons.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for bargainballoons.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for bargainballoons.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "bargainballoons.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='bargainballoons.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for bargainballoons.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for bargainballoons.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='bargainballoons.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data bargainballoons.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion bargainballoons.ca
private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region ToyStores
try
{
foreach (Supplier supp in Suppliers)
{
workingSupplier = supp;
_IsCategory = false;
_IsProduct = false;
CategoryUrl.Clear();
Producturl.Clear();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for " + supp.SupplierName + " Website";
int counterReload = 0;
bool isError = false;
do
{
try
{
counterReload++;
_Work1doc.LoadHtml(_Client1.DownloadString(supp.Url));
isError = false;
Application.DoEvents();
tim(2);
}
catch
{
isError = true;
}
} while (isError && counterReload < 25);
try
{
if (supp.StoreID == 1)
{
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//div[@id=\"ctl00_CC_ProductSearchResultListing_topPaging\"]//div//div//span[@class=\"display-total\"]");
_TotalRecords = Convert.ToInt32(_CollectionCatLink[0].InnerText.Trim());
if ((_TotalRecords % 32) == 0)
{
_Pages = Convert.ToInt32(_TotalRecords / 32);
}
else
{
_Pages = Convert.ToInt32(_TotalRecords / 32) + 1;
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from search page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
for (int Page = 1; Page <= _Pages; Page++)
{
CategoryUrl.Add("http://www.bestToyStores/Search/SearchResults.aspx?path=ca77b9b4beca91fe414314b86bb581f8en20&page=" + Page);
}
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
}
#region categoryPageUrl
foreach (string url in CategoryUrl)
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url;
_Work.RunWorkerAsync();
}
else
{
Url2 = url;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#endregion categoryPageUrl
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + Producturl.Count.ToString();
foreach (var url in Producturl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key; //"http://www.bestToyStores//en-CA/product/-/b0007063.aspx?path=57d9708c19625082a2c2820fd20a3b2cen02";// "http://www.bestToyStores/en-CA/product/traxxas-traxxas-x-maxx-brushless-electric-rc-monster-truck-blue-77076-4/10400679.aspx?path=e334459dbb1955f57c8d232171133dbben02";
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
}
catch(Exception exp) {
_Mail.SendMail("Oops Some issue Occured in scrapping data"+supp.Url+" Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='BestToyStores'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data BestToyStores Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion ToyStores
writer.Close();
this.Close();
}
private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region buy.CA
_ISBuy = true;
_ScrapeUrl = "http://www.bestbuy.ca/";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for BestBuy.ca Website";
int counterReload = 0;
bool isError = false;
do
{
try
{
counterReload++;
_Work1doc.LoadHtml(_Client1.DownloadString("http://www.bestbuy.ca/Search/SearchResults.aspx?path=ca77b9b4beca91fe414314b86bb581f8en20"));
isError = false;
Application.DoEvents();
tim(2);
}
catch
{
isError = true;
}
} while (isError && counterReload < 25);
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//div[@id=\"ctl00_CC_ProductSearchResultListing_topPaging\"]//div//div//span[@class=\"display-total\"]");
if (_CollectionCatLink != null)
{
try
{
_TotalRecords = Convert.ToInt32(_CollectionCatLink[0].InnerText.Trim());
if ((_TotalRecords % 32) == 0)
{
_Pages = Convert.ToInt32(_TotalRecords / 32);
}
else
{
_Pages = Convert.ToInt32(_TotalRecords / 32) + 1;
}
}
catch
{ }
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (_TotalRecords > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from search page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
for (int Page = 1; Page <= _Pages; Page++)
{
CategoryUrl.Add("http://www.bestbuy.ca/Search/SearchResults.aspx?path=ca77b9b4beca91fe414314b86bb581f8en20&page=" + Page);
}
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (string url in CategoryUrl)
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url;
_Work.RunWorkerAsync();
}
else
{
Url2 = url;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + Producturl.Count.ToString();
// Producturl.Add("http://www.bestbuy.ca//en-CA/product/a-million-ways-to-die-in-the-west-blu-ray-combo-2014/m2212519.aspx?path=3f7735da6b5b73d657426991c57fa2d0en02", "test");
//Producturl.Add("http://www.bestbuy.ca//en-CA/product/how-i-met-your-mother-season-2-2006/m2170295.aspx?path=b3958adf339ed4049fa09a5cb3eddb4fen02", "test");
//Producturl.Add("http://www.bestbuy.ca//en-CA/product/robotech-legacy-vol-4-robotech-masters-full-screen-1985-dvd/m2006185.aspx?path=36c223cc3cf01278529107703abfcd75en02", "test");
//Producturl.Add("http://www.bestbuy.ca//en-CA/product/burn-notice-season-1-2007/m2173839.aspx?path=1805388e7a860db435a6a89c6e620e27en02", "test");
foreach (var url in Producturl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key; //"http://www.bestbuy.ca//en-CA/product/-/b0007063.aspx?path=57d9708c19625082a2c2820fd20a3b2cen02";// "http://www.bestbuy.ca/en-CA/product/traxxas-traxxas-x-maxx-brushless-electric-rc-monster-truck-blue-77076-4/10400679.aspx?path=e334459dbb1955f57c8d232171133dbben02";
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "BestBuy.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='BestBuy.ca'");
_Prd.ProductDatabaseIntegration(Products, "BestBuy.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for BestBuy.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "BestBuy.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='BestBuy.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for BestBuy.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for BestBuy.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "BestBuy.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='BestBuy.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for BestBuy.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for BestBuy.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='BestBuy.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data BestBuy.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion buy.CA
writer.Close();
this.Close();
}