public void Teardown()
{
if (_browser != null)
{
try
{
_browser.Close();
_browser.Dispose();
}
catch (Exception)
{
}
}
try
{
var ieProcessList = Process.GetProcessesByName("iexplore");
foreach (var process in ieProcessList)
{
process.Kill();
}
}
catch (Exception e)
{
Debug.WriteLine(e.ToString());
}
_browser = null;
}
/// <summary>
/// Closes the given IE instance.
/// </summary>
/// <param name="ieInstance">The IE instance to be closed.</param>
public static void CloseIEInstance(IE ieInstance)
{
if (ieInstance != null)
{
ieInstance.Close();
}
}
public void ClickMe()
{
IE ie = new IE("http://localhost/AnthemNxt.Tests/ButtonsAndLabels.aspx");
Assert.Null(ie.Span(Find.ById("ctl00_ContentPlaceHolder_label")).Text);
ie.Button(Find.ByName("ctl00$ContentPlaceHolder$button")).Click();
Assert.NotEqual("", ie.Span(Find.ById("ctl00_ContentPlaceHolder_label")).Text);
// TODO: Check no postback occurred - how can we do this?
ie.Close();
}
public void Home_HasMvcStoreHasTitle_True()
{
var browser = new IE("http://localhost:1100/");
browser.BringToFront();
Assert.IsTrue(browser.ContainsText("ASP.NET MVC MUSIC STORE"));
browser.Close();
}
public XmlDocument Write()
{
IE.Settings.MakeNewIeInstanceVisible = false;
var browser = new IE("menu.htm".File());
var writer = new XmlWriter();
foreach (Link link in browser.Links)
{
if (link.Url.StartsWith("file:///") && !link.Url.Contains("TableOfContents.htm") &&
link.Url.EndsWith("htm"))
{
processFile(link, writer);
}
}
browser.Close();
return writer.XmlDocument;
}
public static void Descargar(string rfc, string contrasena, string carpeta, DateTime fechaDesde, DateTime fechaHasta, TipoBusqueda busqueda)
{
using (IE browser = new IE())
{
//limpiar sesion y login
browser.ClearCookies();
Thread.Sleep(1000);
//java login
browser.GoTo("https://portalcfdi.facturaelectronica.sat.gob.mx");
browser.WaitForComplete();
//entrar por contraseña
browser.GoTo("https://cfdiau.sat.gob.mx/nidp/app/login?id=SATUPCFDiCon&sid=0&option=credential&sid=0");
browser.TextField(Find.ByName("Ecom_User_ID")).AppendText(rfc);
browser.TextField(Find.ByName("Ecom_Password")).AppendText(contrasena);
browser.Button("submit").Click();
browser.WaitForComplete();
//ver si nos pudimos loggear
if (browser.ContainsText("Login failed, please try again") || browser.ContainsText("La entrada no se ha completado"))
{
browser.Close();
throw new Exception("Los datos de acceso son incorrectos para: " + rfc);
}
//seleccionar emitidas o recibidas
if (busqueda == TipoBusqueda.Emitidas)
{
browser.RadioButton("ctl00_MainContent_RdoTipoBusquedaEmisor").Click();
}
else
{
browser.RadioButton("ctl00_MainContent_RdoTipoBusquedaReceptor").Click();
}
browser.Button("ctl00_MainContent_BtnBusqueda").Click();
Log.Write("Tipo busqueda", Log.Information);
//Creating the directory if it doesn't exists
if (!System.IO.Directory.Exists(carpeta))
{
System.IO.Directory.CreateDirectory(carpeta);
}
//facturas emitidas
if (busqueda == TipoBusqueda.Emitidas)
{
browser.WaitUntilContainsText("Fecha Inicial de Emisión");
browser.RadioButton("ctl00_MainContent_RdoFechas").Click();
Thread.Sleep(1000);
//fecha desde
browser.TextField("ctl00_MainContent_CldFechaInicial2_Calendario_text").Value = fechaDesde.ToString("dd/MM/yyyy");
//hasta
browser.TextField("ctl00_MainContent_CldFechaFinal2_Calendario_text").Value = fechaHasta.ToString("dd/MM/yyyy");
Thread.Sleep(1000);
//buscar muchas veces por si marca error de lentitud la pagina del sat >(
while (true)
{
browser.Button("ctl00_MainContent_BtnBusqueda").Click();
Thread.Sleep(3000);
if (browser.ContainsText("lentitud"))
{
browser.Link("closeBtn").Click();
}
else
{
break;
}
}
DescargarFacturasListadas(browser, carpeta);
}
else
{
DateTime mesActual = fechaDesde;
bool primeraVez = true;
while (mesActual < fechaHasta)
{
browser.WaitUntilContainsText("Fecha de Emisión");
browser.RadioButton("ctl00_MainContent_RdoFechas").Click();
Thread.Sleep(1000);
//seleccionar año adecuado
browser.SelectList("DdlAnio").SelectByValue(mesActual.Year.ToString());
//seleccionar mes adecuado
browser.SelectList("ctl00_MainContent_CldFecha_DdlMes").SelectByValue(mesActual.Month.ToString());
if (mesActual.Day < 10 && primeraVez)
{
//seleccionar dia adecuado
//click en buscar por que si no no jala
//buscar muchas veces por si marca error de lentitud la pagina del sat >(
while (true)
{
browser.Button("ctl00_MainContent_BtnBusqueda").Click();
Thread.Sleep(3000);
if (browser.ContainsText("lentitud"))
{
browser.Link("closeBtn").Click();
}
else
{
break;
}
}
Thread.Sleep(1000);
primeraVez = false;
}
browser.SelectList("ctl00_MainContent_CldFecha_DdlDia").SelectByValue(mesActual.Day.ToString("00"));
Thread.Sleep(1000);
//buscar muchas veces por si marca error de lentitud la pagina del sat >(
while (true)
{
browser.Button("ctl00_MainContent_BtnBusqueda").Click();
Thread.Sleep(3000);
if (browser.ContainsText("lentitud"))
{
browser.Link("closeBtn").Click();
}
else
{
break;
}
}
DescargarFacturasListadas(browser, carpeta);
//pasar al siguiente mes
mesActual = mesActual.AddDays(1);
}
}
browser.Link("ctl00_LnkBtnCierraSesion").Click();
Thread.Sleep(2000);
browser.Close();
}
}
private void Go_Click(object sender, EventArgs e)
{
_IsProduct = false;
_percent.Visible = false;
_Bar1.Value = 0;
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
_Worker1 = new IE();
_Worker2 = new IE();
#region Factory.ca
_ScrapeUrl = "http://www.factorydirect.ca/SearchResults.aspx";
try
{
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category Link for factorydirect.ca Website";
int counterReload = 0;
bool isError = false;
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10000);
_Work1doc.LoadHtml(_Worker1.Html);
HtmlNodeCollection _CollectionCatLink = _Work1doc.DocumentNode.SelectNodes("//b[@class=\"nxt-result-total\"]");
if (_CollectionCatLink != null)
{
try
{
_TotalRecords = Convert.ToInt32(_CollectionCatLink[0].InnerText.Trim());
if ((_TotalRecords % 36) == 0)
{
_Pages = Convert.ToInt32(_TotalRecords / 36);
}
else
{
_Pages = Convert.ToInt32(_TotalRecords / 36) + 1;
}
}
catch
{ }
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (_TotalRecords > 0)
{
gridindex = 0;
_Bar1.Value = 0;
_percent.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read products from search page.";
_Stop = false;
time = 0;
_IsCategory = true;
tim(3);
totalrecord.Visible = true;
for (int Page = 1; Page <= _Pages; Page++)
{
CategoryUrl.Add("http://www.factorydirect.ca/SearchResults.aspx#/?search_return=all&res_per_page=36&page=" + Page);
}
totalrecord.Text = "Total No Pages :" + CategoryUrl.Count.ToString();
foreach (string url in CategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url;
_Work.RunWorkerAsync();
}
else
{
Url2 = url;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_lblerror.Visible = true;
_lblerror.Text = "We are going to read product info.";
_IsCategory = false;
_IsProduct = true;
gridindex = 0;
totalrecord.Text = "Total No Products :" + Producturl.Count.ToString();
foreach (var url in Producturl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = url.Key;
_Work.RunWorkerAsync();
}
else
{
Url2 = url.Key;
_Work1.RunWorkerAsync();
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
#region InsertScrappedProductInDatabase
if (Products.Count() > 0)
{
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Mail.SendMail("OOPS there is no any product scrapped by app for factorydirect.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#endregion InsertScrappedProductInDatabase
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Prd.ProductDatabaseIntegration(Products, "factorydirect.ca", 1);
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Text = "Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website";
/****************Email****************/
_Mail.SendMail("Oops there is change in html code on client side. You need to contact with developer in order to check this issue for factorydirect.ca Website as soon as possible because noscrapping of given store is stopped working." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
/*******************End********/
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='factorydirect.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data factorydirect.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
# endregion Factory.CA
writer.Close();
try { _Worker1.Close(); _Worker2.Close(); }
catch
{
}
this.Close();
}
public void SignupPage()
{
clsDBQueryManager SetDb = new clsDBQueryManager();
DataSet ds = new DataSet();
//bool UsePublicProxy = false;
int counter_Username = 0;
foreach (string EmailPassword in LstAccountcreationEmailPassword)
{
StartAgain:
try
{
GlobusHttpHelper httpHelper = new GlobusHttpHelper();
string[] array = Regex.Split(EmailPassword, ":");
string email = array[0];
string password = array[1];
string username = string.Empty;
string ProxyAdress = string.Empty;
string ProxyPort = string.Empty;
string Proxy = string.Empty;
AddToProxyAccountCreationLog("Clearing Cookies");
ClearCookies();
if (chkboxUsePublicProxies.Checked)
{
try
{
ds = SetDb.SelectPublicProxyData();
if (ds.Tables[0].Rows.Count != 0)
{
int index = RandomNumberGenerator.GenerateRandom(0, ds.Tables[0].Rows.Count);
DataRow dr = ds.Tables[0].Rows[countForInstance];
Proxy = dr.ItemArray[0].ToString() + ":" + dr.ItemArray[1].ToString() + ":" + dr.ItemArray[2].ToString() + ":" + dr.ItemArray[3].ToString();
string[] ProxyList = Regex.Split(Proxy, ":");
ProxyAdress = ProxyList[0];
ProxyPort = ProxyList[1];
AddToProxyAccountCreationLog("Using Proxy Address : - " + ProxyAdress + ":" + ProxyPort);
}
else
{
AddToProxyAccountCreationLog("Please Uplaod Public Proxies");
break;
}
}
catch (Exception ex)
{
}
}
AddToProxyAccountCreationLog("Setting Proxy");
Thread.Sleep(2000);
SetProxy(ProxyAdress, ProxyPort);
Thread.Sleep(2000);
IE explorer = new IE();
Thread.Sleep(1000);
if (LstAccountcreationUsername.Count > 0)
{
if (counter_Username < LstAccountcreationUsername.Count)
{
username = LstAccountcreationUsername[counter_Username];
counter_Username++;
}
else
{
AddToProxyAccountCreationLog("*********** /n All Usernames have been taken /n ***********");
break;
}
}
else
{
AddToProxyAccountCreationLog("Please Upload Usernames To Create Twitter Accounts");
break;
}
string name = LstAccountCreationName[countForInstance];
string capcthaUrl = string.Empty;
try
{
explorer.GoTo("https://twitter.com/signup");
string PageSource = explorer.Html.ToString();
if (PageSource.Contains("Sign out"))
{
explorer.Link(Find.ById("signout-button")).Click();
Thread.Sleep(2000);
explorer.GoTo("https://twitter.com/signup");
Thread.Sleep(2000);
}
}
catch (Exception ex)
{
AddToProxyAccountCreationLog("Taking too Long To respond.Page Not Loaded Fully");
break;
}
try
{
foreach (TextField item in explorer.TextFields)
{
string Html = item.OuterHtml.ToString();
if (item.Name == "user[name]")
{
AddToProxyAccountCreationLog("Name :" + name);
item.TypeText(name);
Thread.Sleep(1000);
break;
}
}
foreach (TextField item in explorer.TextFields)
{
if (item.Name == "user[email]")
{
AddToProxyAccountCreationLog("Email :" + email);
item.TypeText(email);
Thread.Sleep(1000);
break;
}
}
string EmailCheck = explorer.Html.ToString();
Thread.Sleep(8000);
if (EmailCheck.Contains("taken error active") || EmailCheck.Contains("invalid error active") || EmailCheck.Contains("blank error active"))
{
AddToProxyAccountCreationLog("Error In Email - " + email);
GlobusFileHelper.AppendStringToTextfileNewLine(email + ":" + password + ":" + ProxyAdress + ":" + ProxyPort, Globals.path_DesktopFolder + "\\TwtDominator\\BrowserAccountUnsuccessfull.txt");
}
else
{
foreach (TextField item in explorer.TextFields)
{
string Html = item.OuterHtml.ToString();
if (item.Name == "user[user_password]")
{
AddToProxyAccountCreationLog("Password :"******"invalid error active") || !CheckPassword.Contains("blank error active") || !CheckPassword.Contains("blank error active") || CheckPassword.Contains("weak error active"))
{
AddToProxyAccountCreationLog("Password Not Secure creating Other");
password = password + RandomStringGenerator.RandomString(5);
if (password.Count() > 15)
{
password = password.Remove(13); //Removes the extra characters
}
foreach (TextField item in explorer.TextFields)
{
string Html = item.OuterHtml.ToString();
if (item.Name == "user[user_password]")
{
AddToProxyAccountCreationLog("Password :"******"Username : "******""))
//{
if (item.Name == "user[screen_name]")
{
item.TypeText(username);
Thread.Sleep(1000);
break;
}
//}
}
string Usernamecheck = explorer.Html.ToString();
if (!Usernamecheck.Contains("taken error active") || !Usernamecheck.Contains("invalid error active") || !Usernamecheck.Contains("blank error active"))
{
if (username.Count() > 12)
{
username = username.Remove(8); //Removes the extra characters
}
username = username + RandomStringGenerator.RandomString(5);
if (username.Count() > 15)
{
username = username.Remove(13); //Removes the extra characters
}
foreach (TextField item in explorer.TextFields)
{
if (item.Name == "user[screen_name]")
{
item.TypeText(username);
Thread.Sleep(1000);
break;
}
}
}
Thread.Sleep(2000);
string PageSource = explorer.Html.ToString();
if (PageSource.Contains("sign-up-box"))
{
AddToProxyAccountCreationLog("Creating Twitter Account With Email : " + email);
explorer.Button(Find.ByValue("Create my account")).Click();
Thread.Sleep(2000);
}
string PageSourceSignup = explorer.Html.ToString();
if (PageSource.Contains("signout-button") && PageSource.Contains("js-signout-button"))
{
AddToProxyAccountCreationLog("Creating Twitter Account With Email : " + email);
GlobusFileHelper.AppendStringToTextfileNewLine(email + ":" + password, Globals.Path_BrowserCreatedAccounts);
string GetSignout = explorer.Html.ToString();
if (GetSignout.Contains("Sign out"))
{
explorer.Link(Find.ById("signout-button")).Click();
Thread.Sleep(2000);
countForInstance++;
}
}
else if (PageSource.Contains("Are you human?"))
{
AddToProxyAccountCreationLog("Checking For Human??");
Thread.Sleep(3000);
try
{
foreach (Div Item in explorer.Divs)
{
string Url = Item.OuterHtml.ToString();
if (Item.Id == "recaptcha_image")
{
int startIndex = Url.IndexOf("src=\"");
string start = Url.Substring(startIndex).Replace("src=\"", "");
int endIndex = start.IndexOf("\"");
string end = start.Substring(0, endIndex);
capcthaUrl = end;
break;
}
}
int i = 0;
WebClient webclient = new WebClient();
StartWebClient:
Thread.Sleep(2000);
try
{
byte[] args = webclient.DownloadData(capcthaUrl);
string[] arr1 = new string[] { BaseLib.Globals.DBCUsername, BaseLib.Globals.DBCPassword, "" };
string CapcthaString = DecodeDBC(arr1, args);
foreach (TextField item in explorer.TextFields)
{
string Html = item.OuterHtml.ToString();
if (item.Id == "recaptcha_response_field")
{
AddToProxyAccountCreationLog("Adding Capctha Response");
item.TypeText(CapcthaString);
break;
}
}
explorer.Button(Find.ByValue("Create my account")).Click();
}
catch (Exception ex)
{
i++;
if (ex.Message.Contains("An exception occurred during a WebClient request."))
{
AddToProxyAccountCreationLog("Error In Capctha Download Trying Again - " + i);
if (i <= 3)
{
goto StartWebClient;
}
else
{
GlobusFileHelper.AppendStringToTextfileNewLine(email + ":" + password + ":" + ProxyAdress + ":" + ProxyPort, Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\TwtDominator\\UnsuccessfullBrowserAcounts.txt");
}
}
}
}
catch (Exception ex)
{
}
string ConfirmId = explorer.Html.ToString();
AddToProxyAccountCreationLog("Confirming created Account");
if (!ConfirmId.Contains("Sign out"))
{
AddToProxyAccountCreationLog("Account Not Created : " + email + ":" + password);
GlobusFileHelper.AppendStringToTextfileNewLine(email + ":" + password + ":" + ProxyAdress + ":" + ProxyPort, Environment.GetFolderPath(Environment.SpecialFolder.Desktop) + "\\TwtDominator\\UnsuccessfullBrowserAcounts.txt");
}
else if (ConfirmId.Contains("Sign out"))
{
GlobusFileHelper.AppendStringToTextfileNewLine(email + ":" + password + ":" + ProxyAdress + ":" + ProxyPort, Globals.Path_BrowserCreatedAccounts);
explorer.GoTo("https://twitter.com/");
explorer.Link(Find.ById("signout-button")).Click();
Thread.Sleep(1000);
}
}
}
}
catch (Exception ex)
{
if (ex.Message == "The remote server returned an error: (504) Gateway Timeout.")
{
AddToProxyAccountCreationLog("Remote Server Returned Error - Time out");
}
explorer.Close();
AddToProxyAccountCreationLog("Closing Explorer Due To Error");
goto StartAgain;
}
explorer.Close();
countForInstance++;
}
catch (Exception ex)
{
Console.WriteLine("Error :: - " + DateTime.Now.ToString() + " || Error - " + ex.Message);
if (ex.Message.Contains("Creating an instance of the COM component with"))
{
goto StartAgain;
}
}
}
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_Stop = false;
time = 0;
#region tigerdirect.ca
_IStigerdirect = true;
_ScrapeUrl = "http://tigerdirect.ca/";
try
{
_Worker1 = new IE();
_Worker2 = new IE();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10);
_Work1doc.LoadHtml(_Worker1.Html);
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//ul[@class=\"mastNav-subCats\"]/li/a");
if (_Collection != null)
{
foreach (HtmlNode node in _Collection)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
try
{
CategoryUrl.Add("http://www.tigerdirect.ca" + (att.Value.Contains("?") ? att.Value + "&recs=30" : att.Value + "?recs=30"), "TGRDRCT" + node.InnerText.Trim());
}
catch
{ }
}
}
}
try
{
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_1565_8718_8718&cm_re=Printers-_-Spot%2001-_-Laser%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_1565_84868_84868&cm_re=Printers-_-Spot%2002-_-Inkjet%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=25&name=scanners&cm_re=Printers-_-Spot%2003-_-Scanners&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/category/category_slc.asp?CatId=243&cm_re=Printers-_-Spot%2004-_-Label%20Printers&pagesize=30", "printer");
CategoryUrl.Add("http://www.tigerdirect.ca/applications/Category/guidedSearch.asp?CatId=21&sel=Detail%3B358_36_84863_84863&cm_re=Printers-_-Spot%2005-_-Mobile&pagesize=30", "printer");
}
catch
{ }
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("tigerdirect.ca", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
int i = 0;
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
i++;
//if (i == 3)
// break;
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
DisplayRecordProcessdetails("We are going to read product url from sub-category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
_401index = 0;
_IsSubcat = true;
foreach (var Caturl in subCategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
}
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch (Exception exp)
{
MessageBox.Show(exp.Message);
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "tigerdirect.ca", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='tigerdirect.ca'");
_Mail.SendMail("OOPS there is no any product scrapped by app for tigerdirect.ca Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='tigerdirect.ca'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data tigerdirect.ca Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
public static string Goto(String text, IE ie)
{
int i = 0;
while (i < Loop)
{
i++;
try
{
ie.GoTo(text);
ie.WaitForComplete();
//ie.WaitUntilContainsText("message");
return string.Empty;
}
catch (Exception ex)
{
if (i == Loop)
{
return ex.Message;
}
ie.Close();
Thread.Sleep(60000);
ie.Reopen();
}
}
return string.Empty;
}
public static string Goto(string Url, IE ie)
{
int i = 0;
while (i < Loop)
{
i++;
try
{
ie.GoTo(Url);
ie.WaitForComplete();
return string.Empty;
}
catch (Exception ex)
{
if (i == Loop)
{
return ex.Message;
}
ie.Close();
Thread.Sleep(60000);
ie.Reopen();
}
}
return string.Empty;
}
private void processFile(Link link, XmlWriter writer)
{
string path = link.Url;
string title = link.InnerHtml;
writer.WritePage(title, path);
Debug.WriteLine(title + " = " + path);
var browser = new IE(path, false);
foreach (Element header in browser.Elements)
{
if (header.TagName == "H2")
{
processHeader(header, writer, path);
}
if (header.TagName == "H4")
{
processSubHeader(header, writer, path);
}
}
browser.Close();
}
private IEnumerable<string> DeepHarvestShoeNode(HtmlNode node, ProductData product)
{
//Debugger.Launch();
var productLink = BaseAddress + node.SelectNodes("p[@class='view_buy']/a").First().Attributes["href"].Value;
var browser = new IE(productLink, false);
browser.GoTo(productLink);
browser.ShowWindow(NativeMethods.WindowShowStyle.Hide);
var mainProductHtml = new HtmlDocument();
var doc = HtmlNode.CreateNode("");
IEnumerable<string> images = new string[0];
try
{
mainProductHtml.LoadHtml(GetHtmlString(productLink));
// //div[@id="productright"]/div[@class=product_info]/p
doc = mainProductHtml.DocumentNode;
images = GetVariantImages(browser);
browser.Close();
product.Handle = new Uri(productLink).AbsolutePath.Replace("/products/", string.Empty).Replace("/", "-");
product.Body = "\"" + doc.SelectNodes("//div[@id='productright']/div[@class='product_info']/p").First().InnerText.Replace("\"", "'") + "\"";
product.Type = "Mens Shoes"; DiscernType(product.Body, product.Title);
}
catch (Exception e)
{
Console.WriteLine("Exception thrown trying to parse: {0}", productLink);
}
HtmlNodeCollection colours = null;
HtmlNodeCollection sizes = null;
var productForm = doc.SelectNodes("//div[@id='product_form']").First();
product.Option1Name = "Title";
product.Option1Value = "Title";
if (productForm.InnerHtml.Contains("Colour"))
{
colours = doc.SelectNodes("//div[@id='colours_js_box']").First().SelectNodes("//a[contains(@class,'product_colour')]");
product.Option1Name = "Colour";
product.Option1Value = colours.First().InnerText;
}
if (productForm.InnerHtml.Contains("Size"))
{
sizes = doc.SelectNodes("//div[@id='sizes_js_box']").First().SelectNodes("//a[contains(@class,'product_sizes')]");
if (product.Option1Name == "Title")
{
product.Option1Name = "Colour";
product.Option1Value = sizes.First().InnerText;
}
else
{
product.Option2Name = "Size";
product.Option2Value = sizes.First().InnerText;
}
}
product.Sku = productLink;
product.Taxable = "FALSE";
product.RequiresShipping = "TRUE";
product.FulfillmentService = "manual";
product.InventoryPolicy = "continue";
product.Vendor = "Henry James";
product.InventoryQuantity = "0";
product.Tags = "Mens Shoes";
product.Sizes = sizes;
product.Colours = colours;
return images;
}
private IEnumerable<string> GetVariantImages(Browser browser)
{
var variantImages = new List<string>();
var links = browser.Div("colours_js_box").Divs.First().Links.Select(x => x.Title);
foreach (var link in links)
{
Browser b = new IE(browser.Url);
b.ShowWindow(NativeMethods.WindowShowStyle.Hide);
b.Link(Find.ByTitle(link)).Click();
var d = new HtmlDocument();
d.LoadHtml(GetHtmlString(b.Url));
b.Close();
var node = d.DocumentNode.SelectNodes("//img[@id='productimage']").First();
variantImages.Add(BaseAddress + node.Attributes["src"].Value);
}
return variantImages;
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
Url.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region 401Games
_IS401games = true;
_ScrapeUrl = "http://store.401games.ca/";
try
{
_Worker1 = new IE();
_Worker2 = new IE();
// _Worker1.Visible = false;
// _Worker2.Visible = false;
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Worker1.GoTo(_ScrapeUrl);
_Worker1.WaitForComplete();
System.Threading.Thread.Sleep(10000);
_Work1doc.LoadHtml(_Worker1.Html);
//HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"col left\"]");
//CategoryUrl = CommanFunction.GetCategoryUrl(_Collection, "ul", "//li/a", "http://store.401games.ca", "#st=&begin=1&nhit=40");
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"sub-menu\"]");
if (_Collection != null)
{
HtmlNodeCollection menu = _Collection[0].SelectNodes("..//ul[@class=\"submenu\"]//li//a");
foreach (HtmlNode node in menu)
{
foreach (HtmlAttribute att in node.Attributes)
{
if (att.Name == "href")
CategoryUrl.Add(att.Value, node.InnerText.Trim());
}
}
}
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/shipping/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/returns/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.cahttp://payd.moneris.com/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/privacy/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/catalog/93370C/pre-orders#st=&begin=1&nhit=40");
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
FileInfo _Info = new FileInfo(Application.StartupPath + "/Files/Url.txt");
int Days = 14;
try
{
Days = Convert.ToInt32(Config.GetAppConfigValue("store.401games", "FrequencyOfCategoryScrapping"));
}
catch
{
}
if (_Info.CreationTime < DateTime.Now.AddDays(-Days))
_IsCategorypaging = true;
else
_IsCategorypaging = false;
}
else
_IsCategorypaging = true;
if (_IsCategorypaging)
{
foreach (var Caturl in CategoryUrl)
{
try
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl.Key;
BrandName1 = Caturl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl.Key;
BrandName2 = Caturl.Value;
_Work1.RunWorkerAsync();
}
}
catch { }
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
System.Threading.Thread.Sleep(1000);
_Bar1.Value = 0;
_401index = 0;
#region Code to get and write urls from File
if (File.Exists(Application.StartupPath + "/Files/Url.txt"))
{
using (StreamReader Reader = new StreamReader(Application.StartupPath + "/Files/Url.txt"))
{
string line = "";
while ((line = Reader.ReadLine()) != null)
{
try
{
Url.Add(line.Split(new[] { "@#$#" }, StringSplitOptions.None)[0], line.Split(new[] { "@#$#" }, StringSplitOptions.None)[1]);
}
catch
{
}
}
}
}
foreach (var url in _ProductUrlthread1)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
foreach (var url in _ProductUrlthread2)
{
try
{
if (!Url.Keys.Contains(url.Key.ToLower()))
Url.Add(url.Key.ToLower(), url.Value);
}
catch
{
}
}
// Code to write in file
if (_IsCategorypaging)
{
using (StreamWriter writer = new StreamWriter(Application.StartupPath + "/Files/Url.txt"))
{
foreach (var PrdUrl in Url)
{
writer.WriteLine(PrdUrl.Key + "@#$#" + PrdUrl.Value);
}
}
}
#endregion Code to get and write urls from File
_IsCategorypaging = false;
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + Url.Count());
_IsProduct = true;
foreach (var PrdUrl in Url)
{
try
{
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = "http://store.401games.ca" + PrdUrl.Key;
BrandName1 = PrdUrl.Value;
_Work.RunWorkerAsync();
}
else
{
Url2 = "http://store.401games.ca" + PrdUrl.Key;
BrandName2 = PrdUrl.Value;
_Work1.RunWorkerAsync();
}
}
catch
{
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
if (Products.Count() > 0)
{
System.Threading.Thread.Sleep(1000);
_lblerror.Visible = true;
BusinessLayer.ProductMerge _Prd = new BusinessLayer.ProductMerge();
_Prd.ProductDatabaseIntegration(Products, "store.401games", 1);
}
else
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='store.401games'");
_Mail.SendMail("OOPS there is no any product scrapped by app for store.401games Website." + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
}
catch
{
BusinessLayer.DB _Db = new BusinessLayer.DB();
_Db.ExecuteCommand("update Schduler set LastProcessedStatus=0 where StoreName='store.401games'");
_lblerror.Visible = true;
_Mail.SendMail("Oops Some issue Occured in scrapping data store.401games Website" + DateTime.Now.ToString(), "Urgent issue in Scrapper.", false, false, 1);
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
_writer.Close();
this.Close();
}
public void Process()
{
_IsProduct = false;
_Name.Clear();
CategoryUrl.Clear();
SubCategoryUrl.Clear();
_ProductUrlthread1.Clear();
_ProductUrlthread1.Clear();
_ProductUrl.Clear();
_percent.Visible = false;
_Bar1.Value = 0;
_Url.Clear();
_Tbale.Rows.Clear();
_Tbale.Columns.Clear();
dataGridView1.Rows.Clear();
DataColumn _Dc = new DataColumn();
_Dc.ColumnName = "Rowid";
_Dc.AutoIncrement = true;
_Dc.DataType = typeof(int);
_Dc.AutoIncrementSeed = 1;
_Dc.AutoIncrementStep = 1;
_Tbale.Columns.Add(_Dc);
_Tbale.Columns.Add("SKU", typeof(string));
_Tbale.Columns.Add("Product Name", typeof(string));
_Tbale.Columns.Add("Product Description", typeof(string));
_Tbale.Columns.Add("Bullet Points", typeof(string));
_Tbale.Columns.Add("Manufacturer", typeof(string));
_Tbale.Columns.Add("Brand Name", typeof(string));
_Tbale.Columns.Add("Price", typeof(string));
_Tbale.Columns.Add("Currency", typeof(string));
_Tbale.Columns.Add("In Stock", typeof(string));
_Tbale.Columns.Add("Image URL", typeof(string));
_Tbale.Columns.Add("URL", typeof(string));
_lblerror.Visible = false;
_Pages = 0;
_TotalRecords = 0;
gridindex = 0;
_IsCategory = true;
_Stop = false;
time = 0;
#region 402Games
_IS401games = true;
_ScrapeUrl = "http://store.401games.ca/";
try
{
_Worker1 = new IE();
// _Worker2 = new IE();
_lblerror.Visible = true;
_lblerror.Text = "We are going to read category url of " + chkstorelist.Items[0].ToString() + " Website";
_Work1doc.LoadHtml(_Client1.DownloadString(_ScrapeUrl));
HtmlNodeCollection _Collection = _Work1doc.DocumentNode.SelectNodes("//div[@class=\"col left\"]");
CategoryUrl = CommanFunction.GetCategoryUrl(_Collection, "ul", "//li/a", "http://store.401games.ca", "#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/shipping/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/returns/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/terms/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/contact_us/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/product/sitemap/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.cahttp://payd.moneris.com/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/service/privacy/#st=&begin=1&nhit=40");
CategoryUrl.Remove("http://store.401games.ca/catalog/93370C/pre-orders#st=&begin=1&nhit=40");
if (CategoryUrl.Count() > 0)
{
DisplayRecordProcessdetails("We are going to read product url from category pages for " + chkstorelist.Items[0].ToString() + " Website", "Total Category :" + CategoryUrl.Count());
_IsCategorypaging = true;
foreach (string Caturl in CategoryUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = Caturl;
_Work.RunWorkerAsync();
}
else
{
Url2 = Caturl;
_Work1.RunWorkerAsync();
}
break;
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
_Bar1.Value = 0;
_401index = 0;
_IsCategorypaging = false;
_ProductUrl = _ProductUrlthread1.Concat(_ProductUrlthread2).ToList();
DisplayRecordProcessdetails("We are going to read Product information for " + chkstorelist.Items[0].ToString() + " Website", "Total products :" + _ProductUrl.Count());
_IsProduct = true;
foreach (string PrdUrl in _ProductUrl)
{
while (_Work.IsBusy && _Work1.IsBusy)
{
Application.DoEvents();
}
while (_Stop)
{
Application.DoEvents();
}
if (!_Work.IsBusy)
{
Url1 = "http://store.401games.ca"+PrdUrl;
_Work.RunWorkerAsync();
}
else
{
Url2 ="http://store.401games.ca"+ PrdUrl;
_Work1.RunWorkerAsync();
}
}
}
while (_Work.IsBusy || _Work1.IsBusy)
{
Application.DoEvents();
}
MessageBox.Show("Process Completed.");
}
catch
{
}
#region closeIEinstance
try
{
_Worker1.Close();
_Worker2.Close();
}
catch
{
}
#endregion closeIEinstance
#endregion
}