public IWebElement findElement(TestStep ts)
{
IWebElement we = null;
String xpath = ts.Xpath();
if (ts.userXPath != null)
{
xpath = ts.userXPath;
}
if (ts.index > 0)
{
we = ch.FindElementsByXPath(xpath)[ts.index - 1];
}
else
{
we = ch.FindElementByXPath(xpath);
}
int y = we.Location.Y;
String js = String.Format("window.scroll(0, {0})", y / 2);
((IJavaScriptExecutor)ch).ExecuteScript(js);
return(we);
}
public void IfEmailAddressIsNotValid_ShowValidationMessage()
{
using (var driver = new ChromeDriver())
{
driver.Navigate().GoToUrl("http://*****:*****@id='loginForm']/form/div[1]/div/span/span");
Assert.AreEqual(0, loginErrorMessage1.Count);
driver.FindElementByXPath("//*[@id='loginForm']/form/div[4]/div/input").Click();
var loginErrorMessage = driver.FindElementByXPath("//*[@id='loginForm']/form/div[1]/div/span/span");
Assert.True(loginErrorMessage.Displayed);
}
}
// well this is my main function I'll call it on Execute button press
private void monster_gulf_grabber_function(int totalPages_Int, String default_Url)
{
/* initializing chrome driver */
var chrome_Driver = ChromeDriverService.CreateDefaultService();
//no need to show disturbing command window so set it hidden
chrome_Driver.HideCommandPromptWindow = true;
//try
//{
var job_region = job_region_selected;
var job_industry = job_type_selected;
//var job_ref_code = "";
var totalPages = 0;
/* create a csv file for output*/
try
{
File.WriteAllText(Path.Combine(path_Desktop, "MonsterGulf_Scrapped_DataSet_" + job_region_selected + "_" + job_type_selected + ".csv"), "Job Title, Company, Description, Location, Experience, Keyskills, Region, Industry, Date\r\n");
}
catch (Exception ex)
{
//incase of file is open or readonly or no permission show message
//result_textBox.AppendText(ex.StackTrace);
MessageBox.Show("Cannot write to file !");
}
//initialize chrome driver
var main_page_chrome_Driver_Obj = new ChromeDriver(chrome_Driver, new ChromeOptions());
do
{
var job_title = "";
var job_company = "";
var job_desc = "";
var job_location = "";
var job_experience = "";
var job_keyskils = "";
var job_date = "";
main_page_chrome_Driver_Obj.Manage().Timeouts().ImplicitlyWait(TimeSpan.FromSeconds(10));
main_page_chrome_Driver_Obj.Navigate().GoToUrl(default_Url + "/searchresult.html?rfr=;loc=" + region_Selection + ";jbc=" + job_type_Selection + ";day=60;srt=pst;ref=http:%2F%2Fjobsearch%2Emonstergulf%2Ecom%2Fsearch%2Ehtml;show_omit=1;res_cnt=" + resultsPerPage_Int + ";n=" + totalPages_Int);
//main_page_chrome_Driver_Obj.Navigate().GoToUrl("http://jobsearch.monstergulf.com/searchresult.html?rfr=refine;day=60;srt=pst;ref=http:%2F%2Fjobsearch.monstergulf.com%2Fsearch.html;show_omit=1;res_cnt=40;n=" + totalPages_Int);
//http://jobsearch.monstergulf.com/searchresult.html?rfr=refine;day=60;srt=pst;ref=http:%2F%2Fjobsearch.monstergulf.com%2Fsearch.html;show_omit=1;res_cnt=40;n=2
//find main pages count.
//select result part of DOM
var main_page_dom = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@class='ns_sresultmain']/div[2]");
//find main pages count.
if (totalPages_Int > 1) { }
else
{
var main_pages_count = main_page_dom[0].FindElements(By.XPath("./div[1]/div/div[2]/div[3]/div/div/div/div/div[@class='liDiv']"));
totalPages = main_pages_count.Count();
}
//find jobs URLs in main page, it contains 40 jobs urls .
var main_page_jobs = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@class='ns_job_wrapper']"); //it gives me 40 count i.e 40 jobs in single main page
//
//pages_num_textBox.AppendText(" " + main_page_jobs.Count().ToString());
if (main_page_jobs.Count() == 40) { }
else
{
DateTime dt = DateTime.Now + TimeSpan.FromSeconds(7);
do
{
main_page_jobs = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@class='ns_job_wrapper']"); //it gives me 40 count i.e 40 jobs in single main page
//MessageBox.Show(main_page_jobs.Count().ToString());
} while (DateTime.Now < dt);
}
for (int mainPage_jobIndex = 0; mainPage_jobIndex < main_page_jobs.Count(); mainPage_jobIndex++)
{
job_title = main_page_jobs[mainPage_jobIndex].FindElement(By.XPath("./div[2]/a/div[@class='ns_jobtitle ns_lt']")).Text;
job_company = main_page_jobs[mainPage_jobIndex].FindElement(By.XPath("./div[2]/a/div[@class='ns_cmpname ns_lt']/h2/strong")).Text;
job_desc = main_page_jobs[mainPage_jobIndex].FindElement(By.XPath("./div[2]/a/div[@class='ns_jobdesc ns_lt']")).Text;
job_desc = job_desc.Replace(System.Environment.NewLine, " ");
string job_location_and_experience = main_page_jobs[mainPage_jobIndex].FindElement(By.XPath("./div[2]/a/div[@class='ns_joblocation ns_lt']")).Text;
string s = job_location_and_experience;
string[] values = s.Split(',');
job_location = "";
for (int array_item = 0; array_item < values.Length - 1; array_item++)
{
job_location += values[array_item].Trim();
}
job_experience = values.Last().Trim();// the only last index of array is experience remaining index are locations
if (main_page_jobs[mainPage_jobIndex].FindElements(By.XPath("./div[2]/a/div[@class='ns_jobkeyskills ns_lt']")).Count() > 0)
{
job_keyskils = main_page_jobs[mainPage_jobIndex].FindElement(By.XPath("./div[2]/a/div[@class='ns_jobkeyskills ns_lt']")).GetAttribute("title");
}
//from 16th Apr 2015 => 16/4/2015
var job_date_before_conversion = main_page_jobs[mainPage_jobIndex].FindElement(By.XPath("./div[2]/a/div[@class='ns_jobdate ns_rt']")).Text;
job_date = timeStampCnversion(job_date_before_conversion);
/* to output the result in textbox*/
//Job Title, Company, Description, Location, Experience, Keyskills, Region, Industry, Date\r\n
result_textBox.AppendText("\nJob Title: " + job_title.ToString().Trim() + "\nCompany: " + job_company.ToString().Trim() + "\nDescription: " + job_desc.ToString().Trim() + "\nLocation: " + job_location.ToString().Trim() + "\nExperience: " + job_experience.ToString().Trim() + "\nKeyskills: " + job_keyskils.ToString().Trim() + "\nRegion: " + job_region.ToString().Trim() + "\nIndustry: " + job_industry.ToString().Trim() + "\nDate: " + job_date.ToString().Trim() + "\r.....................................................................................................\n");
/* add data to above created csv file */
File.AppendAllText(Path.Combine(path_Desktop, "MonsterGulf_Scrapped_DataSet_" + job_region_selected + "_" + job_type_selected + ".csv"), job_title.ToString().Trim().Replace(",", "") + "," + job_company.ToString().Trim().Replace(",", "") + "," + job_desc.ToString().Trim().Replace(",", "") + "," + job_location.ToString().Trim().Replace(",", "") + "," + job_experience.ToString().Trim().Replace(",", "") + "," + job_keyskils.ToString().Trim().Replace(",", "") + "," + job_region.ToString().Trim().Replace(",", "") + "," + job_industry.ToString().Trim().Replace(",", "") + "," + job_date.ToString().Trim().Replace(",", "") + "\r\n");
}
totalPages_Int++;
} while (totalPages_Int <= totalPages + 1);
//close chrome driver/ chrome browser window
main_page_chrome_Driver_Obj.Close();
//relase all recources associated with selenium chrome driver
//}
//catch (Exception ex)
//{
// result_textBox.AppendText(ex.StackTrace);
// MessageBox.Show("Google Chrome not found! Click OK to install it");
// System.Diagnostics.Process.Start("https://www.google.com/chrome/#eula");
// //https://www.google.com/chrome/browser/thankyou.html?standalone=1&system=true&platform=win&installdataindex=defaultbrowser
// //this.Close();
//}
}
public void JobsInDubai_Scraper(string default_Url)
{
var job_title = "";
var job_company = "";
var job_desc = "";
var job_location = "";
var job_nationality = "";
var job_experience = "";
var job_keyskils = "";
var job_function = "";
var job_region = "United Arab Emirates";
var job_industry = "";
var job_date = "";
var job_education = "";
var next_page_class = 0;
var page_number = 1;
nextPage_label.Text = "Yes";
/* initializing chrome driver */
var chrome_Driver = ChromeDriverService.CreateDefaultService();
//no need to show disturbing command window so set it hidden
chrome_Driver.HideCommandPromptWindow = true;
var main_page_chrome_Driver_Obj = new ChromeDriver(chrome_Driver, new ChromeOptions());
/* create a csv file for output*/
try
{
File.WriteAllText(Path.Combine(path_Desktop, "JobsInDubai_Scrapped_DataSet_allTypesJobs.csv"), "Job Title, Company, Description, Location, Experience, Keyskills, Education, Region, Industry, Date\r\n");
}
catch (Exception ex)
{
//incase of file is open or readonly or no permission show message
result_richTextBox.AppendText(ex.StackTrace);
MessageBox.Show("Cannot write to file !");
}
do
{
if (next_page_class == 0) { nextPage_label.Text = "No"; }
//this.page_no_label.Text = page_number.ToString();
main_page_chrome_Driver_Obj.Navigate().GoToUrl(default_Url + "/job_list.asp?page=" + page_number + "&lstIndustryID=0&txtKeyword=&isSearch=False");
// click the Detail View Link to show extra information for jobs
var click_Detail_View = main_page_chrome_Driver_Obj.FindElement(By.XPath("//*/div[@id='divShowAll']/a"));
if (click_Detail_View.Text == "Detail View") { click_Detail_View.Click(); }
//count number of jobs on main page max 10 jobs
var var_job_count = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@class='even']");
var var_job_des = main_page_chrome_Driver_Obj.FindElementsByXPath("//*/table[@class='expand']");
for (int job_index = 0; job_index < var_job_count.Count(); job_index++)
{
job_title = var_job_count[job_index].FindElement(By.XPath("./td[2]")).Text;
job_title = cleanString(job_title);
job_company = var_job_des[job_index].FindElement(By.XPath("./tbody/tr[2]/td[2]/span[1]")).Text;
job_company = cleanString(job_company);
job_location = var_job_des[job_index].FindElement(By.XPath("./tbody/tr[2]/td[2]/span[2]")).Text;
job_location = cleanString(job_location);
var job_full_desc_raw = var_job_des[job_index].FindElement(By.XPath("./tbody/tr[4]/td")).Text;
var job_full_desc = job_full_desc_raw.Replace(System.Environment.NewLine, " ");
job_experience = ParseBetween(job_full_desc, "Experience :", "Years");
job_experience = cleanString(job_experience);
if (job_experience == string.Empty) { job_experience = "Not Required"; } else { job_experience = job_experience + " Years"; }
job_education = ParseBetween(job_full_desc, "Education :", "Experience :");
job_education = cleanString(job_education);
if (job_education == string.Empty) { job_education = "NULL"; }
//All this headach to extract Skills
job_keyskils = ParseBetween(job_full_desc, "Skills :", "Description :");
job_keyskils = cleanString(job_keyskils);
if (job_keyskils == string.Empty)
{
job_keyskils = ParseBetween(job_full_desc, "Skills :", "Responsibilities :");
job_keyskils = cleanString(job_keyskils);
if (job_keyskils == string.Empty)
{
job_keyskils = ParseBetween(job_full_desc, "Responsibilities :", "Description :");
job_keyskils = cleanString(job_keyskils);
if (job_keyskils == string.Empty) { job_keyskils = "NULL"; }
}
}
//End Skills
job_industry = var_job_count[job_index].FindElement(By.XPath("./td[3]")).Text;
job_industry = cleanString(job_industry);
job_date = var_job_count[job_index].FindElement(By.XPath("./td[4]")).Text;
job_date = cleanString(job_date);
job_date = timeStampCnversion(job_date);
job_desc = cleanString(job_full_desc);
//+++++++++++++ values Scraper Region ENd +++++++++++++++++++++++++
//Dispaly Output and write data to CSV file
//Job Title, Company, Description, Location, Experience, Keyskills, Region, Industry, Date\r\n
result_richTextBox.AppendText("\nJob Title: " + job_title.ToString().Trim() + "\nCompany: " + job_company.ToString().Trim() + "\nDescription: " + job_desc.ToString().Trim() + "\nLocation: " + job_location.ToString().Trim() + "\nExperience: " + job_experience.ToString().Trim() + "\nKeyskills: " + job_keyskils.ToString().Trim() + "\nEducation: " + job_education.ToString().Trim() + "\nRegion: " + job_region.ToString().Trim() + "\nIndustry: " + job_industry.ToString().Trim() + "\nDate: " + job_date.ToString().Trim() + "\r.....................................................................................................\n");
File.AppendAllText(Path.Combine(path_Desktop, "JobsInDubai_Scrapped_DataSet_allTypesJobs.csv"), job_title.ToString().Trim().Replace(",", "") + "," + job_company.ToString().Trim().Replace(",", "") + "," + job_desc.ToString().Trim().Replace(",", "") + "," + job_location.ToString().Trim().Replace(",", "") + "," + job_experience.ToString().Trim().Replace(",", "") + "," + job_keyskils.ToString().Trim().Replace(",", "") + "," + job_education.ToString().Trim().Replace(",", "") + "," + job_region.ToString().Trim().Replace(",", "") + "," + job_industry.ToString().Trim().Replace(",", "") + "," + job_date.ToString().Trim().Replace(",", "") + "\r\n");
}
//these three line to loop main pages
var var_next_page_class = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@class='next']");
next_page_class = var_next_page_class.Count();
page_number++;
} while (next_page_class == 1);
main_page_chrome_Driver_Obj.Quit();
main_page_chrome_Driver_Obj.Dispose();
}
//double progress = (x / max) * 100;
private void NaukriGulf_Function(string defautl_Url)
{
/* initializing chrome driver */
var chrome_Driver = ChromeDriverService.CreateDefaultService();
//no need to show disturbing command window so set it hidden
chrome_Driver.HideCommandPromptWindow = true;
//initialize chrome driver
var main_page_chrome_Driver_Obj = new ChromeDriver(chrome_Driver, new ChromeOptions());
/* create a csv file for output*/
try
{
File.WriteAllText(Path.Combine(path_Desktop, "NaukriGulf_Scrapped_DataSet_" + job_region_selected + "_" + job_type_selected + ".csv"), "Job Title, Company, Description, Location, Experience, Keyskills, Region, Industry, Date\r\n");
}
catch (Exception ex)
{
//incase of file is open or readonly or no permission show message
result_richTextBox.AppendText(ex.StackTrace);
MessageBox.Show("Cannot write to file ! \r check whether file is open");
}
int totalPages_Int = 1;
do
{
main_page_chrome_Driver_Obj.Navigate().GoToUrl(defautl_Url + "/jobs-in-uae-" + totalPages_Int + "?fa=" + job_type_Selection); //job_type_Selection = 016; totalPages_Int = 0;
main_page_chrome_Driver_Obj.Manage().Timeouts().ImplicitlyWait(TimeSpan.FromSeconds(1));
//var job_functional_area = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@id='fareaContainer']/div/a");
var main_page_dom = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@id='resultsData']");
if (totalPages_Int != 1)
{
//skip Pages Counting in rest of loop
}
else
{
// only count total number pages once at page one
var main_pages_count = main_page_dom[0].FindElement(By.XPath(".//div[1]/div[2]/strong[2]")).Text;
//wola ! I need only digits, but it has " 15302 Advertisements ", extract only digits
var str_main_pages_count = Regex.Match(main_pages_count, @"\d+").Value;
//It needs to be converted to iNT
int_main_pages_total_jobs_count = Int32.Parse(str_main_pages_count);
int_main_pages_count = (int)Math.Ceiling((double)int_main_pages_total_jobs_count / (double)25);
}
var main_page_jobs = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@class='artical']"); //25 jobs per page
if (main_page_jobs.Count() == 25) { }
else
{
DateTime dt = DateTime.Now + TimeSpan.FromSeconds(7);
do
{
main_page_jobs = main_page_chrome_Driver_Obj.FindElementsByXPath("//*[@class='artical']"); //25 jobs per page
//MessageBox.Show(main_page_jobs.Count().ToString());
} while (DateTime.Now < dt);
}
//+++++++++++++++++++++++++ 25 Jobs Loop in Single Main Page starts ++++++++++++++++++++++++
current_number_jobs = current_number_jobs + (int)(main_page_jobs.Count());
for (int job_index = 0; job_index < main_page_jobs.Count(); job_index++)
{
// declare Initialize all variables to nothing
var job_title = "";
var job_company = "";
var job_desc = "";
var job_location = "";
var job_region = "";
//var job_nationality = "";
var job_experience = "";
var job_keyskils = "";
// var job_function = "";
//var job_role = "";
var job_industry = job_type_selected;
var job_date = "";
//var job_ref_code = "";
//+++++++++++++ values Scraper Region +++++++++++++++++++++++++
job_title = main_page_jobs[job_index].FindElement(By.XPath("./div[@class='aCont']/div/a/span[1]")).Text;
job_company = main_page_jobs[job_index].FindElement(By.XPath("./div[@class='aCont']/div/a/span[2]")).Text;
var job_desc_full = main_page_jobs[job_index].FindElement(By.XPath("./div[@class='aCont']/a/p")).Text; //.GetAttribute("href");
job_desc = job_desc_full.Replace(System.Environment.NewLine, " ");
//int index = job_desc_full.IndexOf("-?");
//if (index > 0)
// job_desc = job_desc_full.Substring(0, index);
//job_desc = job_desc.Replace("http://www.naukrigulf.com/job-listings-", string.Empty);
job_location = main_page_jobs[job_index].FindElement(By.XPath("./div[@class='aCont']/div/p/span[2]")).Text;
string input_job_location = job_location;
string[] values_job_location = input_job_location.Split('-'); //Abu Dhabi - United Arab Emirates
job_region = values_job_location.Last(); // United Arab Emirates
job_experience = main_page_jobs[job_index].FindElement(By.XPath("./div[@class='aCont']/div/p/span[1]")).Text;
try
{
string input_job_keyskils = main_page_jobs[job_index].FindElement(By.XPath("./div[@class='aCont']/a/span")).Text;
string removeString = "Keywords: "; //Keywords: sharepoint, MOSS, .Net, ms sql
job_keyskils = input_job_keyskils.Replace(removeString, ""); //sharepoint, MOSS, .Net, ms sql
}
catch (Exception ex)
{
job_keyskils = "NULL";
//result_richTextBox.AppendText(ex.StackTrace);
//continue;
}
var job_date_before_conversion = main_page_jobs[job_index].FindElement(By.XPath(".//./span[@class='jbpost']/span[1]")).Text;
job_date = timeStampCnversion(job_date_before_conversion + " 2015");
//job_desc.Split(new[] { "years-" }, StringSplitOptions.None)[1];
//+++++++++++++ values Scraper Region ENd +++++++++++++++++++++++++
//Dispaly Output and write data to CSV file
//Job Title, Company, Description, Location, Experience, Keyskills, Region, Industry, Date\r\n
result_richTextBox.AppendText("\nJob Title: " + job_title.ToString().Trim() + "\nCompany: " + job_company.ToString().Trim() + "\nDescription: " + job_desc.ToString().Trim() + "\nLocation: " + job_location.ToString().Trim() + "\nExperience: " + job_experience.ToString().Trim() + "\nKeyskills: " + job_keyskils.ToString().Trim() + "\nRegion: " + job_region.ToString().Trim() + "\nIndustry: " + job_industry.ToString().Trim() + "\nDate: " + job_date.ToString().Trim() + "\r.....................................................................................................\n");
File.AppendAllText(Path.Combine(path_Desktop, "NaukriGulf_Scrapped_DataSet_" + job_region_selected + "_" + job_type_selected + ".csv"), job_title.ToString().Trim().Replace(",", "") + "," + job_company.ToString().Trim().Replace(",", "") + "," + job_desc.ToString().Trim().Replace(",", "") + "," + job_location.ToString().Trim().Replace(",", "") + "," + job_experience.ToString().Trim().Replace(",", "") + "," + job_keyskils.ToString().Trim().Replace(",", "") + "," + job_region.ToString().Trim().Replace(",", "") + "," + job_industry.ToString().Trim().Replace(",", "") + "," + job_date.ToString().Trim().Replace(",", "") + "\r\n");
//show progress/status
this.job_number_status_label.Text = current_number_jobs.ToString() + "/" + int_main_pages_total_jobs_count.ToString();
this.result_richTextBox.AppendText("\r\n" + job_title + "\t\t");
} //End Sub Page Loop
this.job_pages_status_label.Text = totalPages_Int + "/" + int_main_pages_count;
this.scraper_progressbar_label.Visible = true;
this.scraper_progressBar.Visible = true;
scraper_progressBar.Value = totalPages_Int * scraper_progressBar.Maximum / int_main_pages_count;
Application.DoEvents();
if (scraper_progressBar.Value != 100)
{
scraper_progressbar_label.Text = string.Format("{0}% Completed", scraper_progressBar.Value);
}
else { scraper_progressbar_label.Text = "Finished!"; }
totalPages_Int++;
} while (totalPages_Int <= int_main_pages_count);//End Main Page Loop
//dispose all the data associated with chromedriver
main_page_chrome_Driver_Obj.Dispose();
//Quit chrome driver
main_page_chrome_Driver_Obj.Quit();
} //end NaukriGulf_Function
