private string ExecuteCommand(string url)
{
try
{
using (var phantom = new PhantomJSDriver())
{
var indexFile = Server.MapPath("~/Scripts/PhantomJS/index.js");
var scriptSource = System.IO.File.ReadAllText(indexFile);
var script = phantom.ExecutePhantomJS(scriptSource);
phantom.Navigate().GoToUrl("https://www.bing.com");
phantom.FindElement(By.Id("sb_form_q")).SendKeys("learn2automate");
//Click on Search
phantom.FindElement(By.Id("sb_form_go")).Click();
Screenshot sh = phantom.GetScreenshot();
sh.SaveAsFile(@"C:\Temp.jpg", ImageFormat.Png);
phantom.Quit();
}
}
catch (Exception ex)
{
}
return string.Empty;
}
/// <summary>
/// Starting point for the application.
/// </summary>
/// <param name="args"> The arguments passed in from the console. </param>
public static void Main(string[] args)
{
PhantomJSDriverService service = PhantomJSDriverService.CreateDefaultService();
service.IgnoreSslErrors = true;
service.LoadImages = false;
Console.WriteLine("gebruikersnaam");
string gebruikersnaam = Console.ReadLine();
Console.WriteLine("wachtwoord");
string wachtwoord = Console.ReadLine();
IWebDriver driver = new PhantomJSDriver(service);
driver.Navigate().GoToUrl("https://mijn.ing.nl/internetbankieren/SesamLoginServlet");
IWebElement gebruikersnaamElem = driver.FindElement(By.XPath("//div[@id='gebruikersnaam']/descendant::input"));
gebruikersnaamElem.SendKeys(gebruikersnaam);
IWebElement wachtwoordElem = driver.FindElement(By.XPath("//div[@id='wachtwoord']/descendant::input"));
wachtwoordElem.SendKeys(wachtwoord);
IWebElement button = driver.FindElement(By.CssSelector("button.submit"));
button.Click();
((ITakesScreenshot)driver).GetScreenshot().SaveAsFile("ing.png", ImageFormat.Png);
System.IO.File.WriteAllText("ing.html", driver.PageSource);
driver.Quit();
Console.WriteLine("We are done");
Console.ReadKey();
}
public static void OpenPhantomJs()
{
// initialize a WebDriver instance
IWebDriver driver = new PhantomJSDriver();
// load google search page
driver.Navigate().GoToUrl("https://www.google.ca");
// print title
Console.WriteLine("Page title: " + driver.Title);
// enter search word and submit
IWebElement element = driver.FindElement(By.Name("q"));
element.SendKeys("Cheese");
element.Submit();
// print title
Console.WriteLine("Page title: " + driver.Title);
// quit the driver
driver.Quit();
}
public void DownloadStroy(string url)
{
if (File.Exists(OutputFile))
{
File.Delete(OutputFile);
}
WriteFile("<!DOCTYPE html><html xmlns=\"http://www.w3.org/1999/xhtml\"><head><meta charset=\"utf-8\"/></head><body>");
IWebDriver driver = new PhantomJSDriver();
driver.Navigate().GoToUrl(url);
var isTextAvailable = true;
while (isTextAvailable)
{
Log($"Reading page: {driver.Title}");
var contentDiv = driver.FindElement(By.ClassName("b-story-body-x"));
WriteFile(contentDiv.GetInnerHtml());
var nextButton = driver.TryFindElement(By.CssSelector("a.b-pager-next"));
if (nextButton != null)
{
nextButton.Click();
Log("Navigating to next page");
}
else
{
isTextAvailable = false;
}
}
WriteFile("</body></html>");
driver.Quit();
}
public Property Property(Property property)
{
try
{
var query = string.Format("http://www.zillow.com/homedetails/{0}_zpid/", property.Zpid);
var driver = new PhantomJSDriver();
driver.Navigate().GoToUrl(query);
var body = driver.FindElementByTagName("body");
Console.Write(body.Text);
Thread.Sleep(2000);
var webElement = driver.FindElement(By.ClassName("main-wrapper"));
property.Link = query;
try
{
var thumbnailElem = webElement.FindElement(By.ClassName("hip-photo"));
if (thumbnailElem != null)
{
property.Thumbnail = thumbnailElem.GetAttribute("src");
}
var fullAddressElem = webElement.FindElement(By.TagName("h1"));
if (fullAddressElem != null)
{
var split = fullAddressElem.Text.Split(',');
property.Address = split[0];
}
var cityElem = webElement.FindElement(By.ClassName("addr_city"));
if (cityElem != null)
{
var split1 = cityElem.Text.Split(',');
var split2 = split1[1].Split(' ');
property.City = split1[0];
property.State = split2[1];
}
var priceElem = webElement.FindElement(By.ClassName("main-row"));
if (priceElem != null)
{
if (priceElem.Text.Contains("$"))
{
property.Price = Convert.ToDouble(priceElem.Text.Replace("$", "").Replace("from: ", ""));
}
}
var valueChangeElem = webElement.FindElement(By.ClassName("zest-change"));
if (valueChangeElem != null)
{
var split = valueChangeElem.Text.Split(' ');
property.ValueChange = Convert.ToDouble(split[0].Replace("$", "").Replace(",", ""));
}
var zestimateElms = webElement.FindElements(By.ClassName("zest-value"));
foreach (var zestimateElm in zestimateElms)
{
var text = zestimateElm.Text;
if (text.Contains("/mo"))
{
property.RentZestimate = Convert.ToDouble(text.Replace("/mo", "").Replace("$", ""));
}
else if(!text.Contains("X"))
{
property.Zestimate = Convert.ToDouble(text.Replace("$", ""));
}
}
var zsgListUl = webElement.FindElements(By.ClassName("zsg-list_square"));
foreach (var element in zsgListUl)
{
var lis = element.FindElements(By.TagName("li"));
foreach (var li in lis)
{
var text = li.Text;
if (text.Contains("Family"))
{
property.PropertyType = text;
}
if (text.Contains("Built in"))
{
var modifiedBuiltIn = text.Replace("Built in ", "");
property.YearBuilt = Convert.ToInt32(modifiedBuiltIn);
}
if (text.Contains("Lot"))
{
var modifiedSqft = text.Replace("Lot: ", "").Replace(" sqft", "").Replace(",","");
if (modifiedSqft.Contains("acres"))
{
modifiedSqft = modifiedSqft.Replace(" acres", "");
property.LotSizeSqFt = (int) (Convert.ToDouble(modifiedSqft)*43560);
}
else
{
property.LotSizeSqFt = Convert.ToInt32(modifiedSqft);
}
}
if (text.Contains("Unit count: "))
{
property.NumUnits = Convert.ToInt32(text.Replace("Unit count: ", ""));
}
if (text.Contains("on Zillow"))
{
var textSplit = text.Split(' ');
property.DaysOnZillow = Convert.ToInt32(textSplit[0]);
}
}
}
property.AnnualTaxes = property.Price * .01;
var addrBbsElms = webElement.FindElements(By.ClassName("addr_bbs"));
foreach (var addrBbs in addrBbsElms)
{
var text = addrBbs.Text;
if (text.Contains("sqft"))
{
var splitSqft = text.Split(' ');
property.FinishedSqFt = Convert.ToInt32(splitSqft[0].Replace(",",""));
}
if (text.Contains("bath"))
{
var formatedBaths = text.Replace("bath", "").Replace("--", "").Replace(" ", "").Replace("s","");
if (formatedBaths != "")
{
property.Bathrooms = Convert.ToInt32(formatedBaths);
}
}
if (text.Contains("bed"))
{
var formattedBds = text.Replace("bed", "").Replace("--", "").Replace(" ", "").Replace("s", "");
if (formattedBds != "")
{
property.Bedrooms = Convert.ToInt32(formattedBds);
}
}
}
var statusElem = webElement.FindElement(By.ClassName("status-icon-row"));
if (statusElem != null)
{
property.Status = statusElem.Text;
}
var descElem = webElement.FindElement(By.ClassName("notranslate"));
if (descElem != null)
{
property.HomeDescription = descElem.Text;
}
var insuranceElem = webElement.FindElement(By.Id("home-ins-out"));
if (insuranceElem != null)
{
property.Insurance = Convert.ToDouble(insuranceElem.Text.Replace("$", ""));
}
property.LoanAmount = property.Price * .80;
property.LoanType = "30 year fixed";
property.LoanTerm = 360;
//var interestElem = webElement.FindElements(By.XPath("//*[@id='zmm-calc-input-program']/option"));
//if (interestElem != null)
//{
// newProperty.InterestRate = Convert.ToDouble(interestElem[0].Text);
//}
property.DownPayment = property.Price * .20;
var monthlyPaymentElem = webElement.FindElement(By.Id("pi-payment-out"));
if (monthlyPaymentElem != null)
{
property.DebtServicePI = Convert.ToDouble(monthlyPaymentElem.Text.Replace("$", ""));
}
property.ClosingCosts = property.Price * .025;
if (property.CurrentRent < 1)
{
property.CurrentRent = property.RentZestimate;
}
}
catch (Exception ex)
{
Console.WriteLine("Error searching Zillow. " + ex.ToString());
}
driver.Quit();
return property;
}
catch (Exception ex)
{
Console.WriteLine("Error searching Zillow. " + ex.ToString());
}
return null;
}
public void TestPhantomJS()
{
driver = new PhantomJSDriver();
OpenPage<GoogleHomePage>("http://www.google.com/");
driver.Quit();
}
// get data from website using webdriver and save its contents into given IWebElement arguments.
void fetchData(ref PhantomJSDriver driver, ref IWebElement myContent, ref IWebElement bidDetails, ref string[] enteredData)
{
driver = getWebBrowserDriverInit();
string market = enteredData[0];
string enteredDateFrom = enteredData[1];
string enteredDateTo = enteredData[2];
Exception tmpEx = null; // for possible exception recognition
try
{
driver.Navigate().GoToUrl("https://kapalk1.mavir.hu/kapar/lt-publication.jsp?locale=en_GB");
// TODO edit/delete if needed - this is not best approach (webpage can change)
if (!(driver.Title.CompareTo("Kapar") == 0))
{
driver.Quit();
throw new Exception("Error, webpage is not accessable or there is something wrong with your connection.");
}
driver.FindElementByXPath("/html/body/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/div/table/tbody/tr/td[2]/input").Clear();
driver.FindElementByXPath("/html/body/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/div/table/tbody/tr/td[2]/input").SendKeys(enteredDateFrom); //SendKeys("12/4/2015");
driver.FindElementByXPath("/html/body/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/div/table/tbody/tr/td[4]/input").Clear();
driver.FindElementByXPath("/html/body/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/div/table/tbody/tr/td[4]/input").SendKeys(enteredDateTo); //SendKeys("12/10/2015");
driver.FindElementByXPath("/html/body/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/div/table/tbody/tr/td[6]/select").SendKeys(market); //SendKeys("All");
driver.FindElementByXPath("/html/body/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/div/table/tbody/tr/td[8]/button").Click();
WebDriverWait waitForContent = new WebDriverWait(driver, TimeSpan.FromSeconds(60));
myContent = waitForContent.Until<IWebElement>((d) =>
{
return d.FindElement(By.XPath("/html/body/table/tbody/tr[3]/td/table"));
});
// xpath for 1st row in table
driver.FindElementByXPath("/html/body/table/tbody/tr[3]/td/table/tbody/tr[2]/td[21]/a").Click();
WebDriverWait waitForBidDetails = new WebDriverWait(driver, TimeSpan.FromSeconds(60));
bidDetails = waitForContent.Until<IWebElement>((d) =>
{
return d.FindElement(By.XPath("//html/body/div/div/table/tbody/tr[2]/td[2]/div/div/table/tbody/tr[1]/td/table"));
});
}
catch (Exception e)
{
tmpEx = e;
}
if(tmpEx != null)
throw new Exception(tmpEx.Message);
}
public void ZergNetDotcom(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
{
Stopwatch watch = new Stopwatch();
watch.Start();
List<string> adIds = new List<string>();
List<string> adUrls = new List<string>();
List<string> adImages = new List<string>();
List<string> adTitle = new List<string>();
List<string> imgwidth = new List<string>();
List<string> imgheight = new List<string>();
try
{
ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
//NetworkFunctions ns = new NetworkFunctions();////"//div[@class='zergentity']/div/a"
List<NetworkFunctions> List = new List<NetworkFunctions>();
foreach (var v1 in getAdds)
{
string s2 = v1.GetAttribute("href");
adUrls.Add(s2);
try
{
string s1 = s2.Split('/')[4];
// ns.adId = s1.Trim();
// List.Add(ns);
adIds.Add(s1.Trim());
}
catch { }
try
{
string s4 = v1.GetAttribute("outerText");
adTitle.Add(s4);
// ns.adTitle = s4;
// List.Add(ns);
}
catch
{ }
}
ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath(path2));//"//div[@class='zergentity']/a/img"
foreach (var v2 in getImages)
{
string s3 = v2.GetAttribute("src");
adImages.Add(s3);
//ns.adImage = s3;
//List.Add(ns);
// s3 = s3.Split('_','/')[2];
// adIds.Add(s3);
try
{
string s5 = v2.GetAttribute("style");
string width = s5.Split(';', ':')[5];
string height = s5.Split(';', ':')[7];
imgwidth.Add(width);
imgheight.Add(height);
}
catch
{ }
}
watch.Stop();
string time = watch.Elapsed.ToString();
if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
{
_form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);
}
}
catch(Exception)
{ }
driver.Quit();
driver.Dispose();
adIds = null;
adUrls = null;
adImages = null;
adTitle = null;
}
public void Taboola(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
{
Stopwatch watch = new Stopwatch();
watch.Start();
List<string> adIds = new List<string>();
List<string> adUrls = new List<string>();
List<string> adImages = new List<string>();
List<string> adTitle = new List<string>();
List<string> imgwidth = new List<string>();
List<string> imgheight = new List<string>();
try
{
ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
// NetworkFunctions ns = new NetworkFunctions();
List<NetworkFunctions> List = new List<NetworkFunctions>();
foreach (var v1 in getAdds)
{
try
{
string s1 = v1.GetAttribute("data-item-id");
if (s1.Contains("~"))
{
s1 = s1.Split('~')[4];
adIds.Add(s1.Trim().Replace("-", ""));
//ns.adId = s1;
//List.Add(ns);
}
else
{
adIds.Add(s1.Trim().Replace("-", ""));
//ns.adId = s1.Trim().Replace("-", "");
// List.Add(ns);
}
}
catch { }
try
{
string s4 = v1.GetAttribute("data-item-title").Replace(" ", "");
if (s4 != "")
{
adTitle.Add(s4);
//ns.adTitle = s4;
//List.Add(ns);
}
}
catch
{ }
try
{
string s3 = v1.GetAttribute("data-item-thumb");
if (s3 != null)
{
adImages.Add(s3);
//ns.adImage = s3;
//List.Add(ns);
}
}
catch
{ }
}
ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath(path2));
foreach (var v2 in getImages)
{
string s2 = v2.GetAttribute("href");
adUrls.Add(s2);
//ns.adUrl = s2;
//List.Add(ns);
}
watch.Stop();
string time = watch.Elapsed.ToString();
if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
{
_form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);
}
}
catch (Exception)
{ }
driver.Quit();
driver.Dispose();
adIds = null;
adUrls = null;
adImages = null;
adTitle = null;
}
public void Outbrain(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
{
Stopwatch watch = new Stopwatch();
watch.Start();
List<string> adIds = new List<string>();
List<string> adUrls = new List<string>();
List<string> adImages = new List<string>();
List<string> adTitle = new List<string>();
List<string> imgwidth = new List<string>();
List<string> imgheight = new List<string>();
try
{
ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
// NetworkFunctions ns = new NetworkFunctions();////a[contains(@onmousedown,'paid.outbrain.com/')]
List<NetworkFunctions> List = new List<NetworkFunctions>();
foreach (var v1 in getAdds)
{
string s2 = v1.GetAttribute("href");
adUrls.Add(s2);
//ns.adUrl = s2;
//List.Add(ns);
}
ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath(path2));
foreach (var v2 in getImages)//"//img[@class='ob-rec-image ob-show']"
{
string s3 = v2.GetAttribute("src");
adImages.Add(s3);
string id = s3.Split('/')[8];
adIds.Add(id);
//ns.adImage = s3;
//List.Add(ns);
}
ReadOnlyCollection<IWebElement> getID = driver.FindElements(By.XPath(path3));
foreach (var v3 in getID)//"//span[@class='ob-unit ob-rec-text']"
{
string v7 = v3.GetAttribute("title");
if (v7 != null)
{
adTitle.Add(v7);
// List.Add(ns);
}
}
watch.Stop();
string time = watch.Elapsed.ToString();
if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
{
_form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);
}
}
catch (Exception)
{ }
driver.Quit();
driver.Dispose();
adIds = null;
adUrls = null;
adImages = null;
adTitle = null;
}
public void Mgid(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
{
Stopwatch watch = new Stopwatch();
watch.Start();
List<string> adIds = new List<string>();
List<string> adUrls = new List<string>();////div[@class='text-elements']/div/a //div[@class='mcimg']/a/img
List<string> adImages = new List<string>();////div[contains(@class,'mgline teaser')]
List<string> adTitle = new List<string>();
List<string> imgwidth = new List<string>();
List<string> imgheight = new List<string>();
try
{
ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath("//div[@class='text-elements']/div/a"));
//NetworkFunctions ns = new NetworkFunctions();
List<NetworkFunctions> List = new List<NetworkFunctions>();
foreach (var v1 in getAdds)
{
string s2 = v1.GetAttribute("href");
adUrls.Add(s2);
//ns.adUrl = s2;
///List.Add(ns);
try
{
string s4 = v1.GetAttribute("outerText");
adTitle.Add(s4);
//ns.adTitle = s4;
//List.Add(ns);
}
catch
{ }
}
ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath("//div[@class='mcimg']/a/img"));
foreach (var v2 in getImages)
{
string s3 = v2.GetAttribute("src");
if (s3 != null)
{
adImages.Add(s3);
//ns.adImage = s3;
//List.Add(ns);
s3 = s3.Split('_')[1].Replace(".jpg", "");
string width = s3.Split('x')[0];
string height = s3.Split('x')[1];
imgwidth.Add(width);
imgheight.Add(height);
}
//try
//{
// string s5 = v2.GetAttribute("width");
// if (s5 != null)
// {
// imgwidth.Add(s5);
// //ns.imgHeight = s5;
// //List.Add(ns);
// }
//}
//catch
//{ }
//try
//{
// string s6 = v2.GetAttribute("height");
// if (s6 != null)
// {
// imgheight.Add(s6);
// // ns.imgWidth = s6;
// //List.Add(ns);
// }
//}
//catch
//{ }
}
ReadOnlyCollection<IWebElement> getID = driver.FindElements(By.XPath(path3));
foreach (var v3 in getID)
{
string v7 = v3.GetAttribute("class");
if (v7 != null)
{
adIds.Add(v7.Split('-')[1].Trim());
// List.Add(ns);
}
}
watch.Stop();
string time = watch.Elapsed.ToString();
if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
{
_form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);
}
}
catch (Exception)
{ }
driver.Quit();
driver.Dispose();
adIds = null;
adUrls = null;
adImages = null;
adTitle = null;
}
public void Adblade(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
{
Stopwatch watch = new Stopwatch();
watch.Start();
var service1 = PhantomJSDriverService.CreateDefaultService();
service1.HideCommandPromptWindow = true;
var driver1 = new PhantomJSDriver(service1);
List<string> adIds = new List<string>();
List<string> adUrls = new List<string>();
List<string> adImages = new List<string>();
List<string> adTitle = new List<string>();
List<string> imgwidth = new List<string>();
List<string> imgheight = new List<string>();
try
{
ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
//NetworkFunctions ns = new NetworkFunctions();
List<NetworkFunctions> List = new List<NetworkFunctions>();
foreach (var v1 in getAdds)
{
try
{
string s2 = v1.GetAttribute("href");
try
{
driver1.Navigate().GoToUrl(s2);
//driver.GetScreenshot().SaveAsFile("image3.png", ImageFormat.Png);
IWebElement geturl = driver1.FindElement(By.XPath("//meta[@property='og:url']"));
string content = geturl.GetAttribute("content");
adUrls.Add(content);
}
catch (Exception ex)
{
adUrls.Add(s2);
}
}
catch
{ }
// ns.adUrl = s2;
//List.Add(ns);
try
{
string s4 = v1.GetAttribute("outerText");
adTitle.Add(s4);
// ns.adTitle = s4;
//List.Add(ns);
}
catch
{ }
}
ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath(path2));
foreach (var v2 in getImages)
{//http://staticd.cdn.adblade.com/banners/images/298x224/4365_548f126ed8024.jpg
string s3 = v2.GetAttribute("src");
if (s3 != null)
{
adImages.Add(s3);
//ns.adImage = s3;
//List.Add(ns);
s3 = s3.Split('/')[5];
string width = s3.Split('x')[0];
string height = s3.Split('x')[1];
imgwidth.Add(width);
imgheight.Add(height);
}
}
ReadOnlyCollection<IWebElement> getID = driver.FindElements(By.XPath(path3));
foreach (var v3 in getID)
{
try
{
string v7 = v3.GetAttribute("data-adid");
if (v7 != null)
{
adIds.Add(v7);
//ns.adId = v7.Trim();
// List.Add(ns);
}
}
catch { }
}
watch.Stop();
string time = watch.Elapsed.ToString();
if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
{
_form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);
}
}
catch (Exception)
{ }
driver.Quit();
driver.Dispose();
driver1.Quit();
driver1.Dispose();
adIds = null;
adUrls = null;
adImages = null;
adTitle = null;
}
public void ContentDotAd(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
{
Stopwatch watch = new Stopwatch();
watch.Start();
List<string> adIds = new List<string>();
List<string> adUrls = new List<string>();
List<string> adImages = new List<string>();
List<string> adTitle = new List<string>();
List<string> imgwidth = new List<string>();
List<string> imgheight = new List<string>();
// List<NetworkFunctions> List = new List<NetworkFunctions>();
try
{
ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
foreach (var v1 in getAdds)
{
string s1 = v1.GetAttribute("onclick");
s1 = s1.Split(',')[2].Replace("'", "");
adIds.Add(s1.Trim());
string s2 = v1.GetAttribute("href");
adUrls.Add(s2);
}//div[@class='ac_container']/a/img
ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath("//div[@class='ac_container']/a/img"));
foreach (var v2 in getImages)
{
string s3 = v2.GetAttribute("src");
adImages.Add(s3);
string s4 = v2.GetAttribute("title");
adTitle.Add(s4);
string height = v2.GetAttribute("height");
string width = v2.GetAttribute("width");
imgwidth.Add(height);
imgheight.Add(width);
}
watch.Stop();
string time = watch.Elapsed.ToString();
if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
{
_form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent,time);
}
}
catch(Exception)
{ }
driver.Quit();
driver.Dispose();
adIds = null;
adUrls = null;
adImages = null;
adTitle = null;
}
public List<Property> Properties(string query)
{
var results = new List<Property>();
try
{
var driver = new PhantomJSDriver();
driver.Navigate().GoToUrl(query);
var body = driver.FindElementByTagName("body");
Console.Write(body.Text);
Thread.Sleep(2000);
var articles = driver.FindElementsByTagName("article");
foreach (var webElement in articles)
{
try
{
var newProperty = new Property
{
Thumbnail = webElement.FindElement(By.ClassName("photo")).GetAttribute("data-photourl"),
Link = webElement.FindElement(By.ClassName("hdp-link")).GetAttribute("href")
};
var fullAddress = webElement.FindElement(By.ClassName("image-loaded")).GetAttribute("alt");
var split = fullAddress.Split(',');
var splitCount = split.Count();
var city = split[splitCount - 2];
var state = split.Last();
newProperty.Address = split[0];
newProperty.City = city;
newProperty.State = state;
results.Add(newProperty);
}
catch (Exception ex)
{
Console.WriteLine("Error searching Zillow. " + ex.ToString());
}
}
driver.Quit();
return results.ToList();
}
catch (Exception ex)
{
Console.WriteLine("Error searching Zillow. " + ex.ToString());
}
return null;
}
public void YahooGemini(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
{
Stopwatch watch = new Stopwatch();
watch.Start();
List<string> adIds = new List<string>();
List<string> adUrls = new List<string>();
List<string> adImages = new List<string>();
List<string> adTitle = new List<string>();
List<string> imgwidth = new List<string>();
List<string> imgheight = new List<string>();
try
{
ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath("//li[@class='content has-image voh-parent cf']/div/div/div/h3/a"));
//NetworkFunctions ns = new NetworkFunctions();
List<NetworkFunctions> List = new List<NetworkFunctions>();
foreach (var v1 in getAdds)
{
string s2 = v1.GetAttribute("href");
adUrls.Add(s2);
//ns.adUrl = s2;
//List.Add(ns);
try
{
string s4 = v1.GetAttribute("outerText");
adTitle.Add(s4);
//ns.adTitle = s4;
//List.Add(ns);
}
catch
{ }
}
ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath("//li[@class='content has-image voh-parent cf']/div/div/img"));
foreach (var v2 in getImages)
{
string s3 = v2.GetAttribute("src");
if (s3 != null)
{
adImages.Add(s3);
//ns.adImage = s3;
//List.Add(ns);
}
try
{
string s5 = v2.GetAttribute("width");
if (s5 != null)
{
imgwidth.Add(s5);
//ns.imgHeight = s5;
//List.Add(ns);
}
}
catch
{ }
try
{
string s6 = v2.GetAttribute("height");
if (s6 != null)
{
imgheight.Add(s6);
// ns.imgWidth = s6;
// List.Add(ns);
}
}
catch
{ }
}
ReadOnlyCollection<IWebElement> getID = driver.FindElements(By.XPath("//li[@class='content has-image voh-parent cf']"));
foreach (var v3 in getID)
{
try
{
string v7 = v3.GetAttribute("data-uuid");
if (v7 != null)
{
adIds.Add(v7);
//ns.adId = v7.Trim();
//List.Add(ns);
}
}
catch { }
}
watch.Stop();
string time = watch.Elapsed.ToString();
if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
{
_form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);
}
}
catch (Exception)
{ }
driver.Quit();
driver.Dispose();
adIds = null;
adUrls = null;
adImages = null;
adTitle = null;
}
public void ScheduleGET()
{
var driverService = PhantomJSDriverService.CreateDefaultService();
driverService.HideCommandPromptWindow = true; // Disables verbose phantomjs output
IWebDriver driver = new PhantomJSDriver(driverService);
//IWebDriver driver = new FirefoxDriver(); // Debug with firefox.
Console.WriteLine("Logging into Office 365.");
driver.Navigate().GoToUrl("https://wegmans.sharepoint.com/resources/Pages/LaborPro.aspx");
if (driver.Title.ToString() == "Sign in to your account")
{
IWebElement loginentry = driver.FindElement(By.XPath("//*[@id='cred_userid_inputtext']"));
loginentry.SendKeys(Username);
IWebElement rememberme = driver.FindElement(By.XPath("//*[@id='cred_keep_me_signed_in_checkbox']"));
rememberme.Click();
}
WebDriverWait wait = new WebDriverWait(driver, TimeSpan.FromSeconds(30));
try { wait.Until((d) => { return (d.Title.ToString().Contains("Sign In") || d.Title.ToString().Contains("My Schedule")); }); } // Sometimes it skips the second login page.
catch (WebDriverTimeoutException)
{
driver.Quit();
throw new ScheduleGETException("Did not recieve an appropriate response from the Sharepoint server. The connection most likely timed out.");
}
Console.WriteLine("Logging into Sharepoint.");
if (driver.Title.ToString() == "Sign In")
{
try { wait.Until((d) => { return (d.FindElement(By.XPath("//*[@id='passwordInput']"))); }); }
catch (Exception)
{
driver.Quit();
throw new ScheduleGETException("Password input box did not load correctly.");
}
IWebElement passwordentry = driver.FindElement(By.XPath("//*[@id='passwordInput']"));
password = ConvertToUnsecureString(securePwd);
passwordentry.SendKeys(password);
ClearPassword();
passwordentry.Submit();
}
try { wait.Until((d) => { return (d.Title.ToString().Contains("Sign In") || d.Title.ToString().Contains("My Schedule")); }); } // Checks to see if the password was incorrect.
catch (WebDriverTimeoutException)
{
driver.Quit();
throw new ScheduleGETException("Did not recieve an appropriate response from the Sharepoint server. The connection most likely timed out.");
}
if (driver.Title.ToString() == "Sign In")
{
IWebElement error = driver.FindElement(By.XPath("//*[@id='error']"));
string errorString = error.Text.ToString();
if (errorString.Contains("Incorrect user ID or password"))
{
while (driver.Title.ToString() == "Sign In")
{
IWebElement usernameentry = driver.FindElement(By.XPath("//*[@id='userNameInput']"));
IWebElement passwordentry = driver.FindElement(By.XPath("//*[@id='passwordInput']"));
usernameentry.Clear();
passwordentry.Clear();
Console.WriteLine("You seem to have entered the wrong username or password.");
GetLoginCreds();
Console.WriteLine("Trying again...");
usernameentry.SendKeys(Username);
password = ConvertToUnsecureString(securePwd);
passwordentry.SendKeys(password);
ClearPassword();
passwordentry.Submit();
}
SaveLoginCreds();
}
else
{
Console.WriteLine("An unexpected error has occured with the webpage.");
Console.WriteLine(errorString);
driver.Quit();
throw new ScheduleGETException("An unexpected error has occured with the webpage.");
}
}
Console.WriteLine("Waiting for LaborPro...");
int retries = 2;
while (true) // Retry because this error can be solved by a simple page reload.
{
try { wait.Until((d) => { return (d.SwitchTo().Frame(0)); }); break; } // Waits for the inline frame to load.
catch (WebDriverTimeoutException)
{
Console.WriteLine("LaborPro link's inline frame was not generated properly.");
Console.WriteLine("Reloading the page...");
driver.Navigate().Refresh();
retries--;
if (retries <= 0)
{
driver.Quit();
throw new ScheduleGETException("LaborPro link's inline frame was not generated properly.");
}
}
}
string BaseWindow = driver.CurrentWindowHandle;
try { wait.Until((d) => { return (d.FindElement(By.XPath("/html/body/a"))); }); } // Waits until javascript generates the SSO link.
catch (Exception)
{
if (driver.Title.ToString().Contains("Sign In")) // We were redirected to the sign-in page once again, so let's fill it out again...
{
IWebElement usernameentry = driver.FindElement(By.XPath("//*[@id='userNameInput']"));
IWebElement passwordentry = driver.FindElement(By.XPath("//*[@id='passwordInput']"));
usernameentry.Clear();
passwordentry.Clear();
usernameentry.SendKeys(Username);
password = ConvertToUnsecureString(securePwd);
passwordentry.SendKeys(password);
ClearPassword();
passwordentry.Submit();
}
else
{
driver.Quit();
throw new ScheduleGETException("LaborPro SSO Link was not generated properly.");
}
}
IWebElement accessschedule = driver.FindElement(By.XPath("/html/body/a"));
accessschedule.Click();
string popupHandle = string.Empty;
ReadOnlyCollection<string> windowHandles = driver.WindowHandles;
foreach (string handle in windowHandles)
{
if (handle != driver.CurrentWindowHandle)
{
popupHandle = handle;
break;
}
}
driver.SwitchTo().Window(popupHandle);
Console.WriteLine("Accessing LaborPro.");
try { wait.Until((d) => { return (d.Title.ToString().Contains("Welcome")); }); }
catch (WebDriverTimeoutException)
{
throw new ScheduleGETException("Did not properly switch to LabroPro Window.");
}
Schedules.Add(driver.PageSource.ToString());
for (int i = 0; i < 2; i++) // Clicks "Next" and gets the schedules for the next two weeks.
{
driver.FindElement(By.XPath("//*[@id='pageBody']/form/table[2]/tbody/tr[2]/td/table/tbody/tr[2]/td/table/tbody/tr/td/div/table[1]/tbody/tr/td[1]/a[3]")).Click();
Schedules.Add(driver.PageSource.ToString());
}
ClearPassword(); // We don't need the password anymore, so let's not keep it laying around.
driver.Quit();
Console.WriteLine("Got your Schedule.");
}
public void ScrapeAdds(websites web, bool proxyCondtition, string country, string device, string resolution,string agent)
{
Stopwatch watch = new Stopwatch();
watch.Start();
updateUI(new System.Action(() => grid("Crawler","Starting crawler............",Color.Green)));
List<advertisement> adList = new List<advertisement>();
var service = PhantomJSDriverService.CreateDefaultService();
service.HideCommandPromptWindow = true;
var driver = new PhantomJSDriver(service);
updateUI(new System.Action(() => grid("Loading","loading website...............", Color.Green)));
if (proxyCondtition == true)
{
List<string> proxy = Proxy();
Random rand = new Random();
int index = rand.Next(proxy.Count);
string selectProxy = proxy[index];
string _proxy = selectProxy.Split('&')[0];
string _credentials = selectProxy.Split('&')[1];
Proxy prox = new Proxy();
prox.HttpProxy = string.Format(_proxy);
service.ProxyType = "http";
service.Proxy = prox.HttpProxy;
}
driver.Navigate().GoToUrl(web.website_url);
try
{
//List<string> resolution = ScreenResolution();
//Random ran = new Random();
//int index = ran.Next(resolution.Count);
//string ScrnRes = resolution[index];
string ScrnRes = resolution;
int x = int.Parse(ScrnRes.Split('x')[0]);
int y = int.Parse(ScrnRes.Split('x')[1]);
driver.Manage().Window.Size = new Size(x, y);
}
catch (Exception ex)
{ }
// driver.GetScreenshot().SaveAsFile("image2.png", ImageFormat.Png);
foreach (var networkName in web.website_networks)
{
ad_network adN = new ad_network();
//adN = adN.SearchByName(networkName);
adN = ANlist.Find(a => a.name.Equals(networkName));
if (adN != null)
{
updateUI(new System.Action(() => grid("Network","Network Found:"+ adN.name, Color.Green)));
string path1 = null;
string path2 = null;
string path3 = null;
string xpaths = adN.xpath;
string[] path = xpaths.Split('|');
if (path.Length == 3)
{
path1 = path[0];
path2 = path[1];
path3 = path[2];
}
if (path.Length == 2)
{
path1 = path[0];
path2 = path[1];
path3 = "";
}
NetworkFunctions nf = new NetworkFunctions(this);
if (adN.name == "Mgid")
{
updateUI(new System.Action(() => grid("Scraping:", "Migid Ads", Color.Green)));
nf.Mgid(web.website_url, driver, path1, path2, path3, adN, agent);
}
if (adN.name == "Adblade")
{
updateUI(new System.Action(() => grid("Scraping:", "Adblade Ads", Color.Green)));
nf.Adblade(web.website_url, driver, path1, path2, path3, adN, agent);
}
if (adN.name == "Gravity")
{
updateUI(new System.Action(() => grid("Scraping:", "Gravity Ads", Color.Green)));
nf.Gravity(web.website_url, driver, path1, path2, path3, adN, agent);
}
if (adN.name == "Taboola")
{
updateUI(new System.Action(() => grid("Scraping:", "Taboola Ads", Color.Green)));
nf.Taboola(web.website_url, driver, path1, path2, path3, adN, agent);
}
if (adN.name == "Outbrain")
{
updateUI(new System.Action(() => grid("Scraping:", "Outbrain Ads", Color.Green)));
nf.Outbrain(web.website_url, driver, path1, path2, path3, adN, agent);
}
if (adN.name == "Zergnet")
{
updateUI(new System.Action(() => grid("Scraping:", "Zergnet Ads", Color.Green)));
nf.ZergNetDotcom(web.website_url, driver, path1, path2, path3, adN, agent);
}
if (adN.name == "YahooGemini")
{
updateUI(new System.Action(() => grid("Scraping:", "YahooGemini Ads", Color.Green)));
nf.YahooGemini(web.website_url, driver, path1, path2, path3, adN, agent);
}
if (adN.name == "Hexagram")
{
updateUI(new System.Action(() => grid("Scraping:", "Hexagram Ads", Color.Green)));
nf.Hexagram(web.website_url, driver, path1, path2, path3, adN, agent);
}
if (adN.name == "Contentad" || adN.name == "Content.ad")
{
updateUI(new System.Action(() => grid("Scraping:", "Content.ad Ads", Color.Green)));
nf.ContentDotAd(web.website_url, driver, path1, path2, path3, adN, agent);
}
//if (adN.name == "Mgid")
//{
// ls = nf.Mgid(driver, path1, path2, path3);
//}
}
}
watch.Stop();
List<websites> list = new List<websites>();
list = web.SelectAll();
string ScrapingTime = watch.Elapsed.ToString();
foreach (var v in list)
{
if (v.website_url == web.website_url)
{
web.Update(v._id, ScrapingTime);
}
}
updateUI(new System.Action(() => grid("Time:", web.website_url+":" + ScrapingTime, Color.Green)));
driver.Quit();
driver.Dispose();
}