public void SaveThumbnailToDisk(string url, string filename) { var options = new EdgeOptions(); options.AddArgument("headless");//Comment if we want to see the window. string path = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); EdgeDriver driver = null; try { driver = new EdgeDriver(path, options); driver.Manage().Window.Size = new System.Drawing.Size(2000, 4000); driver.Navigate().GoToUrl(url); Thread.Sleep(5000); var screenshot = (driver as ITakesScreenshot).GetScreenshot(); Log.Information("Saving thumbnail file '{filename}'", filename); screenshot.SaveAsFile(filename); } catch (Exception ex) { Log.Error(ex, "ERROR: Unable to save webpage thumbnail"); } finally { if (driver != null) { driver.Close(); driver.Quit(); } } }
private RemoteWebDriver Initialize(bool useFirefox = false, bool isMobile = false) { RemoteWebDriver driver; if (useFirefox) { var firefoxOptions = new FirefoxOptions(); firefoxOptions.SetPreference(Constants.PrivateBrowsingKey, true); if (isMobile) { firefoxOptions.SetPreference(Constants.UserAgentKey, Constants.MobileUserAgent); } driver = new FirefoxDriver(firefoxOptions); } else { EdgeOptions driverOptions = new EdgeOptions { UseChromium = true }; if (isMobile) { driverOptions.AddArgument($"{ Constants.EdgeUserAgentArgument}={Constants.MobileUserAgent}"); } driver = new EdgeDriver(driverOptions); driver.Manage().Cookies.DeleteAllCookies(); } driver.Manage().Window.Maximize(); return(driver); }
public override IWebDriver GetDriver() { var options = new EdgeOptions(); options.UseChromium = true; options.AddArgument("ignore-certificate-errors"); var service = EdgeDriverService.CreateChromiumService(DriversPath); return(new OpenQA.Selenium.Edge.EdgeDriver(service, options, TimeSpan.FromMinutes(3))); }
public EdgeOptions ToEdgeOptions(EdgeBrowserSettings settings) { var result = new EdgeOptions(); if (settings.Options.UseInPrivateBrowsing) { result.AddArgument("inprivate"); } return(result); }
public void CreateWebDriver(ScenarioContext context) { var currentDirectory = System.IO.Directory.GetCurrentDirectory(); System.Diagnostics.Debug.WriteLine($"AdminAssistant.Blazor.Test - The Directory where your webdriver exe should be placed is: {currentDirectory}"); // Launch Microsoft Edge (Chromium) - See https://github.com/microsoft/edge-selenium-tools // Requires the WebDriver for your version of Edge (chromium) - see https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/#downloads var options = new EdgeOptions(); options.UseChromium = true; options.AddArgument("--start-maximized"); options.AddArgument("--disable-notifications"); IWebDriver webDriver = new EdgeDriver(options); webDriver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(5); context["WEB_DRIVER"] = webDriver; }
public static EdgeOptions EdgeOptions() { var options = new EdgeOptions(); options.AddArgument("start-maximized"); options.UseChromium = true; options.PageLoadStrategy = PageLoadStrategy.Normal; options.UseInPrivateBrowsing = true; return(options); }
private IWebDriver SetupEdgeDriver() { var ed = new EdgeOptions(); ed.AddArgument("headless"); #if DEBUG new DriverManager().SetUpDriver(new EdgeConfig()); return(new EdgeDriver(ed)); #else var envEdgeWebDriver = Environment.GetEnvironmentVariable("EdgeWebDriver"); return(new EdgeDriver(envEdgeWebDriver, ed)); #endif }
public static EdgeOptions GetEdgeOptions(string lang, bool headless = false, PlatformType platformType = PlatformType.Any) { EdgeOptions options = new EdgeOptions(); options.UseChromium = true; options.AddArguments("disable-gpu", $"--lang={lang}"); if (headless) { options.AddArgument("headless"); } SetPlatform(options, platformType); return(options); }
private static IWebDriver CreateDriver() { try { var programDirectory = System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); var edgeService = EdgeDriverService.CreateChromiumService($"{programDirectory}\\Drivers", $"{DriverName}.exe"); edgeService.SuppressInitialDiagnosticInformation = true; edgeService.HideCommandPromptWindow = true; var options = new EdgeOptions { UseChromium = true }; options.AddArgument("headless"); options.AddArgument("disable-gpu"); return(new EdgeDriver(edgeService, options)); } catch (InvalidOperationException) { Console.WriteLine("Unable to download due to MicrosoftEdge not up to date. Please update it before proceeding."); throw; } }
public void Test() { ChromeOptions chromeOptions = new ChromeOptions(); chromeOptions.AddArgument("--headless"); EdgeOptions edgeOptions = new EdgeOptions(); edgeOptions.AddArgument("--headless"); //driver = new ChromeDriver(chromeOptions); driver = new EdgeDriver(edgeOptions); googleMaps = new GoogleMapsPageObjects(driver); googleMaps.GoToPage(); googleMaps.ClickModal(); googleMaps.Route.Click(); }
public static RemoteWebDriver Create(BrowserType driverType, bool headless = false) { var headlessArg = "--headless"; RemoteWebDriver driver; switch (driverType) { case BrowserType.Chrome: var chromeOption = new ChromeOptions(); if (headless) { chromeOption.AddArgument(headlessArg); } driver = new ChromeDriver(chromeOption); break; case BrowserType.Firefox: var firefoxOption = new FirefoxOptions(); if (headless) { firefoxOption.AddArgument(headlessArg); } driver = new FirefoxDriver(firefoxOption); break; case BrowserType.Edge: var edgeOption = new EdgeOptions(); if (headless) { edgeOption.UseChromium = true; edgeOption.AddArgument(headlessArg); } driver = new EdgeDriver(edgeOption); break; default: throw new ArgumentException($"Driver type not supported: {driverType}"); } driver.Manage().Window.Maximize(); return(driver); }
public (string, Uri) WebDriverUrlToDisk(string url, string urlHash, string filename) { Log.Debug("Loading Selenium URL '{urlHash}':'{url}'", urlHash, url); var options = new EdgeOptions(); options.AddArgument("headless");//Comment if we want to see the window. string path = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); EdgeDriver driver = null; try { driver = new EdgeDriver(path, options); driver.Navigate().GoToUrl(url); // Delete the file if it already exists if (File.Exists(filename)) { File.Delete(filename); } Log.Debug("Saving text file '{fileName}'", filename); File.WriteAllText(filename, driver.PageSource); return(filename, new Uri(driver.Url)); } catch (Exception ex) { Log.Error(ex, "WebDriverUrlToDisk: Unexpected error '{message}'", ex.Message); } finally { if (driver != null) { driver.Close(); driver.Quit(); } } return(string.Empty, new Uri(url)); }
private static EdgeDriver EdgeDriver(Device device) { EdgeOptions options = new EdgeOptions { UseChromium = true }; EdgeDriver _driver = null; switch (device) { case Device.Desktop: options.AddArgument("--start-maximized"); _driver = new EdgeDriver(driversPath, options); break; case Device.Tablet: options.EnableMobileEmulation("iPad"); _driver = new EdgeDriver(driversPath, options); break; } return(_driver); }
private static EdgeDriver EdgeDriver(Device device) { EdgeDriverService service = EdgeDriverService.CreateDefaultService(driversPath, "msedgedriver.exe"); EdgeOptions options = new EdgeOptions(); EdgeDriver _driver = null; switch (device) { case Device.Desktop: _driver = new EdgeDriver(service); _driver.Manage().Window.Size = new Size(1200, 700); break; case Device.Tablet: options.UseChromium = true; options.EnableMobileEmulation("iPad"); options.AddArgument("--window-size=768,700"); _driver = new EdgeDriver(driversPath, options); break; } return(_driver); }
/// <summary> /// Set the WebDriver browser by browser name. /// </summary> /// <param name="browserName">Options not case sensitive and are: "chrome", "firefox", "chromeheadless", "firefoxheadless", "ie"</param> /// <exception cref="ArgumentException"></exception> /// <returns>Returns a the appropriate WebDriver by browser name</returns> private IWebDriver SetBrowser(string browserName) { if (IsRemote) { switch (browserName.ToLower()) { case "firefox": var rfireoptions = AcceptUknownIssuer(); return(new RemoteWebDriver(RemoteHubUri, rfireoptions.ToCapabilities(), GlobalWaitTime)); case "chrome": var rcoptions = new ChromeOptions(); return(new RemoteWebDriver(RemoteHubUri, rcoptions.ToCapabilities(), GlobalWaitTime)); case "edge": var edgeHeadlessOptions = new EdgeOptions(); edgeHeadlessOptions.UseChromium = true; return(new RemoteWebDriver(RemoteHubUri, edgeHeadlessOptions.ToCapabilities(), GlobalWaitTime)); default: throw new ArgumentException("Provide a valid browser name."); } } else { switch (browserName.ToLower()) { case "firefox": FirefoxDriverService fireservice = FirefoxDriverService.CreateDefaultService(BaseApplicationPath); var fireoptions = AcceptUknownIssuer(); return(new FirefoxDriver(fireservice, fireoptions, GlobalWaitTime)); case "chrome": var coptions = new ChromeOptions(); return(new ChromeDriver(BaseApplicationPath, coptions, GlobalWaitTime)); case "firefoxheadless": FirefoxDriverService firehservice = FirefoxDriverService.CreateDefaultService(BaseApplicationPath); var firehoptions = AcceptUknownIssuer(); firehoptions.AddArgument("--headless"); return(new FirefoxDriver(firehservice, firehoptions, GlobalWaitTime)); case "chromeheadless": var choptions = new ChromeOptions(); choptions.AddArgument("headless"); choptions.AddArgument("window-size=1200x800"); return(new ChromeDriver(BaseApplicationPath, choptions, GlobalWaitTime)); case "edge": var edgeOptions = new EdgeOptions(); edgeOptions.UseChromium = true; return(new EdgeDriver(BaseApplicationPath, edgeOptions, GlobalWaitTime)); case "edgeheadless": var edgeHeadlessOptions = new EdgeOptions(); edgeHeadlessOptions.UseChromium = true; edgeHeadlessOptions.AddArgument("--headless --disable-gpu"); return(new EdgeDriver(BaseApplicationPath, edgeHeadlessOptions, GlobalWaitTime)); default: throw new ArgumentException("Provide a valid browser name."); } } }
public static List <string[]> GetRightStufAnimeData(string bookTitle, char bookType, bool memberStatus, byte currPageNum) { // Initialize the html doc for crawling HtmlDocument doc = new HtmlDocument(); EdgeOptions edgeOptions = new EdgeOptions(); edgeOptions.UseChromium = true; edgeOptions.PageLoadStrategy = PageLoadStrategy.Eager; edgeOptions.AddArgument("headless"); edgeOptions.AddArgument("disable-gpu"); edgeOptions.AddArgument("disable-extensions"); edgeOptions.AddArgument("inprivate"); EdgeDriver edgeDriver = new EdgeDriver(edgeOptions); edgeDriver.Navigate().GoToUrl(getUrl(bookType, currPageNum, bookTitle)); Thread.Sleep(3500); doc.LoadHtml(edgeDriver.PageSource); // Get the page data from the HTML doc HtmlNodeCollection titleData = doc.DocumentNode.SelectNodes("//span[@itemprop='name']"); HtmlNodeCollection priceData = doc.DocumentNode.SelectNodes("//span[@itemprop='price']"); HtmlNodeCollection stockStatusData = doc.DocumentNode.SelectNodes("//div[@class='product-line-stock-container '] | //span[@class='product-line-stock-msg-out-text']"); HtmlNode pageCheck = doc.DocumentNode.SelectSingleNode("//li[@class='global-views-pagination-next']"); try{ double GotAnimeDiscount = 0.05; decimal priceVal; string priceTxt, stockStatus, currTitle; Regex removeWords = new Regex(@"[^a-z']"); for (int x = 0; x < titleData.Count; x++) { currTitle = titleData[x].InnerText; if (removeWords.Replace(currTitle.ToLower(), "").IndexOf(removeWords.Replace(bookTitle.ToLower(), "")) == 0) { priceVal = System.Convert.ToDecimal(priceData[x].InnerText.Substring(1)); priceTxt = memberStatus ? "$" + (priceVal - (priceVal * (decimal)GotAnimeDiscount)).ToString("0.00") : priceData[x].InnerText; stockStatus = stockStatusData[x].InnerText; if (stockStatus.IndexOf("In Stock") != -1) { stockStatus = "IS"; } else if (stockStatus.IndexOf("Out of Stock") != -1) { stockStatus = "OOS"; } else if (stockStatus.IndexOf("Pre-Order") != -1) { stockStatus = "PO"; } else { stockStatus = "OOP"; } dataList.Add(new string[] { currTitle, priceTxt.Trim(), stockStatus, "RightStufAnime" }); //Console.WriteLine(currTitle + " " + priceTxt + " " + stockStatus); } } if (pageCheck != null) { currPageNum++; GetRightStufAnimeData(bookTitle, bookType, memberStatus, currPageNum); } else { edgeDriver.Quit(); foreach (string link in links) { Console.WriteLine(link); } } } catch (NullReferenceException ex) { Console.Error.WriteLine(ex); Environment.Exit(1); } using (StreamWriter outputFile = new StreamWriter(@"C:\MangaWebScrape\MangaWebScrape\Data_Files\RightStufAnimeData.txt")) { foreach (string[] data in dataList) { outputFile.WriteLine(data[0] + " " + data[1] + " " + data[2] + " " + data[3]); } } return(dataList); }
public static List <string[]> GetBarnesAndNobleData(string bookTitle, char bookType, byte currPageNum) { // Initialize the html doc for crawling HtmlDocument doc = new HtmlDocument(); EdgeOptions edgeOptions = new EdgeOptions(); edgeOptions.UseChromium = true; edgeOptions.PageLoadStrategy = PageLoadStrategy.Eager; edgeOptions.AddArgument("headless"); edgeOptions.AddArgument("disable-gpu"); edgeOptions.AddArgument("disable-extensions"); edgeOptions.AddArgument("inprivate"); EdgeDriver edgeDriver = new EdgeDriver(edgeOptions); edgeDriver.Navigate().GoToUrl(GetUrl(bookTitle, currPageNum, bookType)); Thread.Sleep(2000); doc.LoadHtml(edgeDriver.PageSource); HtmlNodeCollection titleData = doc.DocumentNode.SelectNodes("//a[@class=' ']"); Console.WriteLine(); HtmlNodeCollection priceData = doc.DocumentNode.SelectNodes("//a[@class=' link']//span[last()]"); HtmlNodeCollection stockStatusData = doc.DocumentNode.SelectNodes("//div[1][@class='availability-spacing flex']//p"); HtmlNode pageCheck = doc.DocumentNode.SelectSingleNode("//li[@class='pagination__next ']"); if (bookType == 'N') { HtmlNodeCollection formatTypeData = doc.DocumentNode.SelectNodes("//span[@class='format']"); for (int x = 0; x < formatTypeData.Count; x++) { if (formatTypeData[x].InnerText.IndexOf("NOOK") != -1) { titleData.RemoveAt(x); formatTypeData.RemoveAt(x); x--; } } formatTypeData = null; //Free the format type list from memory } try{ string stockStatus, currTitle; Regex removeExtra = new Regex(@"[^a-z']"); for (int x = 0; x < titleData.Count; x++) { currTitle = titleData[x].InnerText; if (removeExtra.Replace(currTitle.ToLower(), "").IndexOf(removeExtra.Replace(bookTitle.ToLower(), "")) == 0) { stockStatus = stockStatusData[x].InnerText; if (stockStatus.IndexOf("Available Online") != -1) { stockStatus = "IS"; } else if (stockStatus.IndexOf("Out of Stock Online") != -1) { stockStatus = "OOS"; } else if (stockStatus.IndexOf("Pre-order Now") != -1) { stockStatus = "PO"; } dataList.Add(new string[] { currTitle, priceData[x].InnerText.Trim(), stockStatus, "Barnes & Noble" }); } } if (pageCheck != null) { currPageNum++; GetBarnesAndNobleData(bookTitle, bookType, currPageNum); } else { edgeDriver.Quit(); foreach (string link in links) { Console.WriteLine(link); } } } catch (NullReferenceException ex) { Console.Error.WriteLine(ex); Environment.Exit(1); } using (StreamWriter outputFile = new StreamWriter(@"C:\MangaWebScrape\MangaWebScrape\Data_Files\BarnesAndNobleData.txt")) { foreach (string[] data in dataList) { outputFile.WriteLine(data[0] + " " + data[1] + " " + data[2] + " " + data[3]); } } return(dataList); }
private async Task BingSearchAsync(RewardType rewardType, int current, int target, bool useFirefox = false) { try { wordList = await DownloadJsonDataAsync <List <string> >(Constants.WordsListUrl); //Edge browser if (rewardType == RewardType.EdgeBonus || !useFirefox) { var options = new EdgeOptions { UseChromium = true, }; if (rewardType == RewardType.Mobile) { options.AddArgument($"{ Constants.EdgeUserAgentArgument}={Constants.MobileUserAgent}"); } using (var edgeDriver = new EdgeDriver(options)) { var edgeWait = new WebDriverWait(edgeDriver, TimeSpan.FromSeconds(30)); await LoginAsync(edgeDriver, edgeWait); Search(edgeDriver, edgeWait, Constants.BingSearchURL + "Give me Edge points"); await Task.Delay(2000); try { if (rewardType == RewardType.Mobile) { var hamburg = edgeWait.Until(d => d.FindElement(By.Id(Constants.MHamburger))); hamburg?.Click(); var signin = edgeWait.Until(d => d.FindElement(By.Id(Constants.HbS))); signin?.Click(); await Task.Delay(1500); } else { try { var id_p = edgeWait.Until(d => d.FindElement(By.Id(Constants.ID_P))); if (id_p != null) { id_p?.Click(); } } catch (Exception ex) { Debug.WriteLine(ex.Message); Debug.WriteLine(ex.StackTrace); var id_a = edgeWait.Until(d => d.FindElement(By.Id(Constants.ID_A))); if (id_a != null) { id_a?.Click(); } } } } catch (Exception ex) { Debug.WriteLine(ex.Message); Debug.WriteLine(ex.StackTrace); } while (current < target) { var nextInt = rand.Next(wordList.Count + 1); Search(edgeDriver, edgeWait, Constants.BingSearchURL + wordList[nextInt <= wordList.Count ? nextInt : 0]); current += 5; if (current >= target) { var currentBreakDown = CheckBreakDown(edgeDriver, edgeWait); if (currentBreakDown.ContainsKey(rewardType)) { current = currentBreakDown[rewardType].Current; Console.WriteLine("{0} points of {1} completed", currentBreakDown[rewardType].Current, currentBreakDown[rewardType].Maximum); } } } edgeDriver?.Quit(); } } //Use Firefox else { var options = new FirefoxOptions(); TimeSpan timeout = TimeSpan.FromSeconds(60); if (rewardType == RewardType.Mobile) { options.SetPreference(Constants.UserAgentKey, Constants.MobileUserAgent); } options.SetPreference(Constants.PrivateBrowsingKey, true); using (var firefoxDriver = new FirefoxDriver(options)) { WebDriverWait driverWait = new WebDriverWait(firefoxDriver, timeout); await LoginAsync(firefoxDriver, driverWait); firefoxDriver.Navigate().GoToUrl(Constants.BingSearchURL + "Edge Points"); await Task.Delay(2000); try { if (rewardType == RewardType.Mobile) { var hamburg = driverWait.Until(d => d.FindElement(By.Id(Constants.MHamburger))); hamburg?.Click(); var signin = driverWait.Until(d => d.FindElement(By.Id(Constants.HbS))); signin?.Click(); await Task.Delay(1500); } else { try { var id_p = driverWait.Until(d => d.FindElement(By.Id(Constants.ID_P))); if (id_p != null) { id_p.Click(); } } catch (Exception ex) { Debug.WriteLine(ex.Message); Debug.WriteLine(ex.StackTrace); var id_a = driverWait.Until(d => d.FindElement(By.Id(Constants.ID_A))); if (id_a != null) { id_a.Click(); } } } } catch (Exception ex) { Debug.WriteLine(ex.Message); Debug.WriteLine(ex.StackTrace); } while (current < target) { var nextInt = rand.Next(wordList.Count + 1); Search(firefoxDriver, driverWait, Constants.BingSearchURL + wordList[nextInt <= wordList.Count ? nextInt : 0]); current += 5; if (current >= target) { var currentBreakDown = CheckBreakDown(firefoxDriver, driverWait); if (currentBreakDown.ContainsKey(rewardType)) { current = currentBreakDown[rewardType].Current; Console.WriteLine($"{currentBreakDown[rewardType].Current} points of {currentBreakDown[rewardType].Maximum} completed"); } } } firefoxDriver?.Quit(); } } } catch (Exception ex) { Debug.WriteLine(ex.Message); Debug.WriteLine(ex.StackTrace); } }
public static List <string[]> GetRobertsAnimeCornerStoreData(string bookTitle, char bookType) { EdgeOptions edgeOptions = new EdgeOptions(); edgeOptions.UseChromium = true; edgeOptions.PageLoadStrategy = PageLoadStrategy.Eager; edgeOptions.AddArgument("headless"); edgeOptions.AddArgument("disable-gpu"); edgeOptions.AddArgument("disable-extensions"); edgeOptions.AddArgument("inprivate"); EdgeDriver edgeDriver = new EdgeDriver(edgeOptions); // Initialize the html doc for crawling HtmlDocument doc = new HtmlDocument(); string linkPage = getPageData(edgeDriver, bookTitle, bookType, doc); if (linkPage == null) { Console.Error.WriteLine("Error! Invalid Series Title"); Environment.Exit(1); } else if (linkPage.Equals("DNE")) { Console.Error.WriteLine(bookTitle + " does not exist at this website"); edgeDriver.Quit(); } else { try{ // Start scraping the URL where the data is found edgeDriver.Navigate().GoToUrl(linkPage); Thread.Sleep(2000); // Get the html doc for crawling doc.LoadHtml(edgeDriver.PageSource); List <HtmlNode> titleData = doc.DocumentNode.SelectNodes("//font[@face='dom bold, arial, helvetica']//b").Where(title => title.InnerText.ToLower().IndexOf(bookTitle.ToLower()) != -1).ToList(); List <HtmlNode> priceData = doc.DocumentNode.SelectNodes("//font[@color='#ffcc33']").Where(price => price.InnerText.IndexOf("$") != -1).ToList(); string currTitle; Regex pattern = new Regex(@"#[\d]+( )"); for (int x = 0; x < titleData.Count; x++) { currTitle = titleData[x].InnerText.Replace(",", ""); currTitle = currTitle.Substring(0, pattern.Match(currTitle).Groups[1].Index); dataList.Add(new string[] { currTitle, priceData[x].InnerText.Trim(), currTitle.IndexOf("Pre Order") != -1 ? "PO" : "IS", "RobertsAnimeCornerStore" }); } foreach (string link in links) { Console.WriteLine(link); } edgeDriver.Quit(); } catch (NullReferenceException ex) { Console.Error.WriteLine(ex); Environment.Exit(1); } } using (StreamWriter outputFile = new StreamWriter(@"C:\MangaWebScrape\MangaWebScrape\Data_Files\RobertsAnimeCornerStoreData.txt")) { foreach (string[] data in dataList) { outputFile.WriteLine(data[0] + " " + data[1] + " " + data[2] + " " + data[3]); } } return(dataList); }