private void InitializeChrome() { if (Directory.Exists(_downloadDirectory)) { Directory.Delete(_downloadDirectory, true); } Directory.CreateDirectory(_downloadDirectory); ChromeOptions options = new ChromeOptions(); options.AddLocalStatePreference("profile.default_content_settings.popups", 0); options.AddLocalStatePreference("download.default_directory", _downloadDirectory); options.AddUserProfilePreference("profile.default_content_settings.popups", 0); options.AddUserProfilePreference("download.default_directory", _downloadDirectory); options.LeaveBrowserRunning = true; //silent start options.AddArguments("headless", "disable-gpu", "start-maximized", "window-size=1920,1080"); ChromeDriverService service = ChromeDriverService.CreateDefaultService(); service.HideCommandPromptWindow = true; //start chrome _driver = new ChromeDriver(options); var param = new Dictionary <string, object>(); param.Add("behavior", "allow"); param.Add("downloadPath", _downloadDirectory); _driver.ExecuteChromeCommand("Page.setDownloadBehavior", param); }
public CloudFlareServiceSelenium(IConfiguration configuration, ILogger <CloudFlareServiceSelenium> logger) { _logger = logger; var userAgent = RandomUserAgent.Generate(); _logger.LogInformation($"Initialize Chrome driver with userAgent: {userAgent}"); var chromeOptions = new ChromeOptions(); if (configuration.IsHeadless()) { chromeOptions.AddArguments("headless"); _logger.LogInformation("Enable Headless mode"); } chromeOptions.AddExcludedArgument("enable-automation"); chromeOptions.AddAdditionalCapability("useAutomationExtension", false); chromeOptions.AddArgument("--disable-blink-features=AutomationControlled"); chromeOptions.AddArgument("--lang=en-US"); _driver = new ChromeDriver(chromeOptions); _driver.ExecuteScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"); _driver.ExecuteChromeCommand("Network.setUserAgentOverride", new Dictionary <string, object> { { "userAgent", userAgent } }); _driver.Navigate().GoToUrl(CloudFlareLoginUrl); _waiter = new WebDriverWait(_driver, TimeSpan.FromMinutes(3)); }
public static void AddScriptToEvaluateOnNewDocument(this ChromeDriver webDriver, string script) { var parameters = new Dictionary <string, object> { ["source"] = script }; webDriver.ExecuteChromeCommand("Page.addScriptToEvaluateOnNewDocument", parameters); }
public static void EnableHeadlessDownload(this ChromeDriver chromeDriver, string downloadPath = null) { var parametros = new Dictionary <string, object> { { "behavior", "allow" }, { "downloadPath", downloadPath ?? AppDomain.CurrentDomain.BaseDirectory } }; chromeDriver.ExecuteChromeCommand("Page.setDownloadBehavior", parametros); }
public void Initialize() { scenarioName = featureName.CreateNode <Scenario>(_scenarioContext.ScenarioInfo.Title); var param = new Dictionary <string, object>(); ChromeOptions option = new ChromeOptions(); option.PageLoadStrategy = PageLoadStrategy.Normal; //headless = (GetValueFromConfig("headless") != "") ? bool.Parse(GetValueFromConfig("headless")) : true; var headlessConfig = GetValueFromAppSettings().Headless; headless = string.IsNullOrEmpty(headlessConfig)? false: Convert.ToBoolean(headlessConfig); var chromeDriverExePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); if (headless) { option.AddArguments("--headless"); // Add option to download file with headless mode string userName = System.Security.Principal.WindowsIdentity.GetCurrent().Name.Split('\\')[1]; string dir = "c:\\Users\\" + userName + "\\Downloads"; option.AddUserProfilePreference("download.prompt_for_download", "false"); option.AddUserProfilePreference("download.directory_upgrade", "true"); option.AddUserProfilePreference("download.prompt_for_download", "false"); option.AddUserProfilePreference("safebrowsing.enabled", "false"); option.AddUserProfilePreference("safebrowsing.disable_download_protection", "true"); option.AddArguments("--disable-web-security"); option.AddUserProfilePreference("download.default_directory", dir); param.Add("behavior", "allow"); param.Add("downloadPath", dir); ChromeDriver drv = new ChromeDriver(ChromeDriverService.CreateDefaultService(chromeDriverExePath), option, TimeSpan.FromMinutes(3)); drv.ExecuteChromeCommand("Page.setDownloadBehavior", param); _driver = drv; } else { _driver = new ChromeDriver(ChromeDriverService.CreateDefaultService(chromeDriverExePath), option, TimeSpan.FromMinutes(3)); } _driver.Manage().Window.Size = new Size(1920, 1080); _driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(60); _wait = new WebDriverWait(_driver, TimeSpan.FromSeconds(30)); _objectContainer.RegisterInstanceAs <IWebDriver>(_driver); _objectContainer.RegisterInstanceAs <WebDriverWait>(_wait); //_objectContainer.RegisterInstanceAs<IConfiguration>(config); }
static void Main(string[] args) { // ダウンロード場所 var downloadPath = @"D:\DL"; // chromedriver.exeの場所 var driverPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); // chromedriver.exe のログ出力設定 // 何かあった時に参照する // 上書きされるので、残す場合はファイル名にダイムスタンプをつけるとか var service = ChromeDriverService.CreateDefaultService(driverPath); service.LogPath = @"D:\logs\ChromeDriver.log"; service.EnableVerboseLogging = false; var options = new ChromeOptions(); // ヘッドレスモード options.AddArgument("--headless"); // 通常モード時のダウンロード場所指定 options.AddUserProfilePreference("download.default_directory", downloadPath); options.AddUserProfilePreference("download.prompt_for_download", "false"); options.AddUserProfilePreference("download.directory_upgrade", "true"); using (var webDriver = new ChromeDriver(service, options)) { try { // ヘッドレスモードモード時のダウンロード場所指定 var parameter = new Dictionary <string, object>(); parameter["behavior"] = "allow"; parameter["downloadPath"] = downloadPath; webDriver.ExecuteChromeCommand("Page.setDownloadBehavior", parameter); // CSVダウンロード操作 webDriver.Url = @"http://jusyo.jp/csv/new.php"; Debug.WriteLine("表示したよ"); var link = webDriver.FindElement(By.XPath("//a[contains(text(), 'csv_tohoku.zip')]")); link.Click(); Debug.WriteLine("ダウンロード"); } finally { // ダウンロード完了を待機。面倒なので 1秒待機 // WebDriverWait を利用したダウンロード完了待ちを作成する? // ダウンロード中とダウンロード完了を見分けれる? Thread.Sleep(1000); Debug.WriteLine("終了するよ"); webDriver.Quit(); } } }
/// <summary> /// ASCII /// </summary> /// <param name="value"></param> /// <returns></returns> public static void Save(ChromeDriver driver, string path, ScreenshotImageFormat type, bool isFull = false) { if (isFull) { var filePath = path; Dictionary <string, Object> metrics = new Dictionary <string, Object>(); metrics["width"] = driver.ExecuteScript("return Math.max(window.innerWidth,document.body.scrollWidth,document.documentElement.scrollWidth)"); metrics["height"] = driver.ExecuteScript("return Math.max(window.innerHeight,document.body.scrollHeight,document.documentElement.scrollHeight)"); //返回当前显示设备的物理像素分辨率与 CSS 像素分辨率的比率 metrics["deviceScaleFactor"] = driver.ExecuteScript("return window.devicePixelRatio"); metrics["mobile"] = driver.ExecuteScript("return typeof window.orientation !== 'undefined'"); driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics); driver.GetScreenshot().SaveAsFile(filePath, ScreenshotImageFormat.Png); } else { Screenshot shot = driver.TakeScreenshot(); shot.SaveAsFile(path, type); } }
// 사이트 캡쳐 private bool Capture(ChromeDriver driver, string bunzi, string ho) { try { Dictionary <string, Object> metrics = new Dictionary <string, Object>(); metrics["width"] = driver.ExecuteScript("return Math.max(window.innerWidth,document.body.scrollWidth,document.documentElement.scrollWidth)"); metrics["height"] = driver.ExecuteScript("return Math.max(window.innerHeight,document.body.scrollHeight,document.documentElement.scrollHeight)"); metrics["deviceScaleFactor"] = (double)driver.ExecuteScript("return window.devicePixelRatio"); metrics["mobile"] = driver.ExecuteScript("return typeof window.orientation !== 'undefined'"); driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics); string path_to_save_screenshot = SavePath + @"/" + bunzi + "-" + ho + ".png"; driver.GetScreenshot().SaveAsFile(path_to_save_screenshot, ScreenshotImageFormat.Png); return(true); } catch { return(false); } }
static void Page(byte number, string go_to_page, string wait_for_page)//download material from page { Console.WriteLine("\tPage" + number); RunJS(go_to_page); RunJS(wait_for_page); //wait for page ReadOnlyCollection <object> table_items = (ReadOnlyCollection <object>)RunJS("let ret=[],table=document.querySelector(\'#grdPost > tbody\').children;let l=table.length-2;for(let i=1;i<l;i++)ret.push(table[i].children[2].innerText);return ret"); //get all items on page byte i = 2, j, content_number; int hyphen; string subject; List <byte> haves; foreach (string table_value in table_items) { if (subjects.IsMatch(table_value)) //my subject { hyphen = table_value.IndexOf('-'); subject = table_value.Substring(hyphen + 2, table_value.LastIndexOf('-') - hyphen - 3); content_number = Convert.ToByte(table_value.Substring(8, hyphen - 9)); haves = memory[subject]; if (!haves.Contains(content_number)) //need material //download { RunJS("document.querySelector(\'#grdPost > tbody > tr:nth-child(" + i + ") > td:nth-child(5) > input[type=button]\').click()"); //click "Click" and go to download page j = 0; ReadOnlyCollection <object> files = (ReadOnlyCollection <object>)RunJS("let ret=[],table=document.querySelector(\'#grdDoc > tbody\').children;let l=table.length-1;for(let i=1;i<l;i++)ret.push(table[i].children[1].innerText);return ret;"); //get filename foreach (string filename in files) { disable_security["downloadPath"] = String.Format(download_dir, subject); //set download path browser.ExecuteChromeCommand("Page.setDownloadBehavior", disable_security); //disables Chrome security feature,copied from Stack Overflow RunJS("document.querySelector(\'#grdDoc_btnDownload_" + j + "\').click()"); //click "Click" Console.WriteLine("\t\t" + filename); j++; } haves.Add(content_number); //remember browser.Navigate().Back(); //move back RunJS(go_to_page); RunJS(wait_for_page); //counter to website bug } } i++; } }
/// <summary> /// Takes a screenshot of a web page. /// </summary> /// <param name="url">The URL of the web page to screenshoot.</param> /// <param name="filePath">The image file to save to.</param> /// <param name="width">The device width of the browser</param> /// <returns>The full file path of the saved image</returns> public NavigationTiming TakeScreenshot(string url, string filePath, int?width = null) { var outputDir = Path.GetDirectoryName(filePath); Directory.CreateDirectory(outputDir); _driver.Navigate().GoToUrl(url); ResizeWindow(); if (_options.HighlightBrokenLinks) { HighlightBrokenLinks(url); } // Sometimes, when this isn't high enough, the performance timings return incomplete data. int screenshotDelay = 3000; Thread.Sleep(screenshotDelay); var requestStats = GetRequestStats(); var screenshot = _driver.GetScreenshot(); screenshot.SaveAsFile(filePath, _imageFormat); ClearResize(); return(requestStats); // LOCAL FUNCTIONS NavigationTiming GetRequestStats() { try { var stats = (string)_driver.ExecuteScript( @"return (window && window.performance && JSON.stringify([...window.performance.getEntriesByType('navigation'),{ }][0])) || '{}'"); var deprecatedTiming = (string)_driver.ExecuteScript( @"return (window && window.performance && window.performance.timing && JSON.stringify(window.performance.timing)) || '{}'"); Logger.Default.Log($"Stats for {url}..."); Logger.Default.Log(stats); Logger.Default.Log(deprecatedTiming); string ConvertToInt(Match m) { var origValue = m.Value; if (!double.TryParse(origValue, out var dblVal)) { return(origValue); } var intValue = (int)Math.Round(dblVal); return(intValue.ToString()); } stats = Regex.Replace(stats, @"\d+\.\d+", ConvertToInt); Logger.Default.Log("stats converted to int: " + stats); return(JsonConvert.DeserializeObject <NavigationTiming>((string)stats)); } catch (Exception ex) { Debug.WriteLine("Error parsing request timings: " + ex.Message); return(new NavigationTiming()); } } /// <summary> /// Repeatedly resize the window to account for lazily loaded elements. /// </summary> /// <param name="width"></param> void ResizeWindow() { // Adapted from https://stackoverflow.com/a/56535317 int numTries = 0; const int maxTries = 8; int prevHeight; int calculatedHeight = -1; string autoWidthCommand = @"return Math.max( window.innerWidth, document.body.scrollWidth, document.documentElement.scrollWidth)"; // Repeatedly resize height to allow new elements to (lazily) load. do { prevHeight = calculatedHeight; var calculatedWidth = width.HasValue ? $"return {width}" : autoWidthCommand; calculatedHeight = CalculateDocHeight(); // TODO: Set device-specific user agents. Dictionary <string, object> metrics = new Dictionary <string, object> { ["width"] = _driver.ExecuteScript(calculatedWidth), ["height"] = calculatedHeight, ["deviceScaleFactor"] = ScaleFactor(false), ["mobile"] = _driver.ExecuteScript("return typeof window.orientation !== 'undefined'") }; _driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics); } while (calculatedHeight != prevHeight && ++numTries < maxTries); // LOCAL FUNCTIONS int CalculateDocHeight() { // Sometimes a long, sometimes double, etc object jsHeight = _driver.ExecuteScript( @"return Math.max( document.body.scrollHeight, document.body.offsetHeight, document.documentElement.clientHeight, document.documentElement.scrollHeight, document.documentElement.offsetHeight, document.documentElement.getBoundingClientRect().height)"); double numericHeight = Convert.ToDouble(jsHeight); return((int)Math.Ceiling(numericHeight)); } // False for a 1:1 pixel ratio with the image // True for an easier-to-read image on the current monitor. double ScaleFactor(bool shouldScaleImage) => shouldScaleImage ? (double)_driver.ExecuteScript("return window.devicePixelRatio") : 1.0; } void ClearResize() { try { _driver.ExecuteChromeCommand("Emulation.clearDeviceMetricsOverride", new Dictionary <string, object>()); } catch (Exception ex) { // Usually thrown when the driver or browser window is closed. if (!(ex is WebDriverException || ex is NoSuchWindowException)) { throw; } } } void HighlightBrokenLinks(string rawCallingUri) { var standardizedUri = new Uri(rawCallingUri); standardizedUri = standardizedUri.TryStandardize(); if (_brokenLinks?.Any() != true) { return; } var selectors = _brokenLinks .SelectMany(l => l.Value.Sources) .Where(x => standardizedUri.Equals(x.CallingPage)) .Select(x => $@"a[href='{x.Href}']"); var combinedSelector = string.Join(",", selectors); var styles = $@" {combinedSelector} {{ border: 3px dashed red; }}"; var script = $@" var style = document.createElement('style'); style.type = 'text/css'; style.innerHTML = `{styles}`; document.getElementsByTagName('head')[0].appendChild(style);"; _driver.ExecuteScript(script); } }
public void DownloadDataFile() { //easy to find Windows Documents path, a bit more work for Downloads (you'd think it would be easy eh?) //just find Documents, then replace with Downloads and concatenate csv filename string downloadsPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments).Replace("Documents", "Downloads"); string covidDataFilePath = Path.Combine(downloadsPath, QuebecPublicHealthCovidCasesWebPage.DownloadedCsvDataFilename); string renamedCovidDataFilename = Path.Combine(downloadsPath, "CovidDataFile." + DateTime.Now.ToString("yyyyMMdd-HHmm") + ".csv"); var chromeOptions = new ChromeOptions(); chromeOptions.AddArguments(new List <string>() { "headless" }); chromeOptions.AddArguments("--window-size=1024,1080"); //these are required to download files when using headless, this will behave more like a function call //if you want to see what the browser\Selenium is doing, you can comment the headless option part above chromeOptions.AddArguments(new List <string>() { "download.default_directory", downloadsPath }); chromeOptions.AddArguments("--disable-web-security"); chromeOptions.AddArguments("--allow-running-insecure-content"); //using this causes a problem with downloading the file, I'm not sure where it goes when Selenium is headless, //you might be able to explicitly set the download folder in ChromeOptions and then pick it up down below in the same path, //but I don't know where you will run this from, so I can't do this right now ChromeDriver driver = new ChromeDriver(chromeOptions); var settingForHeadlessDownloads = new Dictionary <string, object> { { "behavior", "allow" }, { "downloadPath", downloadsPath } }; driver.ExecuteChromeCommand("Page.setDownloadBehavior", settingForHeadlessDownloads); try { //goto the main page driver.Navigate().GoToUrl(QuebecPublicHealthCovidCasesWebPage.WebPageUrl); //scroll to the confirmed cases chart var confirmedCasesGraphElement = driver.FindElement(By.Id(QuebecPublicHealthCovidCasesWebPage.ConfirmedCasesGraphId)); ((IJavaScriptExecutor)driver).ExecuteScript("arguments[0].scrollIntoView(true);", confirmedCasesGraphElement); //finding out when the data is finished dynamically loading new WebDriverWait(driver, TimeSpan.FromSeconds(30)).Until(ExpectedConditions.ElementExists(By.XPath("//div[@id='evolutionHospitalisations']//*[name()='svg']"))); //click on the ellipsed to see the download csv option and click it IWebElement confirmedCasesGraphEllipse = driver.FindElement(By.XPath(QuebecPublicHealthCovidCasesWebPage.ConfirmedCasesGraphEllipseLocator)); confirmedCasesGraphEllipse.Click(); var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(30)).Until(ExpectedConditions.ElementExists(By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile))); //IWebElement downloadCsvFile = WebDriverExtensions.FindElement(driver, By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile), 30); IWebElement downloadCsvFile = wait.FindElement(By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile)); downloadCsvFile.Click(); //before closing down the browser, lets give the file a bit of time to complete bool fileFound = DownloadDataFile(covidDataFilePath); ReportDataDownladStatus(covidDataFilePath, renamedCovidDataFilename, fileFound); } finally { Console.WriteLine("Just a sec...closing Selenium Chrome web driver is slow, but it should close everything"); driver.Close(); driver.Dispose(); driver.Quit(); } }