private void InitializeChrome()
        {
            if (Directory.Exists(_downloadDirectory))
            {
                Directory.Delete(_downloadDirectory, true);
            }
            Directory.CreateDirectory(_downloadDirectory);

            ChromeOptions options = new ChromeOptions();

            options.AddLocalStatePreference("profile.default_content_settings.popups", 0);
            options.AddLocalStatePreference("download.default_directory", _downloadDirectory);
            options.AddUserProfilePreference("profile.default_content_settings.popups", 0);
            options.AddUserProfilePreference("download.default_directory", _downloadDirectory);
            options.LeaveBrowserRunning = true;

            //silent start
            options.AddArguments("headless", "disable-gpu", "start-maximized", "window-size=1920,1080");
            ChromeDriverService service = ChromeDriverService.CreateDefaultService();

            service.HideCommandPromptWindow = true;
            //start chrome
            _driver = new ChromeDriver(options);
            var param = new Dictionary <string, object>();

            param.Add("behavior", "allow");
            param.Add("downloadPath", _downloadDirectory);
            _driver.ExecuteChromeCommand("Page.setDownloadBehavior", param);
        }
Пример #2
0
        public CloudFlareServiceSelenium(IConfiguration configuration, ILogger <CloudFlareServiceSelenium> logger)
        {
            _logger = logger;
            var userAgent = RandomUserAgent.Generate();

            _logger.LogInformation($"Initialize Chrome driver with userAgent: {userAgent}");
            var chromeOptions = new ChromeOptions();

            if (configuration.IsHeadless())
            {
                chromeOptions.AddArguments("headless");
                _logger.LogInformation("Enable Headless mode");
            }

            chromeOptions.AddExcludedArgument("enable-automation");
            chromeOptions.AddAdditionalCapability("useAutomationExtension", false);
            chromeOptions.AddArgument("--disable-blink-features=AutomationControlled");
            chromeOptions.AddArgument("--lang=en-US");
            _driver = new ChromeDriver(chromeOptions);
            _driver.ExecuteScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})");
            _driver.ExecuteChromeCommand("Network.setUserAgentOverride", new Dictionary <string, object>
            {
                { "userAgent", userAgent }
            });
            _driver.Navigate().GoToUrl(CloudFlareLoginUrl);
            _waiter = new WebDriverWait(_driver, TimeSpan.FromMinutes(3));
        }
Пример #3
0
        public static void AddScriptToEvaluateOnNewDocument(this ChromeDriver webDriver, string script)
        {
            var parameters = new Dictionary <string, object> {
                ["source"] = script
            };

            webDriver.ExecuteChromeCommand("Page.addScriptToEvaluateOnNewDocument", parameters);
        }
Пример #4
0
        public static void EnableHeadlessDownload(this ChromeDriver chromeDriver, string downloadPath = null)
        {
            var parametros = new Dictionary <string, object>
            {
                { "behavior", "allow" },
                { "downloadPath", downloadPath ?? AppDomain.CurrentDomain.BaseDirectory }
            };

            chromeDriver.ExecuteChromeCommand("Page.setDownloadBehavior", parametros);
        }
Пример #5
0
        public void Initialize()
        {
            scenarioName = featureName.CreateNode <Scenario>(_scenarioContext.ScenarioInfo.Title);
            var param = new Dictionary <string, object>();

            ChromeOptions option = new ChromeOptions();

            option.PageLoadStrategy = PageLoadStrategy.Normal;

            //headless = (GetValueFromConfig("headless") != "") ? bool.Parse(GetValueFromConfig("headless")) : true;
            var headlessConfig = GetValueFromAppSettings().Headless;

            headless = string.IsNullOrEmpty(headlessConfig)? false: Convert.ToBoolean(headlessConfig);
            var chromeDriverExePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);

            if (headless)
            {
                option.AddArguments("--headless");

                // Add option to download file with headless mode
                string userName = System.Security.Principal.WindowsIdentity.GetCurrent().Name.Split('\\')[1];
                string dir      = "c:\\Users\\" + userName + "\\Downloads";

                option.AddUserProfilePreference("download.prompt_for_download", "false");
                option.AddUserProfilePreference("download.directory_upgrade", "true");
                option.AddUserProfilePreference("download.prompt_for_download", "false");
                option.AddUserProfilePreference("safebrowsing.enabled", "false");
                option.AddUserProfilePreference("safebrowsing.disable_download_protection", "true");
                option.AddArguments("--disable-web-security");
                option.AddUserProfilePreference("download.default_directory", dir);

                param.Add("behavior", "allow");
                param.Add("downloadPath", dir);

                ChromeDriver drv = new ChromeDriver(ChromeDriverService.CreateDefaultService(chromeDriverExePath), option, TimeSpan.FromMinutes(3));
                drv.ExecuteChromeCommand("Page.setDownloadBehavior", param);
                _driver = drv;
            }
            else
            {
                _driver = new ChromeDriver(ChromeDriverService.CreateDefaultService(chromeDriverExePath), option, TimeSpan.FromMinutes(3));
            }


            _driver.Manage().Window.Size = new Size(1920, 1080);
            _driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(60);

            _wait = new WebDriverWait(_driver, TimeSpan.FromSeconds(30));

            _objectContainer.RegisterInstanceAs <IWebDriver>(_driver);
            _objectContainer.RegisterInstanceAs <WebDriverWait>(_wait);
            //_objectContainer.RegisterInstanceAs<IConfiguration>(config);
        }
Пример #6
0
        static void Main(string[] args)
        {
            // ダウンロード場所
            var downloadPath = @"D:\DL";
            // chromedriver.exeの場所
            var driverPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);

            // chromedriver.exe のログ出力設定
            //  何かあった時に参照する
            //  上書きされるので、残す場合はファイル名にダイムスタンプをつけるとか
            var service = ChromeDriverService.CreateDefaultService(driverPath);

            service.LogPath = @"D:\logs\ChromeDriver.log";
            service.EnableVerboseLogging = false;

            var options = new ChromeOptions();

            // ヘッドレスモード
            options.AddArgument("--headless");

            // 通常モード時のダウンロード場所指定
            options.AddUserProfilePreference("download.default_directory", downloadPath);
            options.AddUserProfilePreference("download.prompt_for_download", "false");
            options.AddUserProfilePreference("download.directory_upgrade", "true");

            using (var webDriver = new ChromeDriver(service, options))
            {
                try
                {
                    // ヘッドレスモードモード時のダウンロード場所指定
                    var parameter = new Dictionary <string, object>();
                    parameter["behavior"]     = "allow";
                    parameter["downloadPath"] = downloadPath;
                    webDriver.ExecuteChromeCommand("Page.setDownloadBehavior", parameter);

                    // CSVダウンロード操作
                    webDriver.Url = @"http://jusyo.jp/csv/new.php";
                    Debug.WriteLine("表示したよ");
                    var link = webDriver.FindElement(By.XPath("//a[contains(text(), 'csv_tohoku.zip')]"));
                    link.Click();
                    Debug.WriteLine("ダウンロード");
                }
                finally
                {
                    // ダウンロード完了を待機。面倒なので 1秒待機
                    //   WebDriverWait を利用したダウンロード完了待ちを作成する?
                    //   ダウンロード中とダウンロード完了を見分けれる?
                    Thread.Sleep(1000);
                    Debug.WriteLine("終了するよ");
                    webDriver.Quit();
                }
            }
        }
Пример #7
0
        /// <summary>
        /// ASCII
        /// </summary>
        /// <param name="value"></param>
        /// <returns></returns>
        public static void Save(ChromeDriver driver, string path, ScreenshotImageFormat type, bool isFull = false)
        {
            if (isFull)
            {
                var filePath = path;

                Dictionary <string, Object> metrics = new Dictionary <string, Object>();
                metrics["width"]  = driver.ExecuteScript("return Math.max(window.innerWidth,document.body.scrollWidth,document.documentElement.scrollWidth)");
                metrics["height"] = driver.ExecuteScript("return Math.max(window.innerHeight,document.body.scrollHeight,document.documentElement.scrollHeight)");
                //返回当前显示设备的物理像素分辨率与 CSS 像素分辨率的比率
                metrics["deviceScaleFactor"] = driver.ExecuteScript("return window.devicePixelRatio");
                metrics["mobile"]            = driver.ExecuteScript("return typeof window.orientation !== 'undefined'");
                driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics);

                driver.GetScreenshot().SaveAsFile(filePath, ScreenshotImageFormat.Png);
            }
            else
            {
                Screenshot shot = driver.TakeScreenshot();
                shot.SaveAsFile(path, type);
            }
        }
Пример #8
0
        // 사이트 캡쳐
        private bool Capture(ChromeDriver driver, string bunzi, string ho)
        {
            try
            {
                Dictionary <string, Object> metrics = new Dictionary <string, Object>();
                metrics["width"]             = driver.ExecuteScript("return Math.max(window.innerWidth,document.body.scrollWidth,document.documentElement.scrollWidth)");
                metrics["height"]            = driver.ExecuteScript("return Math.max(window.innerHeight,document.body.scrollHeight,document.documentElement.scrollHeight)");
                metrics["deviceScaleFactor"] = (double)driver.ExecuteScript("return window.devicePixelRatio");
                metrics["mobile"]            = driver.ExecuteScript("return typeof window.orientation !== 'undefined'");

                driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics);

                string path_to_save_screenshot = SavePath + @"/" + bunzi + "-" + ho + ".png";
                driver.GetScreenshot().SaveAsFile(path_to_save_screenshot, ScreenshotImageFormat.Png);
                return(true);
            }

            catch
            {
                return(false);
            }
        }
Пример #9
0
        static void Page(byte number, string go_to_page, string wait_for_page)//download material from page
        {
            Console.WriteLine("\tPage" + number);
            RunJS(go_to_page);
            RunJS(wait_for_page);                                                                                                                                                                                                                                //wait for page
            ReadOnlyCollection <object> table_items = (ReadOnlyCollection <object>)RunJS("let ret=[],table=document.querySelector(\'#grdPost > tbody\').children;let l=table.length-2;for(let i=1;i<l;i++)ret.push(table[i].children[2].innerText);return ret"); //get all items on page
            byte i = 2, j, content_number; int hyphen; string subject; List <byte> haves;

            foreach (string table_value in table_items)
            {
                if (subjects.IsMatch(table_value))        //my subject
                {
                    hyphen         = table_value.IndexOf('-');
                    subject        = table_value.Substring(hyphen + 2, table_value.LastIndexOf('-') - hyphen - 3);
                    content_number = Convert.ToByte(table_value.Substring(8, hyphen - 9));
                    haves          = memory[subject];
                    if (!haves.Contains(content_number))            //need material
                    //download
                    {
                        RunJS("document.querySelector(\'#grdPost > tbody > tr:nth-child(" + i + ") > td:nth-child(5) > input[type=button]\').click()");                                                                                                                //click "Click" and go to download page
                        j = 0;
                        ReadOnlyCollection <object> files = (ReadOnlyCollection <object>)RunJS("let ret=[],table=document.querySelector(\'#grdDoc > tbody\').children;let l=table.length-1;for(let i=1;i<l;i++)ret.push(table[i].children[1].innerText);return ret;"); //get filename
                        foreach (string filename in files)
                        {
                            disable_security["downloadPath"] = String.Format(download_dir, subject);                 //set download path
                            browser.ExecuteChromeCommand("Page.setDownloadBehavior", disable_security);              //disables Chrome security feature,copied from Stack Overflow
                            RunJS("document.querySelector(\'#grdDoc_btnDownload_" + j + "\').click()");              //click "Click"
                            Console.WriteLine("\t\t" + filename);
                            j++;
                        }
                        haves.Add(content_number);                //remember
                        browser.Navigate().Back();                //move back
                        RunJS(go_to_page); RunJS(wait_for_page);  //counter to website bug
                    }
                }
                i++;
            }
        }
Пример #10
0
        /// <summary>
        /// Takes a screenshot of a web page.
        /// </summary>
        /// <param name="url">The URL of the web page to screenshoot.</param>
        /// <param name="filePath">The image file to save to.</param>
        /// <param name="width">The device width of the browser</param>
        /// <returns>The full file path of the saved image</returns>
        public NavigationTiming TakeScreenshot(string url, string filePath, int?width = null)
        {
            var outputDir = Path.GetDirectoryName(filePath);

            Directory.CreateDirectory(outputDir);
            _driver.Navigate().GoToUrl(url);

            ResizeWindow();
            if (_options.HighlightBrokenLinks)
            {
                HighlightBrokenLinks(url);
            }

            // Sometimes, when this isn't high enough, the performance timings return incomplete data.
            int screenshotDelay = 3000;

            Thread.Sleep(screenshotDelay);

            var requestStats = GetRequestStats();
            var screenshot   = _driver.GetScreenshot();

            screenshot.SaveAsFile(filePath, _imageFormat);
            ClearResize();
            return(requestStats);

            // LOCAL FUNCTIONS

            NavigationTiming GetRequestStats()
            {
                try
                {
                    var stats = (string)_driver.ExecuteScript(
                        @"return (window && window.performance && JSON.stringify([...window.performance.getEntriesByType('navigation'),{ }][0])) || '{}'");
                    var deprecatedTiming = (string)_driver.ExecuteScript(
                        @"return (window && window.performance && window.performance.timing && JSON.stringify(window.performance.timing)) || '{}'");

                    Logger.Default.Log($"Stats for {url}...");
                    Logger.Default.Log(stats);
                    Logger.Default.Log(deprecatedTiming);

                    string ConvertToInt(Match m)
                    {
                        var origValue = m.Value;

                        if (!double.TryParse(origValue, out var dblVal))
                        {
                            return(origValue);
                        }
                        var intValue = (int)Math.Round(dblVal);

                        return(intValue.ToString());
                    }

                    stats = Regex.Replace(stats, @"\d+\.\d+", ConvertToInt);
                    Logger.Default.Log("stats converted to int: " + stats);

                    return(JsonConvert.DeserializeObject <NavigationTiming>((string)stats));
                }
                catch (Exception ex)
                {
                    Debug.WriteLine("Error parsing request timings: " + ex.Message);
                    return(new NavigationTiming());
                }
            }

            /// <summary>
            /// Repeatedly resize the window to account for lazily loaded elements.
            /// </summary>
            /// <param name="width"></param>
            void ResizeWindow()
            {
                // Adapted from https://stackoverflow.com/a/56535317
                int       numTries = 0;
                const int maxTries = 8;
                int       prevHeight;
                int       calculatedHeight = -1;
                string    autoWidthCommand =
                    @"return Math.max(
                        window.innerWidth,
                        document.body.scrollWidth,
                        document.documentElement.scrollWidth)";

                // Repeatedly resize height to allow new elements to (lazily) load.
                do
                {
                    prevHeight = calculatedHeight;
                    var calculatedWidth = width.HasValue ? $"return {width}" : autoWidthCommand;
                    calculatedHeight = CalculateDocHeight();

                    // TODO: Set device-specific user agents.
                    Dictionary <string, object> metrics = new Dictionary <string, object>
                    {
                        ["width"]             = _driver.ExecuteScript(calculatedWidth),
                        ["height"]            = calculatedHeight,
                        ["deviceScaleFactor"] = ScaleFactor(false),
                        ["mobile"]            = _driver.ExecuteScript("return typeof window.orientation !== 'undefined'")
                    };
                    _driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics);
                } while (calculatedHeight != prevHeight && ++numTries < maxTries);

                // LOCAL FUNCTIONS

                int CalculateDocHeight()
                {
                    // Sometimes a long, sometimes double, etc
                    object jsHeight = _driver.ExecuteScript(
                        @"return Math.max(
                        document.body.scrollHeight,
                        document.body.offsetHeight,
                        document.documentElement.clientHeight,
                        document.documentElement.scrollHeight,
                        document.documentElement.offsetHeight,
                        document.documentElement.getBoundingClientRect().height)");
                    double numericHeight = Convert.ToDouble(jsHeight);

                    return((int)Math.Ceiling(numericHeight));
                }

                // False for a 1:1 pixel ratio with the image
                // True for an easier-to-read image on the current monitor.
                double ScaleFactor(bool shouldScaleImage) =>
                shouldScaleImage
                    ? (double)_driver.ExecuteScript("return window.devicePixelRatio")
                    : 1.0;
            }

            void ClearResize()
            {
                try
                {
                    _driver.ExecuteChromeCommand("Emulation.clearDeviceMetricsOverride", new Dictionary <string, object>());
                }
                catch (Exception ex)
                {
                    // Usually thrown when the driver or browser window is closed.
                    if (!(ex is WebDriverException || ex is NoSuchWindowException))
                    {
                        throw;
                    }
                }
            }

            void HighlightBrokenLinks(string rawCallingUri)
            {
                var standardizedUri = new Uri(rawCallingUri);

                standardizedUri = standardizedUri.TryStandardize();

                if (_brokenLinks?.Any() != true)
                {
                    return;
                }

                var selectors =
                    _brokenLinks
                    .SelectMany(l => l.Value.Sources)
                    .Where(x => standardizedUri.Equals(x.CallingPage))
                    .Select(x => $@"a[href='{x.Href}']");

                var combinedSelector = string.Join(",", selectors);

                var styles = $@"
                    {combinedSelector} {{
                        border: 3px dashed red;
                    }}";

                var script = $@"
var style = document.createElement('style');
style.type = 'text/css';
style.innerHTML = `{styles}`;
document.getElementsByTagName('head')[0].appendChild(style);";

                _driver.ExecuteScript(script);
            }
        }
        public void DownloadDataFile()
        {
            //easy to find Windows Documents path, a bit more work for Downloads (you'd think it would be easy eh?)
            //just find Documents, then replace with Downloads and concatenate csv filename
            string downloadsPath            = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments).Replace("Documents", "Downloads");
            string covidDataFilePath        = Path.Combine(downloadsPath, QuebecPublicHealthCovidCasesWebPage.DownloadedCsvDataFilename);
            string renamedCovidDataFilename = Path.Combine(downloadsPath, "CovidDataFile." + DateTime.Now.ToString("yyyyMMdd-HHmm") + ".csv");

            var chromeOptions = new ChromeOptions();

            chromeOptions.AddArguments(new List <string>()
            {
                "headless"
            });
            chromeOptions.AddArguments("--window-size=1024,1080");

            //these are required to download files when using headless, this will behave more like a function call
            //if you want to see what the browser\Selenium is doing, you can comment the headless option part above
            chromeOptions.AddArguments(new List <string>()
            {
                "download.default_directory", downloadsPath
            });
            chromeOptions.AddArguments("--disable-web-security");
            chromeOptions.AddArguments("--allow-running-insecure-content");

            //using this causes a problem with downloading the file, I'm not sure where it goes when Selenium is headless,
            //you might be able to explicitly set the download folder in ChromeOptions and then pick it up down below in the same path,
            //but I don't know where you will run this from, so I can't do this right now
            ChromeDriver driver = new ChromeDriver(chromeOptions);

            var settingForHeadlessDownloads = new Dictionary <string, object>
            {
                { "behavior", "allow" },
                { "downloadPath", downloadsPath }
            };

            driver.ExecuteChromeCommand("Page.setDownloadBehavior", settingForHeadlessDownloads);

            try
            {
                //goto the main page
                driver.Navigate().GoToUrl(QuebecPublicHealthCovidCasesWebPage.WebPageUrl);

                //scroll to the confirmed cases chart
                var confirmedCasesGraphElement = driver.FindElement(By.Id(QuebecPublicHealthCovidCasesWebPage.ConfirmedCasesGraphId));
                ((IJavaScriptExecutor)driver).ExecuteScript("arguments[0].scrollIntoView(true);", confirmedCasesGraphElement);


                //finding out when the data is finished dynamically loading
                new WebDriverWait(driver, TimeSpan.FromSeconds(30)).Until(ExpectedConditions.ElementExists(By.XPath("//div[@id='evolutionHospitalisations']//*[name()='svg']")));

                //click on the ellipsed to see the download csv option and click it
                IWebElement confirmedCasesGraphEllipse = driver.FindElement(By.XPath(QuebecPublicHealthCovidCasesWebPage.ConfirmedCasesGraphEllipseLocator));
                confirmedCasesGraphEllipse.Click();

                var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(30)).Until(ExpectedConditions.ElementExists(By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile)));
                //IWebElement downloadCsvFile = WebDriverExtensions.FindElement(driver, By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile), 30);
                IWebElement downloadCsvFile = wait.FindElement(By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile));
                downloadCsvFile.Click();

                //before closing down the browser, lets give the file a bit of time to complete
                bool fileFound = DownloadDataFile(covidDataFilePath);

                ReportDataDownladStatus(covidDataFilePath, renamedCovidDataFilename, fileFound);
            }
            finally
            {
                Console.WriteLine("Just a sec...closing Selenium Chrome web driver is slow, but it should close everything");
                driver.Close();
                driver.Dispose();
                driver.Quit();
            }
        }