private void InitializeChrome()
{
if (Directory.Exists(_downloadDirectory))
{
Directory.Delete(_downloadDirectory, true);
}
Directory.CreateDirectory(_downloadDirectory);
ChromeOptions options = new ChromeOptions();
options.AddLocalStatePreference("profile.default_content_settings.popups", 0);
options.AddLocalStatePreference("download.default_directory", _downloadDirectory);
options.AddUserProfilePreference("profile.default_content_settings.popups", 0);
options.AddUserProfilePreference("download.default_directory", _downloadDirectory);
options.LeaveBrowserRunning = true;
//silent start
options.AddArguments("headless", "disable-gpu", "start-maximized", "window-size=1920,1080");
ChromeDriverService service = ChromeDriverService.CreateDefaultService();
service.HideCommandPromptWindow = true;
//start chrome
_driver = new ChromeDriver(options);
var param = new Dictionary <string, object>();
param.Add("behavior", "allow");
param.Add("downloadPath", _downloadDirectory);
_driver.ExecuteChromeCommand("Page.setDownloadBehavior", param);
}
public CloudFlareServiceSelenium(IConfiguration configuration, ILogger <CloudFlareServiceSelenium> logger)
{
_logger = logger;
var userAgent = RandomUserAgent.Generate();
_logger.LogInformation($"Initialize Chrome driver with userAgent: {userAgent}");
var chromeOptions = new ChromeOptions();
if (configuration.IsHeadless())
{
chromeOptions.AddArguments("headless");
_logger.LogInformation("Enable Headless mode");
}
chromeOptions.AddExcludedArgument("enable-automation");
chromeOptions.AddAdditionalCapability("useAutomationExtension", false);
chromeOptions.AddArgument("--disable-blink-features=AutomationControlled");
chromeOptions.AddArgument("--lang=en-US");
_driver = new ChromeDriver(chromeOptions);
_driver.ExecuteScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})");
_driver.ExecuteChromeCommand("Network.setUserAgentOverride", new Dictionary <string, object>
{
{ "userAgent", userAgent }
});
_driver.Navigate().GoToUrl(CloudFlareLoginUrl);
_waiter = new WebDriverWait(_driver, TimeSpan.FromMinutes(3));
}
public static void AddScriptToEvaluateOnNewDocument(this ChromeDriver webDriver, string script)
{
var parameters = new Dictionary <string, object> {
["source"] = script
};
webDriver.ExecuteChromeCommand("Page.addScriptToEvaluateOnNewDocument", parameters);
}
public static void EnableHeadlessDownload(this ChromeDriver chromeDriver, string downloadPath = null)
{
var parametros = new Dictionary <string, object>
{
{ "behavior", "allow" },
{ "downloadPath", downloadPath ?? AppDomain.CurrentDomain.BaseDirectory }
};
chromeDriver.ExecuteChromeCommand("Page.setDownloadBehavior", parametros);
}
public void Initialize()
{
scenarioName = featureName.CreateNode <Scenario>(_scenarioContext.ScenarioInfo.Title);
var param = new Dictionary <string, object>();
ChromeOptions option = new ChromeOptions();
option.PageLoadStrategy = PageLoadStrategy.Normal;
//headless = (GetValueFromConfig("headless") != "") ? bool.Parse(GetValueFromConfig("headless")) : true;
var headlessConfig = GetValueFromAppSettings().Headless;
headless = string.IsNullOrEmpty(headlessConfig)? false: Convert.ToBoolean(headlessConfig);
var chromeDriverExePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
if (headless)
{
option.AddArguments("--headless");
// Add option to download file with headless mode
string userName = System.Security.Principal.WindowsIdentity.GetCurrent().Name.Split('\\')[1];
string dir = "c:\\Users\\" + userName + "\\Downloads";
option.AddUserProfilePreference("download.prompt_for_download", "false");
option.AddUserProfilePreference("download.directory_upgrade", "true");
option.AddUserProfilePreference("download.prompt_for_download", "false");
option.AddUserProfilePreference("safebrowsing.enabled", "false");
option.AddUserProfilePreference("safebrowsing.disable_download_protection", "true");
option.AddArguments("--disable-web-security");
option.AddUserProfilePreference("download.default_directory", dir);
param.Add("behavior", "allow");
param.Add("downloadPath", dir);
ChromeDriver drv = new ChromeDriver(ChromeDriverService.CreateDefaultService(chromeDriverExePath), option, TimeSpan.FromMinutes(3));
drv.ExecuteChromeCommand("Page.setDownloadBehavior", param);
_driver = drv;
}
else
{
_driver = new ChromeDriver(ChromeDriverService.CreateDefaultService(chromeDriverExePath), option, TimeSpan.FromMinutes(3));
}
_driver.Manage().Window.Size = new Size(1920, 1080);
_driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(60);
_wait = new WebDriverWait(_driver, TimeSpan.FromSeconds(30));
_objectContainer.RegisterInstanceAs <IWebDriver>(_driver);
_objectContainer.RegisterInstanceAs <WebDriverWait>(_wait);
//_objectContainer.RegisterInstanceAs<IConfiguration>(config);
}
static void Main(string[] args)
{
// ダウンロード場所
var downloadPath = @"D:\DL";
// chromedriver.exeの場所
var driverPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
// chromedriver.exe のログ出力設定
// 何かあった時に参照する
// 上書きされるので、残す場合はファイル名にダイムスタンプをつけるとか
var service = ChromeDriverService.CreateDefaultService(driverPath);
service.LogPath = @"D:\logs\ChromeDriver.log";
service.EnableVerboseLogging = false;
var options = new ChromeOptions();
// ヘッドレスモード
options.AddArgument("--headless");
// 通常モード時のダウンロード場所指定
options.AddUserProfilePreference("download.default_directory", downloadPath);
options.AddUserProfilePreference("download.prompt_for_download", "false");
options.AddUserProfilePreference("download.directory_upgrade", "true");
using (var webDriver = new ChromeDriver(service, options))
{
try
{
// ヘッドレスモードモード時のダウンロード場所指定
var parameter = new Dictionary <string, object>();
parameter["behavior"] = "allow";
parameter["downloadPath"] = downloadPath;
webDriver.ExecuteChromeCommand("Page.setDownloadBehavior", parameter);
// CSVダウンロード操作
webDriver.Url = @"http://jusyo.jp/csv/new.php";
Debug.WriteLine("表示したよ");
var link = webDriver.FindElement(By.XPath("//a[contains(text(), 'csv_tohoku.zip')]"));
link.Click();
Debug.WriteLine("ダウンロード");
}
finally
{
// ダウンロード完了を待機。面倒なので 1秒待機
// WebDriverWait を利用したダウンロード完了待ちを作成する？
// ダウンロード中とダウンロード完了を見分けれる？
Thread.Sleep(1000);
Debug.WriteLine("終了するよ");
webDriver.Quit();
}
}
}
/// <summary>
/// ASCII
/// </summary>
/// <param name="value"></param>
/// <returns></returns>
public static void Save(ChromeDriver driver, string path, ScreenshotImageFormat type, bool isFull = false)
{
if (isFull)
{
var filePath = path;
Dictionary <string, Object> metrics = new Dictionary <string, Object>();
metrics["width"] = driver.ExecuteScript("return Math.max(window.innerWidth,document.body.scrollWidth,document.documentElement.scrollWidth)");
metrics["height"] = driver.ExecuteScript("return Math.max(window.innerHeight,document.body.scrollHeight,document.documentElement.scrollHeight)");
//返回当前显示设备的物理像素分辨率与 CSS 像素分辨率的比率
metrics["deviceScaleFactor"] = driver.ExecuteScript("return window.devicePixelRatio");
metrics["mobile"] = driver.ExecuteScript("return typeof window.orientation !== 'undefined'");
driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics);
driver.GetScreenshot().SaveAsFile(filePath, ScreenshotImageFormat.Png);
}
else
{
Screenshot shot = driver.TakeScreenshot();
shot.SaveAsFile(path, type);
}
}
// 사이트 캡쳐
private bool Capture(ChromeDriver driver, string bunzi, string ho)
{
try
{
Dictionary <string, Object> metrics = new Dictionary <string, Object>();
metrics["width"] = driver.ExecuteScript("return Math.max(window.innerWidth,document.body.scrollWidth,document.documentElement.scrollWidth)");
metrics["height"] = driver.ExecuteScript("return Math.max(window.innerHeight,document.body.scrollHeight,document.documentElement.scrollHeight)");
metrics["deviceScaleFactor"] = (double)driver.ExecuteScript("return window.devicePixelRatio");
metrics["mobile"] = driver.ExecuteScript("return typeof window.orientation !== 'undefined'");
driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics);
string path_to_save_screenshot = SavePath + @"/" + bunzi + "-" + ho + ".png";
driver.GetScreenshot().SaveAsFile(path_to_save_screenshot, ScreenshotImageFormat.Png);
return(true);
}
catch
{
return(false);
}
}
static void Page(byte number, string go_to_page, string wait_for_page)//download material from page
{
Console.WriteLine("\tPage" + number);
RunJS(go_to_page);
RunJS(wait_for_page); //wait for page
ReadOnlyCollection <object> table_items = (ReadOnlyCollection <object>)RunJS("let ret=[],table=document.querySelector(\'#grdPost > tbody\').children;let l=table.length-2;for(let i=1;i<l;i++)ret.push(table[i].children[2].innerText);return ret"); //get all items on page
byte i = 2, j, content_number; int hyphen; string subject; List <byte> haves;
foreach (string table_value in table_items)
{
if (subjects.IsMatch(table_value)) //my subject
{
hyphen = table_value.IndexOf('-');
subject = table_value.Substring(hyphen + 2, table_value.LastIndexOf('-') - hyphen - 3);
content_number = Convert.ToByte(table_value.Substring(8, hyphen - 9));
haves = memory[subject];
if (!haves.Contains(content_number)) //need material
//download
{
RunJS("document.querySelector(\'#grdPost > tbody > tr:nth-child(" + i + ") > td:nth-child(5) > input[type=button]\').click()"); //click "Click" and go to download page
j = 0;
ReadOnlyCollection <object> files = (ReadOnlyCollection <object>)RunJS("let ret=[],table=document.querySelector(\'#grdDoc > tbody\').children;let l=table.length-1;for(let i=1;i<l;i++)ret.push(table[i].children[1].innerText);return ret;"); //get filename
foreach (string filename in files)
{
disable_security["downloadPath"] = String.Format(download_dir, subject); //set download path
browser.ExecuteChromeCommand("Page.setDownloadBehavior", disable_security); //disables Chrome security feature,copied from Stack Overflow
RunJS("document.querySelector(\'#grdDoc_btnDownload_" + j + "\').click()"); //click "Click"
Console.WriteLine("\t\t" + filename);
j++;
}
haves.Add(content_number); //remember
browser.Navigate().Back(); //move back
RunJS(go_to_page); RunJS(wait_for_page); //counter to website bug
}
}
i++;
}
}
/// <summary>
/// Takes a screenshot of a web page.
/// </summary>
/// <param name="url">The URL of the web page to screenshoot.</param>
/// <param name="filePath">The image file to save to.</param>
/// <param name="width">The device width of the browser</param>
/// <returns>The full file path of the saved image</returns>
public NavigationTiming TakeScreenshot(string url, string filePath, int?width = null)
{
var outputDir = Path.GetDirectoryName(filePath);
Directory.CreateDirectory(outputDir);
_driver.Navigate().GoToUrl(url);
ResizeWindow();
if (_options.HighlightBrokenLinks)
{
HighlightBrokenLinks(url);
}
// Sometimes, when this isn't high enough, the performance timings return incomplete data.
int screenshotDelay = 3000;
Thread.Sleep(screenshotDelay);
var requestStats = GetRequestStats();
var screenshot = _driver.GetScreenshot();
screenshot.SaveAsFile(filePath, _imageFormat);
ClearResize();
return(requestStats);
// LOCAL FUNCTIONS
NavigationTiming GetRequestStats()
{
try
{
var stats = (string)_driver.ExecuteScript(
@"return (window && window.performance && JSON.stringify([...window.performance.getEntriesByType('navigation'),{ }][0])) || '{}'");
var deprecatedTiming = (string)_driver.ExecuteScript(
@"return (window && window.performance && window.performance.timing && JSON.stringify(window.performance.timing)) || '{}'");
Logger.Default.Log($"Stats for {url}...");
Logger.Default.Log(stats);
Logger.Default.Log(deprecatedTiming);
string ConvertToInt(Match m)
{
var origValue = m.Value;
if (!double.TryParse(origValue, out var dblVal))
{
return(origValue);
}
var intValue = (int)Math.Round(dblVal);
return(intValue.ToString());
}
stats = Regex.Replace(stats, @"\d+\.\d+", ConvertToInt);
Logger.Default.Log("stats converted to int: " + stats);
return(JsonConvert.DeserializeObject <NavigationTiming>((string)stats));
}
catch (Exception ex)
{
Debug.WriteLine("Error parsing request timings: " + ex.Message);
return(new NavigationTiming());
}
}
/// <summary>
/// Repeatedly resize the window to account for lazily loaded elements.
/// </summary>
/// <param name="width"></param>
void ResizeWindow()
{
// Adapted from https://stackoverflow.com/a/56535317
int numTries = 0;
const int maxTries = 8;
int prevHeight;
int calculatedHeight = -1;
string autoWidthCommand =
@"return Math.max(
window.innerWidth,
document.body.scrollWidth,
document.documentElement.scrollWidth)";
// Repeatedly resize height to allow new elements to (lazily) load.
do
{
prevHeight = calculatedHeight;
var calculatedWidth = width.HasValue ? $"return {width}" : autoWidthCommand;
calculatedHeight = CalculateDocHeight();
// TODO: Set device-specific user agents.
Dictionary <string, object> metrics = new Dictionary <string, object>
{
["width"] = _driver.ExecuteScript(calculatedWidth),
["height"] = calculatedHeight,
["deviceScaleFactor"] = ScaleFactor(false),
["mobile"] = _driver.ExecuteScript("return typeof window.orientation !== 'undefined'")
};
_driver.ExecuteChromeCommand("Emulation.setDeviceMetricsOverride", metrics);
} while (calculatedHeight != prevHeight && ++numTries < maxTries);
// LOCAL FUNCTIONS
int CalculateDocHeight()
{
// Sometimes a long, sometimes double, etc
object jsHeight = _driver.ExecuteScript(
@"return Math.max(
document.body.scrollHeight,
document.body.offsetHeight,
document.documentElement.clientHeight,
document.documentElement.scrollHeight,
document.documentElement.offsetHeight,
document.documentElement.getBoundingClientRect().height)");
double numericHeight = Convert.ToDouble(jsHeight);
return((int)Math.Ceiling(numericHeight));
}
// False for a 1:1 pixel ratio with the image
// True for an easier-to-read image on the current monitor.
double ScaleFactor(bool shouldScaleImage) =>
shouldScaleImage
? (double)_driver.ExecuteScript("return window.devicePixelRatio")
: 1.0;
}
void ClearResize()
{
try
{
_driver.ExecuteChromeCommand("Emulation.clearDeviceMetricsOverride", new Dictionary <string, object>());
}
catch (Exception ex)
{
// Usually thrown when the driver or browser window is closed.
if (!(ex is WebDriverException || ex is NoSuchWindowException))
{
throw;
}
}
}
void HighlightBrokenLinks(string rawCallingUri)
{
var standardizedUri = new Uri(rawCallingUri);
standardizedUri = standardizedUri.TryStandardize();
if (_brokenLinks?.Any() != true)
{
return;
}
var selectors =
_brokenLinks
.SelectMany(l => l.Value.Sources)
.Where(x => standardizedUri.Equals(x.CallingPage))
.Select(x => $@"a[href='{x.Href}']");
var combinedSelector = string.Join(",", selectors);
var styles = $@"
{combinedSelector} {{
border: 3px dashed red;
}}";
var script = $@"
var style = document.createElement('style');
style.type = 'text/css';
style.innerHTML = `{styles}`;
document.getElementsByTagName('head')[0].appendChild(style);";
_driver.ExecuteScript(script);
}
}
public void DownloadDataFile()
{
//easy to find Windows Documents path, a bit more work for Downloads (you'd think it would be easy eh?)
//just find Documents, then replace with Downloads and concatenate csv filename
string downloadsPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments).Replace("Documents", "Downloads");
string covidDataFilePath = Path.Combine(downloadsPath, QuebecPublicHealthCovidCasesWebPage.DownloadedCsvDataFilename);
string renamedCovidDataFilename = Path.Combine(downloadsPath, "CovidDataFile." + DateTime.Now.ToString("yyyyMMdd-HHmm") + ".csv");
var chromeOptions = new ChromeOptions();
chromeOptions.AddArguments(new List <string>()
{
"headless"
});
chromeOptions.AddArguments("--window-size=1024,1080");
//these are required to download files when using headless, this will behave more like a function call
//if you want to see what the browser\Selenium is doing, you can comment the headless option part above
chromeOptions.AddArguments(new List <string>()
{
"download.default_directory", downloadsPath
});
chromeOptions.AddArguments("--disable-web-security");
chromeOptions.AddArguments("--allow-running-insecure-content");
//using this causes a problem with downloading the file, I'm not sure where it goes when Selenium is headless,
//you might be able to explicitly set the download folder in ChromeOptions and then pick it up down below in the same path,
//but I don't know where you will run this from, so I can't do this right now
ChromeDriver driver = new ChromeDriver(chromeOptions);
var settingForHeadlessDownloads = new Dictionary <string, object>
{
{ "behavior", "allow" },
{ "downloadPath", downloadsPath }
};
driver.ExecuteChromeCommand("Page.setDownloadBehavior", settingForHeadlessDownloads);
try
{
//goto the main page
driver.Navigate().GoToUrl(QuebecPublicHealthCovidCasesWebPage.WebPageUrl);
//scroll to the confirmed cases chart
var confirmedCasesGraphElement = driver.FindElement(By.Id(QuebecPublicHealthCovidCasesWebPage.ConfirmedCasesGraphId));
((IJavaScriptExecutor)driver).ExecuteScript("arguments[0].scrollIntoView(true);", confirmedCasesGraphElement);
//finding out when the data is finished dynamically loading
new WebDriverWait(driver, TimeSpan.FromSeconds(30)).Until(ExpectedConditions.ElementExists(By.XPath("//div[@id='evolutionHospitalisations']//*[name()='svg']")));
//click on the ellipsed to see the download csv option and click it
IWebElement confirmedCasesGraphEllipse = driver.FindElement(By.XPath(QuebecPublicHealthCovidCasesWebPage.ConfirmedCasesGraphEllipseLocator));
confirmedCasesGraphEllipse.Click();
var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(30)).Until(ExpectedConditions.ElementExists(By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile)));
//IWebElement downloadCsvFile = WebDriverExtensions.FindElement(driver, By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile), 30);
IWebElement downloadCsvFile = wait.FindElement(By.XPath(QuebecPublicHealthCovidCasesWebPage.DownloadCsvDataFile));
downloadCsvFile.Click();
//before closing down the browser, lets give the file a bit of time to complete
bool fileFound = DownloadDataFile(covidDataFilePath);
ReportDataDownladStatus(covidDataFilePath, renamedCovidDataFilename, fileFound);
}
finally
{
Console.WriteLine("Just a sec...closing Selenium Chrome web driver is slow, but it should close everything");
driver.Close();
driver.Dispose();
driver.Quit();
}
}