public static void RunScraper() { ChromeOptions option = new ChromeOptions(); option.AddArgument("--headless"); option.AddArgument("window-size=1200,1100"); IWebDriver driver = new ChromeDriver(option); driver.Navigate().GoToUrl("https://finance.yahoo.com/"); WebDriverWait waitLogin = new WebDriverWait(driver, TimeSpan.FromSeconds(60)); waitLogin.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.ElementToBeClickable(By.Id("uh-signedin"))); IWebElement loginButton = driver.FindElement(By.Id("uh-signedin")); loginButton.Click(); WebDriverWait waitEnterUsername = new WebDriverWait(driver, TimeSpan.FromSeconds(10)); waitEnterUsername.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.ElementToBeClickable(By.Id("login-username"))); IWebElement userName = driver.FindElement(By.Id("login-username")); userName.SendKeys("meshberge"); userName.SendKeys(Keys.Enter); WebDriverWait waitEnterPassword = new WebDriverWait(driver, TimeSpan.FromSeconds(10)); waitEnterPassword.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.ElementToBeClickable(By.Id("login-passwd"))); IWebElement password = driver.FindElement(By.Id("login-passwd")); password.SendKeys("toonfan1!"); password.SendKeys(Keys.Enter); driver.Navigate().GoToUrl("https://finance.yahoo.com/portfolio/p_1/view/v1"); WebDriverWait waitDataTable = new WebDriverWait(driver, TimeSpan.FromSeconds(30)); waitDataTable.Until(SeleniumExtras.WaitHelpers.ExpectedConditions.ElementExists(By.XPath("//tr"))); IWebElement stockTable = driver.FindElement(By.XPath("//tbody")); List <IWebElement> stocks = driver.FindElements(By.XPath("//tr")).ToList(); List <IWebElement> rows = stockTable.FindElements(By.XPath("//tr")).ToList(); int rowsCount = rows.Count; using (var context = new FinanceDB()) { for (int row = 1; row < rowsCount; row++) { List <IWebElement> cells = rows.ElementAt(row).FindElements(By.TagName("td")).ToList(); int cellsCount = cells.Count; string symbolData = cells.ElementAt(0).Text; string lastPriceData = cells.ElementAt(1).Text; string changeData = cells.ElementAt(2).Text; string changeRateData = cells.ElementAt(3).Text; string currencyData = cells.ElementAt(4).Text; string marketTimeData = cells.ElementAt(5).Text; string volumeData = cells.ElementAt(6).Text; string shareData = cells.ElementAt(7).Text; string averageVolumeData = cells.ElementAt(8).Text; string marketCapData = cells.ElementAt(12).Text; DateTime timeStampData = DateTime.Now; var stockRecord = new Stock { Symbol = symbolData, LastPrice = lastPriceData, Change = changeData, ChangeRate = changeRateData, Currency = currencyData, MarketTime = marketTimeData, Volume = volumeData, Shares = shareData, AverageVolume = averageVolumeData, MarketCap = marketCapData, Timestamp = timeStampData }; context.Stocks.Add(stockRecord); context.SaveChanges(); } driver.Close(); // return RedirectToAction("Recent"); } }