public IEnumerable <StockActiveDto> ScrapStocksActive(StockActiveQueryParameters stockActiveQueryParameters)
{
var options = new ChromeOptions();
// set some options
options.ToCapabilities();
//IWebDriver driver = new RemoteWebDriver(new Uri(""), options); for remote in a Docker container
//Local Chrome driver for debug mode
IWebDriver driver = new ChromeDriver(Environment.CurrentDirectory + "\\ThirdPart\\Selenium\\ChromeDriver", options);
try
{
_stockActiveList = new List <StockActiveDto>();
Uri uri = SetUri();
int pageCount = 100;
int pageOffset = 0;
HtmlNodeCollection nodes = new HtmlNodeCollection(null);
driver.Url = uri.ToString() + $"?offset={pageOffset}&count={pageCount}";
SetFilters(driver, _stockActiveQueryParameters);
bool nextPageRecordsExists = false;
do
{
if (nextPageRecordsExists)
{
driver.FindElement(By.XPath("//*[@id=\"fin-scr-res-table\"]/div[2]/div[2]/button[3]")).Click();
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(driver.PageSource);
var _nodes = doc.DocumentNode.SelectNodes("//*[@id=\"scr-res-table\"]/table/tbody/tr");
if (_nodes == null)
{
break;
}
foreach (var node in _nodes)
{
nodes.Add(node);
}
if (driver.FindElementSafe(By.XPath("//*[@id=\"fin-scr-res-table\"]/div[2]/div[2]/button[3]")) != null && driver.FindElement(By.XPath("//*[@id=\"fin-scr-res-table\"]/div[2]/div[2]/button[3]")).Displayed&& driver.FindElement(By.XPath("//*[@id=\"fin-scr-res-table\"]/div[2]/div[2]/button[3]")).Enabled)
{
nextPageRecordsExists = true;
}
else
{
driver.Quit();
nextPageRecordsExists = false;
}
} while (nextPageRecordsExists);
if (nodes == null)
{
return(_stockActiveList);
}
StockActiveDto stockActiveDto;
foreach (var node in nodes)
{
var nodeUriPath = node.ChildNodes.ElementAt(1).FirstChild.GetAttributeValue("href", "");
string nodeUri = "";
if (nodeUriPath != "")
{
nodeUri = _uriHost + nodeUriPath;
}
stockActiveDto = new StockActiveDto
{
Symbol = node.ChildNodes.ElementAt(1).FirstChild.InnerText,
Url = nodeUri,
Name = node.ChildNodes.ElementAt(2).InnerText,
Price = StrToDecParse(node.ChildNodes.ElementAt(3).FirstChild.InnerText),
Change = StrToDecParse(node.ChildNodes.ElementAt(4).FirstChild.InnerText),
ChangePercentage = StrToDecParse(node.ChildNodes.ElementAt(5).FirstChild.InnerText.Replace("%", string.Empty)),
Volume = StrToDecParse(node.ChildNodes.ElementAt(6).InnerText),
AvgVol3Mth = StrToDecParse(node.ChildNodes.ElementAt(7).InnerText),
MarketCap = StrToDecParse(node.ChildNodes.ElementAt(8).InnerText),
PERatioTTM = StrToDecParse((node.ChildNodes.ElementAt(9).FirstChild.InnerText ?? node.ChildNodes.ElementAt(9).InnerText)),
//Week52Range = node.ChildNodes.ElementAt(10).InnerText - there is a picture loaded by JS.
Week52Range = string.Empty
};
_stockActiveList.Add(stockActiveDto);
}
}
catch (Exception e)
{
_logger.Error(e, "Error");
_stockActiveList = null;
driver.Quit();
}
return(_stockActiveList);
}