} //end SetProxy
private void ScrapeFinviz()
{
int current = 0;
int sect = 0;
while (urls.Count > 0 && !frmSplash.GetCancelled())
{
ChangeProgress(1, String.Format("Scraping Finviz - {0}", sectorHeaders[sect]));
bool paginationFound = false;
int currentStock = 1;
int i = 0;
int pages = 0;
driver.Url = urls.Pop();
driver.Navigate();
string[] numStocks = new WebDriverWait(driver, TimeSpan.FromSeconds(60)).Until(ExpectedConditions.ElementExists(By.ClassName("count-text"))).Text.Split(' ');
if (int.Parse(numStocks[1]) > 20)
{
while (timeOuts < 5 && !paginationFound && !frmSplash.GetCancelled())
{
try
{
var pagination = new WebDriverWait(driver, TimeSpan.FromSeconds(60)).Until(ExpectedConditions.ElementExists(By.ClassName("screener_pagination")));
paginationFound = true;
System.Collections.ObjectModel.ReadOnlyCollection <IWebElement> sectorPages = driver.FindElements(By.ClassName("screener-pages"));
if (sectorPages != null && sectorPages.Count > 0)
{
pages = int.Parse(sectorPages.Last().Text);
}//end if
}
catch (TimeoutException)
{
timeOuts++;
//Console.WriteLine("In Finviz catch. timeOuts = " + timeOuts.ToString());
if (timeOuts > 4)
{
frmScreener.ErrorMessage(new Exception("Number of browser timeouts exceded. Consider taking the following actions and try again.\n1. Clear the browsers cached images and files.\n2. Download an Ad Blocker extension.\n3. Close open browsers.\n4. Close additional running applications."));
Environment.Exit(1);
}
} //end try-catch
} //end while
}
//TakeScreenshot().Save(Path.Combine("C:\\Users\\N\\Pictures\\Run3", String.Format("{0}Page{1}.png", sectorHeaders[sect], i + 1)), ImageFormat.Png);
ChangeProgress(0, String.Format("Scraping Finviz - {0}", sectorHeaders[sect]), int.Parse(numStocks[1]));
do
{
try
{
//Console.WriteLine("In try");
ChangeProgress(0, String.Format("Scraping Finviz - {0}", sectorHeaders[sect]));
var table = new WebDriverWait(driver, TimeSpan.FromSeconds(30)).Until(ExpectedConditions.ElementIsVisible(By.CssSelector("#screener-content > table:nth-child(1) > tbody:nth-child(1) > tr:nth-child(4) > td:nth-child(1) > table:nth-child(1) > tbody:nth-child(1)")));
var rows = table.FindElements(By.TagName("tr"));
for (int j = 1; j < rows.Count && !frmSplash.GetCancelled(); j++)
{
var split = rows.ElementAt(j).Text.Split(new char[] { '\r', '\n' });
var temp = from s in split
where s != ""
select s;
ChangeProgress(1, String.Format("Scraping Finviz - {0} - {1} {2}/{3}", sectorHeaders[sect], temp.ElementAt(0), currentStock, numStocks[1]));
finvizRows.Add(temp.ToArray());
currentStock++;
}//end for
ChangeProgress(0, String.Format("Scraping Finviz - {0}", sectorHeaders[sect]));
current = finvizRows.Count;
/*if (IsElementPresent(By.ClassName("modal-elite-ad_content")))
* {
* driver.FindElement(By.ClassName("modal-elite-ad_content")).Click();
* }//end if*/
SetProxy();
if (i < pages)
{
string temp = driver.Url;
int end = temp.LastIndexOf('&');
temp = temp.Remove(end) + String.Format("&r={0}", (i + 1) * 20 + 1) + temp.Remove(0, end);
driver.Url = temp;
driver.Navigate();
}
i++;
/*if(urls.Count == 0 && i == pages && !frmSplash.GetOnlyScreenerTwoRun())
* {
* if (stocksAdditionalInfo != null)
* {
* var query = from s in stocksAdditionalInfo.Keys
* where finvizRows.Select(x => x[0]).First() != s
* select s;
* urls.Push(new Preferences().CreateFinvizUrlForScreener2(query.ToList()));
* }
* }*/
} catch (TimeoutException)
{
timeOuts++;
//Console.WriteLine("In Finviz catch. timeOuts = " + timeOuts.ToString());
if (timeOuts > 4)
{
frmScreener.ErrorMessage(new Exception("Number of browser timeouts exceded. Consider taking the following actions and try again.\n1. Clear the browsers cached images and files.\n2. Download an Ad Blocker extension.\n3. Close open browsers.\n4. Close additional running applications."));
Environment.Exit(1);
}
}//end try-catch
} while (i < pages && !frmSplash.GetCancelled());//end do while
sect++;
}//end while
//FilterFinvizStocks();
symbols = (from f in finvizRows select f.ElementAt(0)).Distinct().ToList();
symbols.Sort();
}//end ScrapeFinviz
