public bool ClickOnMenus()
{
var menuContainer = new WebDriverWait(driver, TimeSpan.FromSeconds(10)).Until(ExpectedConditions.ElementExists((By.CssSelector("#box-apps-menu"))));
IList <IWebElement> menuLinks = menuContainer.FindElements(By.CssSelector("#app- > a"));
for (int i = 0; i < menuLinks.Count; i++)
{
menuContainer = new WebDriverWait(driver, TimeSpan.FromSeconds(10)).Until(ExpectedConditions.ElementExists((By.CssSelector("#box-apps-menu"))));
menuLinks = menuContainer.FindElements(By.CssSelector("#app- > a"));
IWebElement menuItemLink = driver.FindElement(By.LinkText(menuLinks[i].Text));
menuItemLink.Click();
var selectedMenu = driver.FindElement(By.ClassName("selected"));
IList <IWebElement> innerMenu = selectedMenu.FindElements(By.TagName("li"));
for (int j = 0; j < innerMenu.Count; j++)
{
selectedMenu = driver.FindElement(By.ClassName("docs"));
innerMenu = selectedMenu.FindElements(By.TagName("li"));
IWebElement innerMenuItemLink = driver.FindElement(By.LinkText(innerMenu[j].Text));
innerMenuItemLink.Click();
if (!IsElementPresent(By.TagName("h1")))
{
return(false);
}
}
if (!IsElementPresent(By.TagName("h1")))
{
return(false);
}
}
return(true);
}
public string SortByTimeAndCheckResults(string latest, string inputSearching)
{
DdSortBy.Click();
IWebElement lists =
new WebDriverWait(PublicPandM.PropertiesAndMethods._driver, TimeSpan.FromSeconds(2)).Until(
ExpectedConditions.ElementExists(By.XPath(
"/html[1]/body[1]/div[2]/section[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/div[2]")));
var dataLists = lists.FindElements(By.TagName("div"));
foreach (var item in dataLists)
{
if (item.Text.Contains(latest))
{
item.Click();
//onee I click, jump out of the loop, because some error might happen.
goto done;
}
}
done:
while (!PublicPandM.PropertiesAndMethods._driver.Url.Contains("Latest"))
{
Thread.Sleep(200);
}
var dataDivs = DivListResult.FindElements(By.TagName("h3"));
foreach (var item in dataDivs)
{
if (item.Text == inputSearching)
{
return(item.Text);
}
}
return(null);
}
static private void CalculatePos(string pos)
{
driver.Navigate().GoToUrl(@"https://football.fantasysports.yahoo.com/f1/849675/pointsagainst?season=2018&pos=" + pos + "&mode=average");
IWebElement table = null;
while (true)
{
try
{
table = new WebDriverWait(driver, TimeSpan.FromSeconds(10)).Until(ExpectedConditions.ElementIsVisible(By.Id("statTable0")));
break;
}
catch
{
Thread.Sleep(1000);
}
}
var rows = table.FindElements(By.TagName("tr"));
for (int i = 2; i < rows.Count; i++)
{
HandleRow(rows[i].Text, pos);
}
}
public void OpenMenuByName(string menuName)
{
var menuContainer = new WebDriverWait(driver, TimeSpan.FromSeconds(10)).Until(ExpectedConditions.ElementExists((By.CssSelector("#box-apps-menu"))));
IList <IWebElement> menuLinks = menuContainer.FindElements(By.CssSelector("#app- > a"));
IWebElement menuItemLink = driver.FindElement(By.LinkText(menuName));
menuItemLink.Click();
}
private void CalcPlayerRatings(string position, bool onlyPoints)
{
int lastWeek = Program.lastWeek;
if (position != "DEF")
{
lastWeek = Player.MinWeek();
}
for (int i = 1; i <= lastWeek; i++)
{
Console.Write(position + " : " + i.ToString() + " : ");
int rowCount = 0;
int count = 0;
bool reachedEndOfWeek = false;
do
{
IWebElement table = null;
do
{
driver.Navigate().GoToUrl(@"https://football.fantasysports.yahoo.com/f1/" + this.leagueNumber +
"/players?status=ALL&pos=" + position + "&cut_type=9&stat1=S_W_" + i + "&sort=0&sdir=1&count=" + count);
int j = 0;
while (j < 10)
{
try
{
table = new WebDriverWait(driver, TimeSpan.FromSeconds(10)).Until(
ExpectedConditions.ElementIsVisible(By.Id("players-table-wrapper")));
break;
}
catch (Exception)
{
Thread.Sleep(1000);
}
j++;
}
} while (table == null);
var rows = table.FindElements(By.TagName("tr"));
rowCount = rows.Count;
for (int j = 2; j < rowCount; j++)
{
reachedEndOfWeek = HandlePlayerRow(rows[j], i, position);
}
count += 25;
if (reachedEndOfWeek && onlyPoints)
{
break;
}
Console.Write(".");
} while (rowCount == 27);
Console.WriteLine();
}
}
public void RentType(string rentType)
{
DdpRentType.Click();
IWebElement listsDdpRentType =
new WebDriverWait(PublicPandM.PropertiesAndMethods._driver, TimeSpan.FromSeconds(5)).Until(
ExpectedConditions.ElementExists(By.XPath(
"/html[1]/body[1]/div[2]/section[1]/form[1]/fieldset[1]/div[4]/div[1]/div[2]/div[1]/div[2]")));
var lists = listsDdpRentType.FindElements(By.TagName("div"));
foreach (var item in lists)
{
if (item.Text == rentType)
{
item.Click();
}
}
}
private static void CheckPlayerPosition(string position, int lastWeek = 16, bool onlyPoints = true)
{
for (int i = 1; i < lastWeek + 1; i++)
{
Console.Write(position + " : " + i.ToString() + ": ");
int rowCount = 0;
int count = 0;
bool reachedEndOfWeek = false;
do
{
IWebElement table = null;
driver.Navigate().GoToUrl(@"https://football.fantasysports.yahoo.com/f1/849675/players?status=ALL&pos=" +
position + "&cut_type=9&stat1=S_W_" + i + "&sort=0&sdir=1&count=" + count);
while (true)
{
try
{
table = new WebDriverWait(driver, TimeSpan.FromSeconds(10)).Until(
ExpectedConditions.ElementIsVisible(By.Id("players-table-wrapper")));
break;
}
catch (Exception)
{
Thread.Sleep(1000);
}
}
var rows = table.FindElements(By.TagName("tr"));
rowCount = rows.Count;
for (int j = 2; j < rowCount; j++)
{
reachedEndOfWeek = HandlePlayerRow(rows[j], i, position);
}
count += 25;
if (reachedEndOfWeek && onlyPoints)
{
break;
}
Console.Write(".");
} while (rowCount == 27);
Console.WriteLine();
}
}
public void PropertyType(string propertyType)
{
//use js to execute on invisible element**************************
// IJavaScriptExecutor js = (IJavaScriptExecutor) PublicPandM.PropertiesAndMethods._driver;
// js.ExecuteScript("arguments[0].click()", DpdPropertyType);
//click the PropertyType
DpdPropertyType.Click();
//wait fot the lsit to show up
IWebElement listPropertyType =
new WebDriverWait(PublicPandM.PropertiesAndMethods._driver, TimeSpan.FromSeconds(5)).Until(
ExpectedConditions.ElementExists(
By.XPath(
"/html[1]/body[1]/div[2]/section[1]/form[1]/fieldset[1]/div[2]/div[2]/div[1]/div[2]")));
var lists = listPropertyType.FindElements(By.TagName("div"));
foreach (var item in lists)
{
if (item.Text == propertyType)
{
item.Click();
}
}
}
public static PlayerHeroesPreferenceModel ScrapeHeroPreferencesPhantomJs(string playerId, int days = 60)
{
if (string.IsNullOrWhiteSpace(playerId))
{
return(null);
}
var retries = 5;
while (--retries > 0)
{
try
{
using (var driver = new PhantomJSDriver())
{
var url = BuildPlayerStatsUrl(playerId);
driver.Navigate().GoToUrl(url);
// Open the 'Days to show' selector
driver.FindElementById("ctl00_MainContent_DropDownProfileTimeSpan").Click();
var daySelectParent = new WebDriverWait(driver, TimeSpan.FromSeconds(2)).Until(e =>
{
var elem = e.FindElement(By.Id("ctl00_MainContent_DropDownProfileTimeSpan_DropDown"));
return(elem.GetAttribute("style").Contains("display: block;") ? elem : null);
});
// Select the 'Last X Days' option to show only heroes played in the last so many days
var daySelectText = $"Last {days} Days";
daySelectParent.FindElements(By.CssSelector("li")).First(e => e.Text == daySelectText).Click();
// Do the same for the range (i.e. only show heroes that have been played more than x times)
driver.FindElementById("ctl00_MainContent_DropDownGamesPlayedRequired").Click();
var rangeSelectParent = new WebDriverWait(driver, TimeSpan.FromSeconds(2)).Until(e =>
{
var elem = e.FindElement(By.Id("ctl00_MainContent_DropDownGamesPlayedRequired_DropDown"));
return(elem.GetAttribute("style").Contains("display: block;") ? elem : null);
});
var rangeSelectText = "5+ Games Played";
new WebDriverWait(driver, TimeSpan.FromSeconds(2)).Until(e =>
{
var elem = rangeSelectParent.FindElements(By.CssSelector("li")).FirstOrDefault(li => li.Text == rangeSelectText);
return(elem ?? elem);
}).Click();
// Find the hero stats in the table and parse the stats out
var rows = driver.FindElementsByCssSelector("#RadGridReplayCharacterScoreResultsAverage table tbody tr");
var stats = new List <HeroStatsModel>();
foreach (var row in rows)
{
var dataElems = row.FindElements(By.TagName("td"));
var heroNameElem = dataElems[0];
var gamesPlayedElem = dataElems[2];
var winPercentElem = dataElems[3];
var heroName = heroNameElem.FindElement(By.TagName("a")).GetAttribute("title");
var gamesPlayed = gamesPlayedElem.Text;
var winPercent = winPercentElem.Text.Replace("%", "");
stats.Add(new HeroStatsModel {
Name = WebUtility.HtmlDecode(heroName), GamesPlayed = Convert.ToInt32(gamesPlayed), WinPercent = Convert.ToDouble(winPercent)
});
}
return(new PlayerHeroesPreferenceModel {
HeroStats = stats.OrderByDescending(e => e.GamesPlayed).ThenByDescending(e => e.WinPercent).ToList(), PlayerId = playerId
});
}
}
catch (StaleElementReferenceException)
{
continue;
}
catch
{
return(new PlayerHeroesPreferenceModel {
HeroStats = new List <HeroStatsModel>(), PlayerId = playerId
});
}
}
return(new PlayerHeroesPreferenceModel {
HeroStats = new List <HeroStatsModel>(), PlayerId = playerId
});
}
} //end IsElementPresent
private void ScrapeChartMill()
{
int numSymbols = 20;
int currentStock = 0;
int symbolsCount = symbols.Count;
ChangeProgress(0, "Loading ChartMill page...", finvizRows.Count);
for (int i = 0; i < symbolsCount && !frmSplash.GetCancelled(); i += numSymbols)
{
try
{
//Console.WriteLine("In ChartMill try");
ChangeProgress(0, "Loading ChartMill page...");
if (numSymbols > symbolsCount)
{
numSymbols = symbolsCount;
}
else if (i + numSymbols > symbolsCount)
{
numSymbols = symbolsCount - i;
}
string url = GetChartMillUrl(symbols.GetRange(i, numSymbols));
driver.Url = url;
driver.Navigate();
var table = new WebDriverWait(driver, TimeSpan.FromSeconds(60)).Until(ExpectedConditions.ElementIsVisible(By.XPath("/html/body/app-root/app-menu/div/mat-sidenav-container/mat-sidenav-content/app-stockcharts/app-results/div[2]/div/div/app-property-display/div/div/div/mat-card/mat-card-content/div/table/tbody")));
var rows = table.FindElements(By.TagName("tr"));
foreach (IWebElement r in rows)
{
var sym = r.FindElement(By.ClassName("text-primary")).Text;
var fund = r.FindElements(By.ClassName("cdk-column-fa"));
var growth = r.FindElements(By.ClassName("cdk-column-gr"));
var val = r.FindElements(By.ClassName("cdk-column-val"));
ChangeProgress(1, String.Format("Scraping ChartMill - {0} {1}/{2}", sym, currentStock + 1, finvizRows.Count));
chartMillRows.Add(new object[4]);
chartMillRows[currentStock][0] = sym;
for (int k = 1; k < 4; k++)
{
var stars = (k == 1 ? fund : k == 2 ? growth : val);
var full = stars.ElementAt(0).FindElements(By.CssSelector("svg[data-icon=\"star\"]")).Select(f => f).Where(f => f.GetAttribute("data-prefix") == "fas");
var half = stars.ElementAt(0).FindElements(By.ClassName("fa-star-half-alt"));
double fullCount = full.Count();
double halfCount = (half != null ? (half.Count * 0.5) : 0);
chartMillRows[currentStock][k] = fullCount + halfCount;
}//end nested for
//Console.WriteLine(String.Format("{0} fund={1} growth={2} val={3}", chartMillRows[currentStock][0], chartMillRows[currentStock][1], chartMillRows[currentStock][2], chartMillRows[currentStock][3]));
currentStock++;
}
if (rows != null && rows.Count == 3 && numSymbols > 3)
{
var extraSymbols = url.Split(new char[] { '=', '&' });
var es = extraSymbols[1].Split(' ').ToList();
es.RemoveRange(0, 3);
symbols.AddRange(es);
numSymbols = 3;
}//end if
SetProxy();
}
catch (TimeoutException)
{
numSymbols = 0;
timeOuts++;
//Console.WriteLine("In ChartMill catch. timeOuts = " + timeOuts.ToString());
if (timeOuts > 4)
{
frmScreener.ErrorMessage(new Exception("Number of browser timeouts exceded. Consider taking the following actions and try again.\n1. Clear the browsers cached images and files.\n2. Download an Ad Blocker extension.\n3. Close open browsers.\n4. Close additional running applications."));
Environment.Exit(1);
}
} //end try-catch
} //end for
FilterChartMillStocks();
} //end ScrapeChartMill
} //end SetProxy
private void ScrapeFinviz()
{
int current = 0;
int sect = 0;
while (urls.Count > 0 && !frmSplash.GetCancelled())
{
ChangeProgress(1, String.Format("Scraping Finviz - {0}", sectorHeaders[sect]));
bool paginationFound = false;
int currentStock = 1;
int i = 0;
int pages = 0;
driver.Url = urls.Pop();
driver.Navigate();
string[] numStocks = new WebDriverWait(driver, TimeSpan.FromSeconds(60)).Until(ExpectedConditions.ElementExists(By.ClassName("count-text"))).Text.Split(' ');
if (int.Parse(numStocks[1]) > 20)
{
while (timeOuts < 5 && !paginationFound && !frmSplash.GetCancelled())
{
try
{
var pagination = new WebDriverWait(driver, TimeSpan.FromSeconds(60)).Until(ExpectedConditions.ElementExists(By.ClassName("screener_pagination")));
paginationFound = true;
System.Collections.ObjectModel.ReadOnlyCollection <IWebElement> sectorPages = driver.FindElements(By.ClassName("screener-pages"));
if (sectorPages != null && sectorPages.Count > 0)
{
pages = int.Parse(sectorPages.Last().Text);
}//end if
}
catch (TimeoutException)
{
timeOuts++;
//Console.WriteLine("In Finviz catch. timeOuts = " + timeOuts.ToString());
if (timeOuts > 4)
{
frmScreener.ErrorMessage(new Exception("Number of browser timeouts exceded. Consider taking the following actions and try again.\n1. Clear the browsers cached images and files.\n2. Download an Ad Blocker extension.\n3. Close open browsers.\n4. Close additional running applications."));
Environment.Exit(1);
}
} //end try-catch
} //end while
}
//TakeScreenshot().Save(Path.Combine("C:\\Users\\N\\Pictures\\Run3", String.Format("{0}Page{1}.png", sectorHeaders[sect], i + 1)), ImageFormat.Png);
ChangeProgress(0, String.Format("Scraping Finviz - {0}", sectorHeaders[sect]), int.Parse(numStocks[1]));
do
{
try
{
//Console.WriteLine("In try");
ChangeProgress(0, String.Format("Scraping Finviz - {0}", sectorHeaders[sect]));
var table = new WebDriverWait(driver, TimeSpan.FromSeconds(30)).Until(ExpectedConditions.ElementIsVisible(By.CssSelector("#screener-content > table:nth-child(1) > tbody:nth-child(1) > tr:nth-child(4) > td:nth-child(1) > table:nth-child(1) > tbody:nth-child(1)")));
var rows = table.FindElements(By.TagName("tr"));
for (int j = 1; j < rows.Count && !frmSplash.GetCancelled(); j++)
{
var split = rows.ElementAt(j).Text.Split(new char[] { '\r', '\n' });
var temp = from s in split
where s != ""
select s;
ChangeProgress(1, String.Format("Scraping Finviz - {0} - {1} {2}/{3}", sectorHeaders[sect], temp.ElementAt(0), currentStock, numStocks[1]));
finvizRows.Add(temp.ToArray());
currentStock++;
}//end for
ChangeProgress(0, String.Format("Scraping Finviz - {0}", sectorHeaders[sect]));
current = finvizRows.Count;
/*if (IsElementPresent(By.ClassName("modal-elite-ad_content")))
* {
* driver.FindElement(By.ClassName("modal-elite-ad_content")).Click();
* }//end if*/
SetProxy();
if (i < pages)
{
string temp = driver.Url;
int end = temp.LastIndexOf('&');
temp = temp.Remove(end) + String.Format("&r={0}", (i + 1) * 20 + 1) + temp.Remove(0, end);
driver.Url = temp;
driver.Navigate();
}
i++;
/*if(urls.Count == 0 && i == pages && !frmSplash.GetOnlyScreenerTwoRun())
* {
* if (stocksAdditionalInfo != null)
* {
* var query = from s in stocksAdditionalInfo.Keys
* where finvizRows.Select(x => x[0]).First() != s
* select s;
* urls.Push(new Preferences().CreateFinvizUrlForScreener2(query.ToList()));
* }
* }*/
} catch (TimeoutException)
{
timeOuts++;
//Console.WriteLine("In Finviz catch. timeOuts = " + timeOuts.ToString());
if (timeOuts > 4)
{
frmScreener.ErrorMessage(new Exception("Number of browser timeouts exceded. Consider taking the following actions and try again.\n1. Clear the browsers cached images and files.\n2. Download an Ad Blocker extension.\n3. Close open browsers.\n4. Close additional running applications."));
Environment.Exit(1);
}
}//end try-catch
} while (i < pages && !frmSplash.GetCancelled());//end do while
sect++;
}//end while
//FilterFinvizStocks();
symbols = (from f in finvizRows select f.ElementAt(0)).Distinct().ToList();
symbols.Sort();
}//end ScrapeFinviz
