static void Main(string[] args)
{
//! Go to Home Page Pame Stoi
IWebDriver driver1 = new ChromeDriver(@"C:\Users\g.stavrou\source\repos\HtmlParserProgram\HtmlParserProgram\bin\Debug\netcoreapp2.1");
OddsContext dc = new OddsContext();
List <Companies> companiesSites = dc.Companies.ToList();
foreach (Companies site in companiesSites)
{
if (site.IsActive.Equals(1) && !string.IsNullOrWhiteSpace(site.Link))
{
switch (site.Id)
{
case 1:
new Opap(driver1, site.Link);
break;
}
}
}
PameStoixima pameStoixima = new PameStoixima("https://pamestoixima.opap.gr/el/live-kouponi-stoiximatos", driver1);
//string companyUrl = string.Empty;
//DataBase cmp = new DataBase();
//foreach (DataRow row in cmp.companiesDataTable.Rows)
//{
// string company = row["Descr"].ToString();
// switch (company)
// {
// case "OPAP":
// //PameStoixima pameStoixima = new PameStoixima(row["Link"].ToString() , driver1);
// PameStoixima pameStoixima = new PameStoixima("https://www.pamestoixima.gr/EN/1/sports#action=sports", driver1);
// companyUrl = pameStoixima.url;
// break;
// }
//}
//IJavaScriptExecutor js = (IJavaScriptExecutor)driver1;
//string title = (string)js.ExecuteScript("return document.body.innerHTML");
//string title1 = (string)js.ExecuteScript("return document.body.innerText");
//driver1.FindElement(By.CssSelector("dynamic-root")).Click();
// SendKeys(Keys.Control + "S" + Keys.Control);
System.Threading.Thread.Sleep(2000);
//! Download Page and parse it to HtmlDocument
//HtmlNodeCollection headlineTextDiv = pageDocument.DocumentNode.SelectNodes("(//div[contains(@id,'DynamicContentComponent31-group-48823.1')])");
//foreach (HtmlNode node in headlineTextDiv)
//{
// Console.WriteLine(node.OuterHtml);
//}
////! Find All Matches
//var headlineText = pageDocument.DocumentNode.SelectNodes("(//span[contains(@behavior.id,'ShowEvent')])");
//foreach (HtmlNode node in headlineText)
//{
// Console.WriteLine(node.OuterHtml);
// //! Step 2
// string behaviorID = node.GetAttributeValue("behavior.showevent.idfoevent", "Cant Fetch");
// Console.WriteLine(behaviorID);
// string onmorebetsclickedString = $"(//a[contains(@behavior.onmorebetsclicked.idfoevent,'{behaviorID}')])";
// var onmorebetsclicked = pageDocument.DocumentNode.SelectNodes(onmorebetsclickedString);
// foreach (HtmlNode nodeStep2 in onmorebetsclicked)
// {
// Console.WriteLine(nodeStep2.OuterHtml);
// //Step 3
// string nodesStep3String = $"(//input[contains(@behavior.selectionwithbetdatesclick.mtag,'E{behaviorID}|SCORE')])";
// var nodesStep3 = pageDocument.DocumentNode.SelectNodes(nodesStep3String);
// foreach (HtmlNode nodeStep3 in nodesStep3)
// {
// IWebDriver driver2 = new ChromeDriver();
// driver2.Navigate().GoToUrl("https://www.pamestoixima.gr/EN/1/sports#bo-navigation=16405.1&action=market-group-list&dynamic=17099.1");
// //string pageSource = driver2.PageSource;
// //Console.WriteLine(pageSource);
// Console.Read();
// }
// }
// Console.WriteLine("=======================================================");
//}
//Console.WriteLine(pageDocument.DocumentNode.InnerHtml);
//MainAsync(args).ConfigureAwait(false).GetAwaiter().GetResult();
//using (var context = new OddsContext())
//{
// Companies cmp1 = context.Companies.FirstOrDefault(x => x.Id == 1);
//}
}
private void Fill_GamePick(IWebElement moreBetsPage, Game game)
{
HtmlNodeCollection htmlNodeCollection = null;
int countDiv = 0;
while (htmlNodeCollection == null)
{
string pgData = this.driver1.PageSource.Replace(System.Environment.NewLine, "").Replace("\t", "");
htmlNodeCollection = generalMethods.FindSpecificElements(generalMethods.ParseHtmlPageSource(pgData), string.Format("(//div[@id='MarketListContentComponent32-event-{0}-group-1'])", game.DynamicId));
}
if (htmlNodeCollection != null && htmlNodeCollection.Count > 0)
{
if (htmlNodeCollection[0].ChildNodes.Count > 0 && htmlNodeCollection[0].ChildNodes.Count == 1)
{
countDiv = htmlNodeCollection[0].ChildNodes[0].ChildNodes.Count;
//behavior.id="ToggleMarket"
// \"toggler\"
foreach (HtmlNode gamePick in htmlNodeCollection[0].ChildNodes[0].ChildNodes)
{
if (gamePick.ChildNodes.Count > 0)
{
if (gamePick.ChildNodes[0].ChildNodes.Count > 0)
{
using (OddsContext context = new OddsContext())
{
GamePick gamePickRecord = new GamePick();
// Foreach \"market\" class
foreach (HtmlNode market in gamePick.ChildNodes[0].ChildNodes)
{
//HtmlNode lastNode = findLastChildNode(market);
if (market.Name.Equals("h2"))
{
string GamePickDescr = market.FirstChild.InnerText;
gamePickRecord = context.GamePick.FirstOrDefault(x => x.GameId == game.Id && x.Descr.Equals(GamePickDescr));
if (gamePickRecord == default(GamePick))
{
gamePickRecord = new GamePick();
context.GamePick.Add(gamePickRecord);
gamePickRecord.Id = _dataBase.X_getGID("GamePick");
gamePickRecord.GameId = game.Id;
}
gamePickRecord.Descr = GamePickDescr;
gamePickRecord.OddSumNum = 0;
context.SaveChanges();
}
if (market.Name.Equals("table"))
{
foreach (HtmlNode tableNode in market.FirstChild.ChildNodes)
{
foreach (HtmlNode trNode in tableNode.ChildNodes)
{
if (trNode.ChildNodes.Count > 0)
{
foreach (HtmlNode tdNode in trNode.FirstChild.ChildNodes)
{
gamePickRecord.OddSumNum++;
// Left Td String Value
string leftTd = tdNode.FirstChild.FirstChild.InnerText;
string rightTd = tdNode.FirstChild.LastChild.InnerText;
GamePickValue gamePickValue = context.GamePickValue.FirstOrDefault(x => x.Descr.Equals(leftTd) && x.GamePickId == gamePickRecord.Id);
if (gamePickValue == default(GamePickValue))
{
gamePickValue = new GamePickValue();
context.GamePickValue.Add(gamePickValue);
gamePickValue.Id = _dataBase.X_getGID("GamePickValue");
gamePickValue.PickValue = Convert.ToDouble(rightTd.Replace(',', '.'));
}
else
{
//gamePickValue.ChangedValue = Convert.ToDouble(rightTd.Replace(',', '.'));
}
gamePickValue.Descr = gamePickValue.AlternativeDescr = leftTd;
gamePickValue.OddsUpdated = DateTime.Now;
gamePickValue.GamePickId = gamePickRecord.Id;
gamePickValue.CompanyId = 1;
context.SaveChanges();
}
}
}
}
}
}
}
}
}
string t = gamePick.Name;
}
}
}
using (OddsContext context = new OddsContext())
{
List <GamePick> gamePickRecord = context.GamePick.Where(x => x.GameId == game.Id).ToList();
if (gamePickRecord.Count == countDiv)
{
Console.WriteLine($"Game {game.Descr} has all records");
}
}
}
private void ParsePageData()
{
this.driver1.Navigate().GoToUrl(this.url);
generalMethods.setTimeOut(2);
TimeSpan tm = new TimeSpan(0, 0, 8);
driver1.Manage().Timeouts().ImplicitWait = tm;
//! Get HTML Code
string pageSource = this.driver1.PageSource;
pageSource = pageSource.Replace(Environment.NewLine, "").Replace("\t", "");
string getID = string.Empty;
//! Parse HTML
HttpClient client = new HttpClient();
HtmlDocument pageDocument = new HtmlDocument();
pageDocument = generalMethods.ParseHtmlPageSource(pageSource);
while (htmlNodeCollection == null)
{
htmlNodeCollection = generalMethods.FindSpecificElements(pageDocument, "(//ul[contains(@class,'nodes')])");
}
//! Find FootBall Tab From HomePage
foreach (HtmlNode node in htmlNodeCollection)
{
if (node.ChildNodes.Count > 0)
{
foreach (HtmlNode childNode in node.ChildNodes)
{
if (childNode.Name.Equals("li") && childNode.InnerText.Equals("Football"))
{
foreach (HtmlNode aref in childNode.ChildNodes)
{
if (aref.Name == "a")
{
if (!string.IsNullOrWhiteSpace(aref.Attributes["behavior.node.id"].Value))
{
using (OddsContext context = new OddsContext())
{
Companies companyFootball = context.Companies.FirstOrDefault(x => x.Link.Equals(this.url));
if (companyFootball != default(Companies))
{
companyFootball.DynamicParam = aref.Attributes["behavior.node.id"].Value;
}
context.SaveChanges();
}
}
}
}
//! Find Football Link and Get ID
HtmlAttributeCollection nodeAttributeCollection = childNode.Attributes;
foreach (HtmlAttribute attribute in nodeAttributeCollection)
{
if (attribute.Name.Equals("id") && attribute.Value.Contains("DynamicRootComponent"))
{
IWebElement li = driver1.FindElement(By.Id(attribute.Value));
IWebElement liList = li.FindElement(By.XPath(string.Format("//a[contains(@behavior.node.id, '{0}')]", attribute.Value.Split('-')[1])));
liList.Click();
break;
}
}
}
}
}
}
//! Get Competition Page and Fill
//generalMethods.setTimeOut(4);
htmlNodeCollection = null;
while (htmlNodeCollection == null)
{
pageSource = this.driver1.PageSource.Replace(System.Environment.NewLine, "").Replace("\t", "");
htmlNodeCollection = generalMethods.FindSpecificElements(generalMethods.ParseHtmlPageSource(pageSource), "(//div[contains(@id,'DynamicContentComponent31-menu')])");
}
HtmlNode runningNode = null;
foreach (HtmlNode node in htmlNodeCollection)
{
if (node.ChildNodes.Count > 0)
{
foreach (HtmlNode childNode in node.ChildNodes)
{
if (childNode.Name.Equals("div"))
{
HtmlNode ul = childNode.FirstChild;
foreach (HtmlNode childNode2 in ul.ChildNodes)
{
if (childNode2.Name.Equals("li"))
{
//! Find Li Input and Spanm
if (childNode2.ChildNodes.Count > 0)
{
foreach (HtmlNode childNode3 in childNode2.ChildNodes)
{
if (childNode3.Name.Equals("span"))
{
HtmlAttributeCollection nodeAttributeCollection = childNode3.Attributes;
using (OddsContext context = new OddsContext())
{
//Competition competition = context.Competition.FirstOrDefault(x => x.Descr.Equals(childNode3.InnerText));
Competition competition = context.Competition.FirstOrDefault(x => x.DynamicId.Equals(nodeAttributeCollection["behavior.gotoleague.idfwbonavigation"].Value));
if (competition == default(Competition))
{
competition = new Competition();
context.Competition.Add(competition);
context.Database.CloseConnection();
competition.Id = _dataBase.X_getGID("Competition");
context.Database.OpenConnection();
}
competition.DateUpdated = DateTime.Now;
competition.SportId = 1;
competition.Descr = childNode3.InnerText;
competition.DynamicId = nodeAttributeCollection["behavior.gotoleague.idfwbonavigation"].Value;
context.SaveChanges();
competitionGames(competition);
}
}
}
}
}
}
//IWebElement li = driver1.FindElement(By.Id(attribute.Value));
//Console.WriteLine(childNode.InnerHtml);
}
}
}
}
//generalMethods.setTimeOut(2);
//! Fill For Each Competition the Games
}
private void competitionGames(Competition competition)
{
using (OddsContext context = new OddsContext())
{
//generalMethods.setTimeOut(2);
//! Go to Game URL (DYNAMIC)
if (competition != default(Competition) && !string.IsNullOrWhiteSpace(competition.DynamicId))
{
IWebElement liList = default(IWebElement);
findElement(string.Format("//span[contains(@behavior.gotoleague.idfwbonavigation, '{0}')]", competition.DynamicId), 0);
//WebDriverWait waitForElement = new WebDriverWait(driver1, TimeSpan.FromSeconds(5));
//waitForElement.Until(ExpectedConditions.ElementIsVisible(By.XPath(string.Format("//span[contains(@behavior.gotoleague.idfwbonavigation, '{0}')]", competition.DynamicId))));
while (liList == null)
{
liList = driver1.FindElement(By.XPath(string.Format("//span[contains(@behavior.gotoleague.idfwbonavigation, '{0}')]", competition.DynamicId)));
}
liList.Click();
htmlNodeCollection = null;
int counter = 0;
while (htmlNodeCollection == null)
{
string pgData = this.driver1.PageSource.Replace(System.Environment.NewLine, "").Replace("\t", "");
htmlNodeCollection = generalMethods.FindSpecificElements(generalMethods.ParseHtmlPageSource(pgData), "(//div[contains(@id,'DynamicContentComponent31-groups')])");
if (counter > 1000)
{
return;
}
}
HtmlDocument pageDocumentTable = new HtmlDocument();
HtmlNodeCollection htmlNodeCollection1 = null;
while (htmlNodeCollection1 == null)
{
pageDocumentTable = generalMethods.ParseHtmlPageSource(htmlNodeCollection[0].OuterHtml);
htmlNodeCollection1 = generalMethods.FindSpecificElements(pageDocumentTable, "(//table)");
}
foreach (HtmlNode collection in htmlNodeCollection1)
{
if (collection.ChildNodes.Count > 0)
{
foreach (HtmlNode childNode in collection.ChildNodes)
{
if (childNode.Name.Equals("tbody"))
{
if (childNode.ChildNodes.Count > 0)
{
foreach (HtmlNode tr in childNode.ChildNodes)
{
// For Every Tr in tbody
if (tr.Name.Equals("tr") && tr.ChildNodes.Count > 0)
{
Game game = new Game();
game.CompetitionId = competition.Id;
game.DateUpdated = DateTime.Now;
foreach (HtmlNode td in tr.ChildNodes)
{
if (td.HasClass("eventname"))
{
string date = td.FirstChild.FirstChild.FirstChild.InnerText;
string time = td.FirstChild.FirstChild.LastChild.InnerText;
string MatchHomeTeam = td.LastChild.FirstChild.FirstChild.InnerText;
string AwayTeam = td.LastChild.LastChild.LastChild.InnerText;
game.Descr = string.Format("{0} - {1}", MatchHomeTeam, AwayTeam);
game.HomeTeam = MatchHomeTeam;
game.AwayTeam = AwayTeam;
if (time.Contains("mins"))
{
game.MatchDate = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, (DateTime.Now.Hour + 1), 0, 0);
}
else
{
game.MatchDate = new DateTime(DateTime.Now.Year, Convert.ToInt32(date.Split('/')[1]), Convert.ToInt32(date.Split('/')[0]), Convert.ToInt32(time.Split(':')[0]), Convert.ToInt32(time.Split(':')[1]), 0);
}
}
}
string behaviorID = tr.LastChild.FirstChild.Attributes["behavior.more.id"].Value;
string behaviorName = tr.LastChild.FirstChild.Attributes["behavior.more.id"].Name;
game.DynamicId = behaviorID;
//Game checkIfGameExists = context.Game.FirstOrDefault(x => x.CompetitionId == game.CompetitionId && x.MatchDate == game.MatchDate && x.HomeTeam.Equals(game.HomeTeam) && x.AwayTeam.Equals(game.AwayTeam));
Game checkIfGameExists = context.Game.FirstOrDefault(x => x.CompetitionId == game.CompetitionId && x.DynamicId == behaviorID);
if (checkIfGameExists == default(Game))
{
game.Id = _dataBase.X_getGID("Game");
context.Game.Add(game);
}
else
{
game.Id = checkIfGameExists.Id;
}
context.SaveChanges();
IWebElement moreBetsPage = driver1.FindElement(By.XPath(string.Format("//span[@{1}='{0}' and @behavior.id ='More' and @title='More bets']", behaviorID, behaviorName)));
while (moreBetsPage == null)
{
moreBetsPage = driver1.FindElement(By.XPath(string.Format("//span[@{1}='{0}' and @behavior.id ='More' and @title='More bets']", behaviorID, behaviorName)));
}
moreBetsPage.Click();
generalMethods.setTimeOut(2);
Fill_GamePick(moreBetsPage, game);
driver1.Navigate().Back();
}
}
}
}
}
}
}
context.SaveChanges();
driver1.Navigate().Back();
}
}
}
