protected override async Task ScrapeData() { var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); try { const string url = "https://www.5dimes.eu/"; chromeDriver.Navigate().GoToUrl(url); await Task.Delay(2000); var userElement = chromeDriver.FindElementById("customerID"); userElement.SendKeys("5D2543319"); var passwordElement = chromeDriver.FindElementByName("password"); passwordElement.SendKeys("qwerty123!"); var loginButton = chromeDriver.FindElementById("submit1"); loginButton.Click(); await Task.Delay(2000); var nbaCheckbox = chromeDriver.FindElementByName("Basketball_NBA Props"); nbaCheckbox.Click(); var nbaSubmitElement = chromeDriver.FindElementById("btnContinue"); nbaSubmitElement.Click(); var doc = new HtmlDocument(); doc.LoadHtml(chromeDriver.PageSource); var rawMatches = doc.DocumentNode.SelectNodes("//html/body/form/div[@id='PageElt']/div/div[@id='Middle']/div[@id='contentRight']/div[@id='ManagedContentWrapper']/div[@id='MainContent']/div[@id='mainContentContainer']/div[@id='readingPaneSplitPane']/div[@id='readingPaneContainer']/div[@id='readingPaneContentContainer']/div[@id='contentContainer']/div[@class='linesContainer']/table/tbody/tr"); for (var i = 0; i < rawMatches.Count; i++) { var tdScoreName = rawMatches[i].SelectSingleNode("td[3]"); if (tdScoreName == null) { continue; } var scoreText = tdScoreName.InnerText; var scoreType = scoreText.Contains("points") && !scoreText.Contains("first") && !scoreText.Contains("total") && !scoreText.Contains("wins") ? ScoreType.Point : scoreText.Contains("assists") ? ScoreType.Assist : scoreText.Contains("rebounds") ? ScoreType.Rebound : scoreText.Contains("3-Pt") ? ScoreType.ThreePoint : scoreText.Contains("pts+reb+ast") ? ScoreType.PointReboundAssist : scoreText.Contains("pts+reb") ? ScoreType.PointRebound : scoreText.Contains("pts+ast") ? ScoreType.PointAssist : scoreText.Contains("reb+ast") ? ScoreType.ReboundAssist : string.Empty; if (string.IsNullOrEmpty(scoreType)) { continue; } var scoreSplits = scoreText.Split(); var playerName = scoreSplits[1].Replace(".", " "); var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches); if (match == null) { continue; } var player = ScrapeHelper.FindPlayerInMatch(playerName, match); var overText = rawMatches[i].SelectSingleNode("td[6]").InnerText; var overLine = ScrapeHelper.ConvertMetric($"{ScrapeHelper.RegexMappingExpression(overText, "o(.*)½")}.05"); var over = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overText, @"½ [\-\+](.*)").Trim()); var underText = rawMatches[i + 1].SelectSingleNode("td[6]").InnerText; var underLine = ScrapeHelper.ConvertMetric($"{ScrapeHelper.RegexMappingExpression(underText, "u(.*)½")}.05"); var under = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underText, @"½ [\-\+](.*)").Trim()); Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}"); var metric = new PlayerOverUnder { MatchId = match.Id, Over = over, OverLine = overLine, Under = under, UnderLine = underLine, PlayerId = player.Id, ScoreType = scoreType, ScrapingInformationId = GetScrapingInformation().Id, CreatedAt = DateTime.Now }; PlayerUnderOvers.Add(metric); } } catch (Exception ex) { Logger.Error(ex.ToString()); chromeDriver.Quit(); throw; } chromeDriver.Quit(); }
protected override async Task ScrapeData() { var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); try { const string url = "https://www.betus.com.pa/sportsbook/nba-basketball-lines.aspx"; chromeDriver.Navigate().GoToUrl(url); await Task.Delay(5000); var matchCount = chromeDriver.FindElementsByClassName("props").Count; await UpdateScrapeStatus(10, "Scraping metric data"); var rangeProgress = matchCount != 0 ? 90 / matchCount : 0; var currentRange = 10; for (var i = 0; i < matchCount; i++) { chromeDriver.Navigate().GoToUrl(url); await Task.Delay(3000); var props = chromeDriver.FindElementsByClassName("props"); props[i].Click(); await Task.Delay(3000); var select = new SelectElement(chromeDriver.FindElementByClassName("odds-display")); select.SelectByText("Decimal"); await Task.Delay(2000); var doc = new HtmlDocument(); doc.LoadHtml(chromeDriver.PageSource); currentRange = Math.Min(currentRange + rangeProgress, 90); var matchNode = doc.DocumentNode.SelectSingleNode("//html/body/form/div[@class='sportsbook-visitor']/div[@class='sportsbook-master']/div[@class='rounded-block-white']/span[2]/div[@class='col']/div[@id='game-lines']/div/div[@class='game-block']/div[@class='normal']/div/div[@class='future-lines inline-prop']"); var rawMetrics = matchNode.SelectNodes("table"); foreach (var rawMetric in rawMetrics) { var playerNode = rawMetric.SelectSingleNode("tbody/tr[1]/th[2]").InnerText; var scoreType = playerNode.Contains("Total Points+Rebounds+Assists") ? ScoreType.PointReboundAssist : playerNode.Contains("Total Points+Rebounds") ? ScoreType.PointRebound : playerNode.Contains("Total Points+Assists") ? ScoreType.PointAssist : playerNode.Contains("Total Rebounds+Assists") ? ScoreType.ReboundAssist : playerNode.Contains("Total Points") ? ScoreType.Point : playerNode.Contains("Total Rebounds") ? ScoreType.Rebound : playerNode.Contains("Total Assists") ? ScoreType.Assist : playerNode.Contains("Total Made 3") ? ScoreType.ThreePoint : string.Empty; if (string.IsNullOrEmpty(scoreType)) { continue; } playerNode = playerNode .Replace("\n", string.Empty) .Replace("\r", string.Empty) .Replace("\t", string.Empty) .Trim(); var playerName = ScrapeHelper.RegexMappingExpression(playerNode, @"(.*) \("); var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches); if (match == null) { continue; } var player = ScrapeHelper.FindPlayerInMatch(playerName, match); var overNode = rawMetric.SelectSingleNode("tbody/tr[2]/td[2]").InnerText; overNode = overNode .Replace("\n", string.Empty) .Replace("\r", string.Empty) .Replace("\t", string.Empty) .Replace(" ", " ") .Trim(); var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overNode, @"Over.(\d*.\d*)")); var overPriceNode = rawMetric.SelectSingleNode("tbody/tr[2]/td[3]/a").InnerText; var over = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overPriceNode, @"(\d*.\d*).*")); var underNode = rawMetric.SelectSingleNode("tbody/tr[3]/td[2]").InnerText; underNode = underNode .Replace("\n", string.Empty) .Replace("\r", string.Empty) .Replace(" ", " ") .Replace("\t", string.Empty) .Trim(); var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underNode, @"Under.(\d*.\d*)")); var underPriceNode = rawMetric.SelectSingleNode("tbody/tr[3]/td[3]/a").InnerText; var under = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underPriceNode, @"(\d*.\d*).*")); Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}"); var metric = new PlayerOverUnder { MatchId = match.Id, Over = over, OverLine = overLine, Under = under, UnderLine = underLine, PlayerId = player.Id, ScoreType = scoreType, ScrapingInformationId = GetScrapingInformation().Id, CreatedAt = DateTime.Now }; PlayerUnderOvers.Add(metric); var newProgress = GetScrapingInformation().Progress; newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange); await UpdateScrapeStatus(newProgress, null); } await UpdateScrapeStatus(currentRange, null); } } catch (Exception ex) { Logger.Error(ex.ToString()); chromeDriver.Quit(); throw; } chromeDriver.Quit(); }
protected override async Task ScrapeData() { var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); try { const string url = "https://nj.betamerica.com/sports/basketball/nba/"; chromeDriver.Navigate().GoToUrl(url); await Task.Delay(20000); var doc = new HtmlDocument(); doc.LoadHtml(chromeDriver.PageSource); var rawMatches = doc.DocumentNode.SelectNodes("/html/body/div[4]/div/div[2]/div/section[7]/div[2]/div[2]/div/sb-comp/div[1]/div/sb-lazy-render/div[*]"); var sourceIds = rawMatches .Select(rawMatch => rawMatch.SelectSingleNode("div/a")) .Select(aTag => aTag.GetAttributeValue("href", string.Empty)) .Where(sourceId => !sourceId.Contains("live-betting")).ToList(); await UpdateScrapeStatus(10, "Scraping metric data"); var rangeProgress = sourceIds.Count != 0 ? 90 / sourceIds.Count : 0; var currentRange = 10; foreach (var matchUrl in sourceIds.Select(sourceId => $"https://nj.betamerica.com{sourceId}")) { chromeDriver.Navigate().GoToUrl(matchUrl); await Task.Delay(7000); doc.LoadHtml(chromeDriver.PageSource); currentRange = Math.Min(currentRange + rangeProgress, 90); var rawMetrics = doc.DocumentNode.SelectNodes("//html/body/div[@class='content-main']/div[contains(@class, 'content-main-inner')]/div[@id='pagesWrapper']/div[@id='panel-center-inner']/section[@id='pre-live-betting']/div/div[@class='event-view-views-switcher']/div/ul"); foreach (var rawMetric in rawMetrics) { var liTags = rawMetric.SelectNodes("li"); foreach (var liTag in liTags) { var spanItem = liTag.SelectSingleNode("h4/span[2]/span"); var spanText = spanItem.InnerText; var scoreType = spanText.Contains("Total Points") ? ScoreType.Point : spanText.Contains("Total Rebounds") ? ScoreType.Rebound : spanText.Contains("Total Assists") ? ScoreType.Assist : spanText.Contains("Pts, Rebs, Asts") ? ScoreType.PointReboundAssist : string.Empty; if (string.IsNullOrEmpty(scoreType)) { continue; } var playerName = ScrapeHelper.RegexMappingExpression(spanText, @"(.*) (Total|\(Pts,)"); var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches); if (match == null) { continue; } var player = ScrapeHelper.FindPlayerInMatch(playerName, match); if (player == null) { Logger.Warning($"Cannot find any player {playerName} in match {match.Id}"); continue; } var overUnderFirstData = liTag.SelectSingleNode("div/div/div[2]/button"); var overUnderSecondData = liTag.SelectSingleNode("div/div/div[1]/button"); double?over = 0, overLine = 0, under = 0, underLine = 0; var overUnderFirstLine = overUnderFirstData.SelectSingleNode("span[1]/span[1]").InnerText; if (overUnderFirstLine.Contains("Over")) { overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderFirstLine, @"Over (\d*.\d*) .*")); var overPriceData = overUnderFirstData.SelectSingleNode("span[2]/span").InnerText; var priceNumber = ScrapeHelper.ConvertMetric(overPriceData); over = priceNumber != null ? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) : Math.Round((double)(100 / (priceNumber * -1) + 1), 2) : (double?)null; } else { underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderFirstLine, @"Under (\d*.\d*) .*")); var underPriceData = overUnderFirstData.SelectSingleNode("span[2]/span").InnerText; var priceNumber = ScrapeHelper.ConvertMetric(underPriceData); under = priceNumber != null ? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) : Math.Round((double)(100 / (priceNumber * -1) + 1), 2) : (double?)null; } var overUnderSecondLine = overUnderSecondData.SelectSingleNode("span[1]/span[1]").InnerText; if (overUnderSecondLine.Contains("Over")) { overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderSecondLine, @"Over (\d*.\d*) .*")); var overPriceData = overUnderSecondData.SelectSingleNode("span[2]/span").InnerText; var priceNumber = ScrapeHelper.ConvertMetric(overPriceData); over = priceNumber != null ? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) : Math.Round((double)(100 / (priceNumber * -1) + 1), 2) : (double?)null; } else { underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderSecondLine, @"Under (\d*.\d*) .*")); var underPriceData = overUnderSecondData.SelectSingleNode("span[2]/span").InnerText; var priceNumber = ScrapeHelper.ConvertMetric(underPriceData); under = priceNumber != null ? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) : Math.Round((double)(100 / (priceNumber * -1) + 1), 2) : (double?)null; } Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}"); var metric = new PlayerOverUnder { MatchId = match.Id, Over = over, OverLine = overLine, Under = under, UnderLine = underLine, PlayerId = player.Id, ScoreType = scoreType, ScrapingInformationId = GetScrapingInformation().Id, CreatedAt = DateTime.Now }; PlayerUnderOvers.Add(metric); } var newProgress = GetScrapingInformation().Progress; newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange); await UpdateScrapeStatus(newProgress, null); } await UpdateScrapeStatus(currentRange, null); } } catch (Exception ex) { Logger.Error(ex.ToString()); chromeDriver.Quit(); throw; } chromeDriver.Quit(); }
protected override async Task ScrapeData() { var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); try { const string url = "https://www.nextbet.com/en/sports/227-basketball/22892-usa/44178-nba"; chromeDriver.Navigate().GoToUrl(url); await Task.Delay(10000); var doc = new HtmlDocument(); doc.LoadHtml(chromeDriver.PageSource); var matchNodes = doc.DocumentNode.SelectNodes("//span[@class='period-description']"); var rawMatches = matchNodes.Where(x => !x.InnerText.Contains("Tomorrow")).ToList(); Logger.Information("Scrape metric data"); await UpdateScrapeStatus(10, "Scraping metric data"); var rangeProgress = rawMatches.Count != 0 ? 90 / rawMatches.Count : 0; var currentRange = 10; for (var i = 0; i < rawMatches.Count; i++) { chromeDriver.Navigate().GoToUrl(url); await Task.Delay(10000); var markets = chromeDriver.FindElementsByClassName("more_markets"); markets[i].Click(); await Task.Delay(10000); doc.LoadHtml(chromeDriver.PageSource); var marketContainer = doc.DocumentNode.SelectSingleNode("//div[@id='content']"); var marketGroups = marketContainer.SelectNodes("//div[@class='markets-group-component']"); currentRange = Math.Min(currentRange + rangeProgress, 90); foreach (var marketItem in marketGroups) { string scoreTypeItem; try { scoreTypeItem = marketItem.SelectSingleNode("div/div/h2").InnerText; } catch (Exception) { continue; } var scoreType = scoreTypeItem.Contains("Points + Assists") ? ScoreType.PointAssist : scoreTypeItem.Contains("Points + Rebounds") ? ScoreType.PointRebound : scoreTypeItem.Contains("Assists + Rebounds") ? ScoreType.ReboundAssist : scoreTypeItem.Contains("Points + Assists + Rebounds") ? ScoreType.PointReboundAssist : scoreTypeItem.Contains("Points") ? ScoreType.Point : scoreTypeItem.Contains("Rebounds") ? ScoreType.Rebound : scoreTypeItem.Contains("Assists") ? ScoreType.Assist : scoreTypeItem.Contains("Three Made") ? ScoreType.ThreePoint : string.Empty; if (string.IsNullOrEmpty(scoreType) || scoreTypeItem.Contains("Odd/Even") || !scoreTypeItem.Contains("Match")) { continue; } doc.LoadHtml(marketItem.InnerHtml); var playerMarkets = doc.DocumentNode.SelectNodes("//div[@class='market-component']"); foreach (var playerMarket in playerMarkets) { var playerName = playerMarket.SelectSingleNode("div[@class='player']").InnerText .Replace("\n", string.Empty) .Replace("\r", string.Empty) .Replace("\t", string.Empty) .Trim(); var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches); if (match == null) { continue; } var player = ScrapeHelper.FindPlayerInMatch(playerName, match); var overLineItem = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td/span"); var overLineData = overLineItem.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim(); var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, "Over (.*)")); var priceOverData = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td/span[2]/span/span"); var over = ScrapeHelper.ConvertMetric(priceOverData.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim()); var underLineItem = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td[2]/span"); var underLineData = underLineItem.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim(); var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, "Under (.*)")); var priceUnderData = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td[2]/span[2]/span/span"); var under = ScrapeHelper.ConvertMetric(priceUnderData.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim()); Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}"); var metric = new PlayerOverUnder { MatchId = match.Id, Over = over, OverLine = overLine, Under = under, UnderLine = underLine, PlayerId = player.Id, ScoreType = scoreType, ScrapingInformationId = GetScrapingInformation().Id, CreatedAt = DateTime.Now }; PlayerUnderOvers.Add(metric); var newProgress = GetScrapingInformation().Progress; newProgress = Math.Min(newProgress + currentRange / playerMarkets.Count, currentRange); await UpdateScrapeStatus(newProgress, null); } } await UpdateScrapeStatus(currentRange, null); } Logger.Information("Scrape metric data complete"); await UpdateScrapeStatus(90, "Scrape metric data complete"); } catch (Exception ex) { Logger.Error(ex.ToString()); chromeDriver.Quit(); throw; } chromeDriver.Quit(); }