protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://www.5dimes.eu/";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(2000);
var userElement = chromeDriver.FindElementById("customerID");
userElement.SendKeys("5D2543319");
var passwordElement = chromeDriver.FindElementByName("password");
passwordElement.SendKeys("qwerty123!");
var loginButton = chromeDriver.FindElementById("submit1");
loginButton.Click();
await Task.Delay(2000);
var nbaCheckbox = chromeDriver.FindElementByName("Basketball_NBA Props");
nbaCheckbox.Click();
var nbaSubmitElement = chromeDriver.FindElementById("btnContinue");
nbaSubmitElement.Click();
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
var rawMatches = doc.DocumentNode.SelectNodes("//html/body/form/div[@id='PageElt']/div/div[@id='Middle']/div[@id='contentRight']/div[@id='ManagedContentWrapper']/div[@id='MainContent']/div[@id='mainContentContainer']/div[@id='readingPaneSplitPane']/div[@id='readingPaneContainer']/div[@id='readingPaneContentContainer']/div[@id='contentContainer']/div[@class='linesContainer']/table/tbody/tr");
for (var i = 0; i < rawMatches.Count; i++)
{
var tdScoreName = rawMatches[i].SelectSingleNode("td[3]");
if (tdScoreName == null)
{
continue;
}
var scoreText = tdScoreName.InnerText;
var scoreType =
scoreText.Contains("points") && !scoreText.Contains("first") && !scoreText.Contains("total") && !scoreText.Contains("wins") ? ScoreType.Point :
scoreText.Contains("assists") ? ScoreType.Assist :
scoreText.Contains("rebounds") ? ScoreType.Rebound :
scoreText.Contains("3-Pt") ? ScoreType.ThreePoint :
scoreText.Contains("pts+reb+ast") ? ScoreType.PointReboundAssist :
scoreText.Contains("pts+reb") ? ScoreType.PointRebound :
scoreText.Contains("pts+ast") ? ScoreType.PointAssist :
scoreText.Contains("reb+ast") ? ScoreType.ReboundAssist
: string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var scoreSplits = scoreText.Split();
var playerName = scoreSplits[1].Replace(".", " ");
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
var overText = rawMatches[i].SelectSingleNode("td[6]").InnerText;
var overLine = ScrapeHelper.ConvertMetric($"{ScrapeHelper.RegexMappingExpression(overText, "o(.*)½")}.05");
var over = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overText, @"½ [\-\+](.*)").Trim());
var underText = rawMatches[i + 1].SelectSingleNode("td[6]").InnerText;
var underLine = ScrapeHelper.ConvertMetric($"{ScrapeHelper.RegexMappingExpression(underText, "u(.*)½")}.05");
var under = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underText, @"½ [\-\+](.*)").Trim());
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://www.betus.com.pa/sportsbook/nba-basketball-lines.aspx";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(5000);
var matchCount = chromeDriver.FindElementsByClassName("props").Count;
await UpdateScrapeStatus(10, "Scraping metric data");
var rangeProgress = matchCount != 0 ? 90 / matchCount : 0;
var currentRange = 10;
for (var i = 0; i < matchCount; i++)
{
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(3000);
var props = chromeDriver.FindElementsByClassName("props");
props[i].Click();
await Task.Delay(3000);
var select = new SelectElement(chromeDriver.FindElementByClassName("odds-display"));
select.SelectByText("Decimal");
await Task.Delay(2000);
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var matchNode = doc.DocumentNode.SelectSingleNode("//html/body/form/div[@class='sportsbook-visitor']/div[@class='sportsbook-master']/div[@class='rounded-block-white']/span[2]/div[@class='col']/div[@id='game-lines']/div/div[@class='game-block']/div[@class='normal']/div/div[@class='future-lines inline-prop']");
var rawMetrics = matchNode.SelectNodes("table");
foreach (var rawMetric in rawMetrics)
{
var playerNode = rawMetric.SelectSingleNode("tbody/tr[1]/th[2]").InnerText;
var scoreType =
playerNode.Contains("Total Points+Rebounds+Assists") ? ScoreType.PointReboundAssist :
playerNode.Contains("Total Points+Rebounds") ? ScoreType.PointRebound :
playerNode.Contains("Total Points+Assists") ? ScoreType.PointAssist :
playerNode.Contains("Total Rebounds+Assists") ? ScoreType.ReboundAssist :
playerNode.Contains("Total Points") ? ScoreType.Point :
playerNode.Contains("Total Rebounds") ? ScoreType.Rebound :
playerNode.Contains("Total Assists") ? ScoreType.Assist :
playerNode.Contains("Total Made 3") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
playerNode = playerNode
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\t", string.Empty)
.Trim();
var playerName = ScrapeHelper.RegexMappingExpression(playerNode, @"(.*) \(");
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
var overNode = rawMetric.SelectSingleNode("tbody/tr[2]/td[2]").InnerText;
overNode = overNode
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\t", string.Empty)
.Replace(" ", " ")
.Trim();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overNode, @"Over.(\d*.\d*)"));
var overPriceNode = rawMetric.SelectSingleNode("tbody/tr[2]/td[3]/a").InnerText;
var over = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overPriceNode, @"(\d*.\d*).*"));
var underNode = rawMetric.SelectSingleNode("tbody/tr[3]/td[2]").InnerText;
underNode = underNode
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace(" ", " ")
.Replace("\t", string.Empty)
.Trim();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underNode, @"Under.(\d*.\d*)"));
var underPriceNode = rawMetric.SelectSingleNode("tbody/tr[3]/td[3]/a").InnerText;
var under = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underPriceNode, @"(\d*.\d*).*"));
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://nj.betamerica.com/sports/basketball/nba/";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(20000);
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
var rawMatches = doc.DocumentNode.SelectNodes("/html/body/div[4]/div/div[2]/div/section[7]/div[2]/div[2]/div/sb-comp/div[1]/div/sb-lazy-render/div[*]");
var sourceIds = rawMatches
.Select(rawMatch => rawMatch.SelectSingleNode("div/a"))
.Select(aTag => aTag.GetAttributeValue("href", string.Empty))
.Where(sourceId => !sourceId.Contains("live-betting")).ToList();
await UpdateScrapeStatus(10, "Scraping metric data");
var rangeProgress = sourceIds.Count != 0 ? 90 / sourceIds.Count : 0;
var currentRange = 10;
foreach (var matchUrl in sourceIds.Select(sourceId => $"https://nj.betamerica.com{sourceId}"))
{
chromeDriver.Navigate().GoToUrl(matchUrl);
await Task.Delay(7000);
doc.LoadHtml(chromeDriver.PageSource);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = doc.DocumentNode.SelectNodes("//html/body/div[@class='content-main']/div[contains(@class, 'content-main-inner')]/div[@id='pagesWrapper']/div[@id='panel-center-inner']/section[@id='pre-live-betting']/div/div[@class='event-view-views-switcher']/div/ul");
foreach (var rawMetric in rawMetrics)
{
var liTags = rawMetric.SelectNodes("li");
foreach (var liTag in liTags)
{
var spanItem = liTag.SelectSingleNode("h4/span[2]/span");
var spanText = spanItem.InnerText;
var scoreType =
spanText.Contains("Total Points") ? ScoreType.Point :
spanText.Contains("Total Rebounds") ? ScoreType.Rebound :
spanText.Contains("Total Assists") ? ScoreType.Assist :
spanText.Contains("Pts, Rebs, Asts") ? ScoreType.PointReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(spanText, @"(.*) (Total|\(Pts,)");
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overUnderFirstData = liTag.SelectSingleNode("div/div/div[2]/button");
var overUnderSecondData = liTag.SelectSingleNode("div/div/div[1]/button");
double?over = 0, overLine = 0, under = 0, underLine = 0;
var overUnderFirstLine = overUnderFirstData.SelectSingleNode("span[1]/span[1]").InnerText;
if (overUnderFirstLine.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderFirstLine, @"Over (\d*.\d*) .*"));
var overPriceData = overUnderFirstData.SelectSingleNode("span[2]/span").InnerText;
var priceNumber = ScrapeHelper.ConvertMetric(overPriceData);
over = priceNumber != null
? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) :
Math.Round((double)(100 / (priceNumber * -1) + 1), 2)
: (double?)null;
}
else
{
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderFirstLine, @"Under (\d*.\d*) .*"));
var underPriceData = overUnderFirstData.SelectSingleNode("span[2]/span").InnerText;
var priceNumber = ScrapeHelper.ConvertMetric(underPriceData);
under = priceNumber != null
? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) :
Math.Round((double)(100 / (priceNumber * -1) + 1), 2)
: (double?)null;
}
var overUnderSecondLine = overUnderSecondData.SelectSingleNode("span[1]/span[1]").InnerText;
if (overUnderSecondLine.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderSecondLine, @"Over (\d*.\d*) .*"));
var overPriceData = overUnderSecondData.SelectSingleNode("span[2]/span").InnerText;
var priceNumber = ScrapeHelper.ConvertMetric(overPriceData);
over = priceNumber != null
? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) :
Math.Round((double)(100 / (priceNumber * -1) + 1), 2)
: (double?)null;
}
else
{
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderSecondLine, @"Under (\d*.\d*) .*"));
var underPriceData = overUnderSecondData.SelectSingleNode("span[2]/span").InnerText;
var priceNumber = ScrapeHelper.ConvertMetric(underPriceData);
under = priceNumber != null
? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) :
Math.Round((double)(100 / (priceNumber * -1) + 1), 2)
: (double?)null;
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://www.nextbet.com/en/sports/227-basketball/22892-usa/44178-nba";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(10000);
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
var matchNodes = doc.DocumentNode.SelectNodes("//span[@class='period-description']");
var rawMatches = matchNodes.Where(x => !x.InnerText.Contains("Tomorrow")).ToList();
Logger.Information("Scrape metric data");
await UpdateScrapeStatus(10, "Scraping metric data");
var rangeProgress = rawMatches.Count != 0 ? 90 / rawMatches.Count : 0;
var currentRange = 10;
for (var i = 0; i < rawMatches.Count; i++)
{
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(10000);
var markets = chromeDriver.FindElementsByClassName("more_markets");
markets[i].Click();
await Task.Delay(10000);
doc.LoadHtml(chromeDriver.PageSource);
var marketContainer = doc.DocumentNode.SelectSingleNode("//div[@id='content']");
var marketGroups = marketContainer.SelectNodes("//div[@class='markets-group-component']");
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var marketItem in marketGroups)
{
string scoreTypeItem;
try
{
scoreTypeItem = marketItem.SelectSingleNode("div/div/h2").InnerText;
}
catch (Exception)
{
continue;
}
var scoreType =
scoreTypeItem.Contains("Points + Assists") ? ScoreType.PointAssist :
scoreTypeItem.Contains("Points + Rebounds") ? ScoreType.PointRebound :
scoreTypeItem.Contains("Assists + Rebounds") ? ScoreType.ReboundAssist :
scoreTypeItem.Contains("Points + Assists + Rebounds") ? ScoreType.PointReboundAssist :
scoreTypeItem.Contains("Points") ? ScoreType.Point :
scoreTypeItem.Contains("Rebounds") ? ScoreType.Rebound :
scoreTypeItem.Contains("Assists") ? ScoreType.Assist :
scoreTypeItem.Contains("Three Made") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType) || scoreTypeItem.Contains("Odd/Even") ||
!scoreTypeItem.Contains("Match"))
{
continue;
}
doc.LoadHtml(marketItem.InnerHtml);
var playerMarkets = doc.DocumentNode.SelectNodes("//div[@class='market-component']");
foreach (var playerMarket in playerMarkets)
{
var playerName = playerMarket.SelectSingleNode("div[@class='player']").InnerText
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\t", string.Empty)
.Trim();
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
var overLineItem = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td/span");
var overLineData = overLineItem.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, "Over (.*)"));
var priceOverData = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td/span[2]/span/span");
var over = ScrapeHelper.ConvertMetric(priceOverData.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim());
var underLineItem = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td[2]/span");
var underLineData = underLineItem.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, "Under (.*)"));
var priceUnderData = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td[2]/span[2]/span/span");
var under = ScrapeHelper.ConvertMetric(priceUnderData.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim());
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / playerMarkets.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}