protected override async Task ScrapeData()
{
const string url = "https://cds-api.borgataonline.com/bettingoffer/fixtures?x-bwin-accessid=ZTJhZDliYjgtNTdmOC00Njk0LWIxZmItODI3YzhjZGQ5NmIx&lang=en-US&country=VN&userCountry=VN&streamProviders=unas%2Cperform%2Cimgdge&fixtureTypes=Standard&state=Latest&skip=0&take=50&offerMapping=Filtered&offerCategories=Gridable&fixtureCategories=Gridable,NonGridable,Other&sortBy=Tags&sportIds=7®ionIds=9&competitionIds=6004";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.fixtures[*]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var participants = rawMatch.SelectTokens("$.participants[*]").ToList();
var homeTeam = participants.Last().SelectToken("$.name.value").ToString();
var awayTeam = participants.First().SelectToken("$.name.value").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://cds-api.borgataonline.com/bettingoffer/fixture-view?x-bwin-accessid=ZTJhZDliYjgtNTdmOC00Njk0LWIxZmItODI3YzhjZGQ5NmIx&lang=en-US&country=VN&userCountry=VN&streamProviders=unas%2Cperform%2Cimgdge&offerMapping=All&scoreboardMode=Full&fixtureIds={match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc.SelectTokens("$.fixture.games[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var nameData = rawMetric.SelectToken("$.name.value").ToString();
var scoreType =
nameData.Contains("How many points") ? ScoreType.Point :
nameData.Contains("How many assists") ? ScoreType.Assist :
nameData.Contains("How many rebounds") ? ScoreType.Rebound :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(nameData, @".* will (.*) \(");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overData = rawMetric.SelectToken("$.results[0]");
var over = ScrapeHelper.ConvertMetric(overData.SelectToken("$.odds").ToString());
var overLineData = overData.SelectToken("$.name.value").ToString();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, "Over (.*)"));
var underData = rawMetric.SelectToken("$.results[0]");
var under = ScrapeHelper.ConvertMetric(underData.SelectToken("$.odds").ToString());
var underLineData = underData.SelectToken("$.name.value").ToString();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, "Over (.*)"));
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://eu-offering.kambicdn.org/offering/v2018/888/listView/basketball.json?lang=en_GB&market=ZZ&client_id=2&channel_id=1&ncid=1572482733163&useCombined=true";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.events[?(@.event.group == 'NBA')]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.event.id").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.event.homeName").ToString();
var awayTeam = rawMatch.SelectToken("$.event.awayName").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
foreach (var match in foundMatches)
{
var metricUrl = $"https://eu-offering.kambicdn.org/offering/v2018/888/betoffer/event/{match.SourceId}.json?lang=en_GB&market=ZZ&client_id=2&channel_id=1&ncid=1005710980&includeParticipants=true";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc.SelectTokens("$.betOffers[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var labelData = rawMetric.SelectToken("$.criterion.label").ToString();
var scoreType = labelData.Contains("Points scored by the player") ? ScoreType.Point :
labelData.Contains("Rebounds by the player") ? ScoreType.Rebound :
labelData.Contains("Assists by the player") ? ScoreType.Assist :
labelData.Contains("Points, rebounds & assists by the player") ? ScoreType.PointReboundAssist :
labelData.Contains("3-point field goals made by the player") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var overData = rawMetric.SelectToken("$.outcomes[?(@.label =~ /(Over.*)/)]");
var underData = rawMetric.SelectToken("$.outcomes[?(@.label =~ /(Under.*)/)]");
var playerData = overData.SelectToken("$.participant").ToString();
var playerSplits = playerData.Split(',');
var rvPlayer = playerSplits.Reverse().ToList();
var playerName = string.Join(" ", rvPlayer);
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overLineData = overData.SelectToken("$.label").ToString();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, @"Over.* (\d*.\d*)"));
var overOdd = ScrapeHelper.ConvertMetric(overData.SelectToken("$.odds").ToString());
var over = overOdd / 1000;
var underLineData = underData.SelectToken("$.label").ToString();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, @"Under.* (\d*.\d*)"));
var underOdd = ScrapeHelper.ConvertMetric(underData.SelectToken("$.odds").ToString());
var under = underOdd / 1000;
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://services.bovada.lv/services/sports/event/coupon/events/A/description/basketball/nba?marketFilterId=def&preMatchOnly=true&lang=en";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.[*].events[*]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.link").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var description = rawMatch.SelectToken("$.description").ToString();
var desSplits = description.Split('@');
var homeTeam = desSplits[1];
var awayTeam = desSplits[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://services.bovada.lv/services/sports/event/coupon/events/A/description{match.SourceId}?lang=en";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc
.SelectTokens("$.[*].events[*].displayGroups[?(@.description == 'Player Props')].markets[*]")
.ToList();
foreach (var rawMetric in rawMetrics)
{
var description = rawMetric.SelectToken("$.description").ToString();
var scoreType =
description.Contains("Total Points, Rebounds and Assists") ? ScoreType.PointReboundAssist :
description.Contains("Total Points and Rebounds") ? ScoreType.PointRebound :
description.Contains("Total Points and Assists") ? ScoreType.PointAssist :
description.Contains("Total Rebounds and Assists") ? ScoreType.ReboundAssist :
description.Contains("Total Points") ? ScoreType.Point :
description.Contains("Total Rebounds") ? ScoreType.Rebound :
description.Contains("Total Assists") ? ScoreType.Assist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(description, @"- (.*) \(");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var outcomes = rawMetric.SelectTokens("$.outcomes[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcome in outcomes)
{
var des = outcome.SelectToken("$.description").ToString();
if (des.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.handicap").ToString());
var overPrice = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.decimal").ToString());
over = overPrice != null?Math.Round((double)overPrice, 2) : (double?)null;
}
else if (des.Contains("Under"))
{
underLine = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.handicap").ToString());
var underPrice = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.decimal").ToString());
under = underPrice != null?Math.Round((double)underPrice, 2) : (double?)null;
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://www.betvictor.com/api/top_bets/601600?number_to_show=20&exclude_in_running=true&event_ids_to_exclude=&sport_ids_to_exclude=";
var rawDoc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
await UpdateScrapeStatus(10, "Scraping match data");
var rawMatches = jDoc.SelectTokens("$.[?(@.meeting_description == 'NBA')]");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.event_id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
if (foundMatches.Select(x => x.SourceId).Contains(sourceMatchId))
{
continue;
}
var game = rawMatch.SelectToken("$.event_description").ToString();
var clubs = game.Split('@');
var homeTeam = clubs[1];
var awayTeam = clubs[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
rawDoc = await ScrapeHelper.GetDocument($"https://www.betvictor.com/bv_event_level/en-gb/1/coupons/{match.SourceId}/4787?t=1573192805056");
var jResult = JsonConvert.DeserializeObject <JToken>(rawDoc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jResult.SelectTokens("$.[*].markets[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var description = rawMetric.SelectToken("$.des").ToString();
var scoreType =
description.Contains("Total combined points, assists & rebounds") ? ScoreType.PointReboundAssist :
description.Contains("Total points") ? ScoreType.Point :
description.Contains("Total assists") ? ScoreType.Assist :
description.Contains("Total three pointers") ? ScoreType.ThreePoint :
description.Contains("Total rebounds") ? ScoreType.Rebound :
description.Contains("Total combined points & rebounds") ? ScoreType.PointRebound :
description.Contains("Total combined points & assists") ? ScoreType.PointAssist :
description.Contains("Total combined assists & rebounds") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = rawMetric.SelectToken("$.opponentDescription").ToString();
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var playerMetrics = rawMetric.SelectTokens("$.o[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var metricItem in playerMetrics)
{
var des = metricItem.SelectToken("$.des").ToString();
if (des.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(des, "Over (.*)"));
over = ScrapeHelper.ConvertMetric(metricItem.SelectToken("$.pr").ToString());
}
else if (des.Contains("Under"))
{
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(des, "Under (.*)"));
under = ScrapeHelper.ConvertMetric(metricItem.SelectToken("$.pr").ToString());
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://api.pointsbet.com/api/v2/competitions/7176/events/featured?includeLive=true";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.events[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.key").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.homeTeam").ToString();
var awayTeam = rawMatch.SelectToken("$.awayTeam").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://api.pointsbet.com/api/v2/events/{match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc
.SelectTokens(
@"$.fixedOddsMarkets[?(@.name =~ /(Player Points Over\/Under .*)/ || @.name =~ /(Player Rebounds Over\/Under .*)/ || @.name =~ /(Player Assists Over\/Under .*)/)]")
.ToList();
foreach (var rawMetric in rawMetrics)
{
var metricName = rawMetric.SelectToken("$.name").ToString();
var scoreType =
metricName.Contains("Player Points") ? ScoreType.Point :
metricName.Contains("Player Assists") ? ScoreType.Assist :
metricName.Contains("Player Rebounds") ? ScoreType.Rebound :
metricName.Contains("Player Pts + Rebs + Asts") ? ScoreType.PointReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var outcomeOvers = rawMetric.SelectTokens("$.outcomes[?(@.name =~ /(.* Over .*)/)]").ToList();
var outcomeUnders = rawMetric.SelectTokens("$.outcomes[?(@.name =~ /(.* Under .*)/)]").ToList();
var playerNames = new HashSet <string>();
foreach (var name in outcomeOvers.Select(outcome => outcome.SelectToken("name").ToString()))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(name, "(.*)(?:Over|Under)"));
}
foreach (var name in outcomeUnders.Select(outcome => outcome.SelectToken("name").ToString()))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(name, "(.*)(?:Over|Under)"));
}
foreach (var playerName in playerNames)
{
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var outcomeOver = outcomeOvers.FirstOrDefault(x =>
x.SelectToken("$.name").ToString().Contains(playerName));
var outcomeUnder = outcomeUnders.FirstOrDefault(x =>
x.SelectToken("$.name").ToString().Contains(playerName));
var rawOver = outcomeOver?.SelectToken("price").ToString();
var rawOverLine =
ScrapeHelper.RegexMappingExpression(outcomeOver?.SelectToken("$.name").ToString(),
"(?:Over) (.*)");
var rawUnder = outcomeUnder?.SelectToken("price").ToString();
var rawUnderLine =
ScrapeHelper.RegexMappingExpression(outcomeUnder?.SelectToken("$.name").ToString(),
"(?:Under) (.*)");
Logger.Information($"{player.Name}: {scoreType} - {rawOver} {rawOverLine} | {rawUnder} {rawUnderLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = ScrapeHelper.ConvertMetric(rawOver),
OverLine = ScrapeHelper.ConvertMetric(rawOverLine),
Under = ScrapeHelper.ConvertMetric(rawUnder),
UnderLine = ScrapeHelper.ConvertMetric(rawUnderLine),
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://betbuilder.digitalsportstech.com/api/latestGames?leagueId=123&sb=topsport&status=1";
var rawDoc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
var rawMatches = jDoc.SelectTokens("$.data[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.homeTeam.title").ToString();
var awayTeam = rawMatch.SelectToken("$.visitingTeam.title").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scraping metric data");
_rawData = new List <PlayerOverUnder>();
foreach (var match in foundMatches)
{
var metricUrl = $"https://betbuilder.digitalsportstech.com/api/feed?betType=in,18,19&gameId={match.SourceId}&limit=9999&sb=topsport&tz=7";
rawDoc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
var rawMetrics = jDoc.SelectTokens("$.data[*]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 80);
foreach (var rawMetric in rawMetrics)
{
var statistic = rawMetric.SelectToken("$.statistic.title").ToString();
var scoreType =
statistic.Contains("Pts + Reb + Ast") ? ScoreType.PointReboundAssist :
statistic.Contains("Total Rebounds") ? ScoreType.Rebound :
statistic.Contains("Assists") ? ScoreType.Assist :
statistic.Contains("Points") ? ScoreType.Point :
statistic.Contains("Three Point") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = rawMetric.SelectToken("$.player1.name").ToString();
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var betType = rawMetric.SelectToken("$.betType").ToString();
double?over = 0, overLine = 0, under = 0, underLine = 0;
switch (betType)
{
case "18":
over = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.markets[0].odds").ToString());
overLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.markets[0].value").ToString());
break;
case "19":
under = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.markets[0].odds").ToString());
underLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.markets[0].value").ToString());
break;
}
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
Player = player,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
_rawData.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
MergeData();
Logger.Information("Merge data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://www.5dimes.eu/";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(2000);
var userElement = chromeDriver.FindElementById("customerID");
userElement.SendKeys("5D2543319");
var passwordElement = chromeDriver.FindElementByName("password");
passwordElement.SendKeys("qwerty123!");
var loginButton = chromeDriver.FindElementById("submit1");
loginButton.Click();
await Task.Delay(2000);
var nbaCheckbox = chromeDriver.FindElementByName("Basketball_NBA Props");
nbaCheckbox.Click();
var nbaSubmitElement = chromeDriver.FindElementById("btnContinue");
nbaSubmitElement.Click();
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
var rawMatches = doc.DocumentNode.SelectNodes("//html/body/form/div[@id='PageElt']/div/div[@id='Middle']/div[@id='contentRight']/div[@id='ManagedContentWrapper']/div[@id='MainContent']/div[@id='mainContentContainer']/div[@id='readingPaneSplitPane']/div[@id='readingPaneContainer']/div[@id='readingPaneContentContainer']/div[@id='contentContainer']/div[@class='linesContainer']/table/tbody/tr");
for (var i = 0; i < rawMatches.Count; i++)
{
var tdScoreName = rawMatches[i].SelectSingleNode("td[3]");
if (tdScoreName == null)
{
continue;
}
var scoreText = tdScoreName.InnerText;
var scoreType =
scoreText.Contains("points") && !scoreText.Contains("first") && !scoreText.Contains("total") && !scoreText.Contains("wins") ? ScoreType.Point :
scoreText.Contains("assists") ? ScoreType.Assist :
scoreText.Contains("rebounds") ? ScoreType.Rebound :
scoreText.Contains("3-Pt") ? ScoreType.ThreePoint :
scoreText.Contains("pts+reb+ast") ? ScoreType.PointReboundAssist :
scoreText.Contains("pts+reb") ? ScoreType.PointRebound :
scoreText.Contains("pts+ast") ? ScoreType.PointAssist :
scoreText.Contains("reb+ast") ? ScoreType.ReboundAssist
: string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var scoreSplits = scoreText.Split();
var playerName = scoreSplits[1].Replace(".", " ");
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
var overText = rawMatches[i].SelectSingleNode("td[6]").InnerText;
var overLine = ScrapeHelper.ConvertMetric($"{ScrapeHelper.RegexMappingExpression(overText, "o(.*)½")}.05");
var over = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overText, @"½ [\-\+](.*)").Trim());
var underText = rawMatches[i + 1].SelectSingleNode("td[6]").InnerText;
var underLine = ScrapeHelper.ConvertMetric($"{ScrapeHelper.RegexMappingExpression(underText, "u(.*)½")}.05");
var under = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underText, @"½ [\-\+](.*)").Trim());
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
const string url = "https://fixture.palmerbet.online/fixtures/sports/1c2eeb3a-6bab-4ac2-b434-165cc350180f/matches?pageSize=25";
var rawResult = await ScrapeHelper.GetDocument(url);
var matchObj = JsonConvert.DeserializeObject <JToken>(rawResult);
var rawMatches = matchObj.SelectTokens("$.matches[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.eventId").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.homeTeam.title").ToString();
var awayTeam = rawMatch.SelectToken("$.awayTeam.title").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rawMetricsTasks = foundMatches
.Select(match => $"https://fixture.palmerbet.online/fixtures/sports/matches/{match.SourceId}/markets?pageSize=1000")
.Select(metricUrl => ScrapeHelper.GetDocument(metricUrl))
.ToList();
var rawMetrics = await Task.WhenAll(rawMetricsTasks);
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = rawMetrics.Length != 0 ? 90 / rawMetrics.Length : 0;
var currentRange = 20;
Logger.Information($"Total raw metrics count {rawMetrics.Length}");
var totalMetrics = 0;
try
{
for (var i = 0; i < rawMetrics.Length; i++)
{
var rawMetric = rawMetrics[i];
var metricObj = JsonConvert.DeserializeObject <JToken>(rawMetric);
var metricMarkets = metricObj.SelectTokens("$.markets[?(@.title=~ /(.* - .*)/)]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var metricMarket in metricMarkets)
{
var title = metricMarket.SelectToken("$.title").ToString();
var scoreType =
title.Contains("Points") ? ScoreType.Point :
title.Contains("Rebounds") ? ScoreType.Rebound :
title.Contains("Assists") ? ScoreType.Assist :
title.Contains("Made Threes") ? ScoreType.ThreePoint :
title.Contains("Pts + Ast") ? ScoreType.PointAssist :
title.Contains("Pts + Reb + Ast") ? ScoreType.PointReboundAssist :
title.Contains("Pts + Reb") ? ScoreType.PointRebound :
title.Contains("Reb + Ast") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = title.Split('-')[0].Trim();
var player = ScrapeHelper.FindPlayerInMatch(playerName, foundMatches[i]);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {foundMatches[i].Id}");
continue;
}
totalMetrics++;
var playerId = metricMarket.SelectToken("$.id").ToString();
var priceUrl = $"https://fixture.palmerbet.online/fixtures/sports/markets/{playerId}";
var priceDoc = await ScrapeHelper.GetDocument(priceUrl, 3000);
var priceObj = JsonConvert.DeserializeObject <JToken>(priceDoc);
if (priceObj == null)
{
Logger.Warning($"priceObj is null for player: {playerId}");
continue;
}
var outcomeItems = priceObj.SelectTokens("$.market.outcomes[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcomeItem in outcomeItems)
{
var titleItem = outcomeItem.SelectToken("$.title").ToString();
if (titleItem.Contains("Over"))
{
var rawOver = ScrapeHelper.RegexMappingExpression(titleItem, "(?:Over) (.*)");
var rawPrice = outcomeItem.SelectTokens("$.prices[*]").FirstOrDefault()
.SelectToken("$.priceSnapshot.current").ToString();
overLine = ScrapeHelper.ConvertMetric(rawOver);
over = ScrapeHelper.ConvertMetric(rawPrice);
}
else if (titleItem.Contains("Under"))
{
var rawUnder = ScrapeHelper.RegexMappingExpression(titleItem, "(?:Under) (.*)");
var rawPrice = outcomeItem.SelectTokens("$.prices[*]").FirstOrDefault()
.SelectToken("$.priceSnapshot.current").ToString();
underLine = ScrapeHelper.ConvertMetric(rawUnder);
under = ScrapeHelper.ConvertMetric(rawPrice);
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = foundMatches[i].Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / metricMarkets.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
Logger.Information($"Total metric count: {totalMetrics}");
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
throw;
}
}
protected override async Task ScrapeData()
{
const string url = "https://api.beta.tab.com.au/v1/tab-info-service/sports/Basketball/competitions/NBA/matches?jurisdiction=VIC";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.matches[*]");
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$._links.self").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var matchName = rawMatch.SelectToken("$.name").ToString();
var homeTeam = ScrapeHelper.RegexMappingExpression(matchName, "(.*) v .*");
var awayTeam = ScrapeHelper.RegexMappingExpression(matchName, ".* v (.*)");
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
doc = await ScrapeHelper.GetDocument($"{match.SourceId}");
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMetrics = jDoc.SelectTokens("$.markets[?(@.betOptionSpectrumId =~ /(837|838|839|1789|1791)/)]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var rawMetric in rawMetrics)
{
var spectrumId = rawMetric.SelectToken("$.betOptionSpectrumId").ToString();
var scoreType = spectrumId == "837" ? ScoreType.Point :
spectrumId == "838" ? ScoreType.Rebound :
spectrumId == "839" ? ScoreType.Assist :
spectrumId == "1789" ? ScoreType.PointReboundAssist :
spectrumId == "1791" ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerOvers = rawMetric.SelectTokens("$.propositions[?(@.name =~ /(.* Over .*)/)]").ToList();
var playerUnders = rawMetric.SelectTokens("$.propositions[?(@.name =~ /(.* Under .*)/)]").ToList();
var playerNames = new HashSet <string>();
foreach (var playerName in playerOvers.Select(item => item.SelectToken("name").ToString()).Select(itemName => ScrapeHelper.RegexMappingExpression(itemName, "(.*)(?:Over|Under)")))
{
playerNames.Add(playerName);
}
foreach (var playerName in playerUnders.Select(item => item.SelectToken("name").ToString()).Select(itemName => ScrapeHelper.RegexMappingExpression(itemName, "(.*)(?:Over|Under)")))
{
playerNames.Add(playerName);
}
foreach (var playerName in playerNames)
{
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overOutcomeItem = playerOvers.FirstOrDefault(x => x.SelectToken("$.name").ToString().Contains(playerName));
var overName = overOutcomeItem?.SelectToken("name").ToString() ?? string.Empty;
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overName, @".* (\d*.\d*) .*"));
var over = ScrapeHelper.ConvertMetric(overOutcomeItem?.SelectToken("returnWin").ToString() ?? string.Empty);
var underOutcomeItem = playerUnders.FirstOrDefault(x => x.SelectToken("$.name").ToString().Contains(playerName));
var underName = underOutcomeItem?.SelectToken("name").ToString() ?? string.Empty;
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underName, @".* (\d*.\d*) .*"));
var under = ScrapeHelper.ConvertMetric(underOutcomeItem?.SelectToken("returnWin").ToString() ?? string.Empty);
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://beteasy.com.au/api/sports/sports?filters%5BEventTypes%5D=107&filters%5BMasterCategoryID%5D=37";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(10000);
var pageSource = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var jDocument = JsonConvert.DeserializeObject <JToken>(pageSource);
var rawMatches = jDocument.SelectTokens("$.result.sports.events[*]");
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.MasterEventID").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source match id is null");
continue;
}
var eventName = rawMatch.SelectToken("$.MasterEventName").ToString();
if (string.IsNullOrEmpty(eventName))
{
Logger.Warning("Even name is null");
continue;
}
var homeTeam = eventName.Split('@')[1];
var awayTeam = eventName.Split('@')[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
Logger.Information("Scrape matches done");
await UpdateScrapeStatus(20, "Scrape single metrics data");
var rangeProgress = foundMatches.Count != 0 ? 55 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape single metrics data");
foreach (var match in foundMatches)
{
var singleMetricUrl =
$"https://beteasy.com.au/api/sports/event-group/?id={match.SourceId}&ecGroupOrderByIds%5B%5D=18";
chromeDriver.Navigate().GoToUrl(singleMetricUrl);
await Task.Delay(5000);
currentRange = Math.Min(currentRange + rangeProgress, 55);
var metricPageSource = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var jRawMetrics = JsonConvert.DeserializeObject <JToken>(metricPageSource);
var rawMetrics = jRawMetrics.SelectTokens("$.result.*.BettingType[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var eventName = rawMetric.SelectToken("$.EventName").ToString();
var scoreType =
eventName.Contains("Points") ? ScoreType.Point :
eventName.Contains("Rebounds") ? ScoreType.Rebound :
eventName.Contains("Assists") ? ScoreType.Assist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(eventName, "(.*) (Points|Rebounds|Assists)");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var outcomes = rawMetric.SelectTokens("$.Outcomes[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcome in outcomes)
{
var outcomeName = outcome.SelectToken("$.OutcomeName").ToString();
if (outcomeName.Contains("Over"))
{
var rawOver = ScrapeHelper.RegexMappingExpression(outcomeName, "Over (.*)");
var rawPrice = outcome.SelectToken("$.BetTypes[0]").SelectToken("$.Price").ToString();
overLine = ScrapeHelper.ConvertMetric(rawOver);
over = ScrapeHelper.ConvertMetric(rawPrice);
}
else if (outcomeName.Contains("Under"))
{
var rawUnder = ScrapeHelper.RegexMappingExpression(outcomeName, "Under (.*)");
var rawPrice = outcome.SelectToken("$.BetTypes[0]").SelectToken("$.Price").ToString();
underLine = ScrapeHelper.ConvertMetric(rawUnder);
under = ScrapeHelper.ConvertMetric(rawPrice);
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape single metrics data complete");
await UpdateScrapeStatus(55, "Scrape single metrics data complete");
Logger.Information("Scrape combination metrics data");
rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
currentRange = 55;
await UpdateScrapeStatus(55, "Scrape combination metrics data");
foreach (var match in foundMatches)
{
var combinationMetricUrl =
$"https://beteasy.com.au/api/sports/event-group/?id={match.SourceId}&ecGroupOrderByIds%5B%5D=24";
chromeDriver.Navigate().GoToUrl(combinationMetricUrl);
await Task.Delay(5000);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var metricPageSource = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var jRawMetrics = JsonConvert.DeserializeObject <JToken>(metricPageSource);
var rawMetrics = jRawMetrics.SelectTokens("$.result.*.BettingType[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var eventName = rawMetric.SelectToken("$.EventName").ToString();
var scoreType =
eventName.Contains("Points + Rebounds + Assists") ? ScoreType.PointReboundAssist :
eventName.Contains("Points + Rebounds") ? ScoreType.PointRebound :
eventName.Contains("Points + Assists") ? ScoreType.PointAssist :
eventName.Contains("Rebounds + Assists") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var outcomes = rawMetric.SelectTokens("$.Outcomes[*]").ToList();
Player player = null;
foreach (var outcomeName in outcomes.Select(outcome => outcome.SelectToken("$.OutcomeName").ToString()))
{
if (outcomeName.Contains("Over"))
{
var playerName = ScrapeHelper.RegexMappingExpression(outcomeName, "(.*) Over");
player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player != null)
{
continue;
}
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
}
else if (outcomeName.Contains("Under"))
{
var playerName = ScrapeHelper.RegexMappingExpression(outcomeName, "(.*) Under");
player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player != null)
{
continue;
}
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
}
}
if (player == null)
{
continue;
}
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcome in outcomes)
{
var outcomeName = outcome.SelectToken("$.OutcomeName").ToString();
if (outcomeName.Contains("Over"))
{
var rawOver = ScrapeHelper.RegexMappingExpression(outcomeName, "Over (.*)");
var rawPrice = outcome.SelectToken("$.BetTypes[0]").SelectToken("$.Price").ToString();
overLine = string.IsNullOrEmpty(rawOver) ? (double?)null : Convert.ToDouble(rawOver);
over = string.IsNullOrEmpty(rawPrice) ? (double?)null : Convert.ToDouble(rawPrice);
}
else if (outcomeName.Contains("Under"))
{
var rawUnder = ScrapeHelper.RegexMappingExpression(outcomeName, "Under (.*)");
var rawPrice = outcome.SelectToken("$.BetTypes[0]").SelectToken("$.Price").ToString();
underLine = string.IsNullOrEmpty(rawUnder) ? (double?)null : Convert.ToDouble(rawUnder);
under = string.IsNullOrEmpty(rawPrice) ? (double?)null : Convert.ToDouble(rawPrice);
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape combination metrics data complete");
await UpdateScrapeStatus(90, "Scrape combination metrics data complete");
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://nodejs08.tglab.io/cache/20/en/us/11624/prematch-by-tournaments.json";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(2000);
var pageSource = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var jDocument = JsonConvert.DeserializeObject <JToken>(pageSource);
var rawMatches = jDocument.SelectTokens("$.events[*]");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.teams.home").ToString();
var awayTeam = rawMatch.SelectToken("$.teams.away").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
var tempMetrics = new List <PlayerOverUnder>();
foreach (var match in foundMatches)
{
var metricUrl = $"https://nodejs08.tglab.io/cache/20/en/us/{match.SourceId}/single-pre-event.json?hidenseek=95183b1a46914705952f351b5bc156fd8e1e7b57be65d492e09186129ff70e8a1ba8";
chromeDriver.Navigate().GoToUrl(metricUrl);
await Task.Delay(2000);
var pageSourceMarket = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var marketData = JsonConvert.DeserializeObject <JToken>(pageSourceMarket);
var rawMetrics = marketData.SelectTokens("$.odds.*");
foreach (var rawMetric in rawMetrics)
{
var teamName = rawMetric.SelectToken("$.team_name.en").ToString();
if (!teamName.Contains("total points"))
{
continue;
}
var playerSplits = ScrapeHelper.RegexMappingExpression(teamName, "(.*) total").Split('.');
var playerName = $"{playerSplits[0]} {playerSplits[1]}";
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var metric = tempMetrics.FirstOrDefault(x => x.PlayerId == player.Id);
if (teamName.Contains("OVER"))
{
if (metric != null)
{
metric.OverLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.additional_value_raw").ToString());
metric.Over = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.odd_value").ToString());
}
else
{
var newMetric = new PlayerOverUnder
{
MatchId = match.Id,
PlayerId = player.Id,
OverLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.additional_value_raw").ToString()),
Over = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.odd_value").ToString()),
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
tempMetrics.Add(newMetric);
}
}
else if (teamName.Contains("UNDER"))
{
if (metric != null)
{
metric.UnderLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.additional_value_raw").ToString());
metric.Under = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.odd_value").ToString());
}
else
{
var newMetric = new PlayerOverUnder
{
MatchId = match.Id,
PlayerId = player.Id,
UnderLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.additional_value_raw").ToString()),
Under = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.odd_value").ToString()),
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
tempMetrics.Add(newMetric);
}
}
}
}
foreach (var metric in tempMetrics)
{
Logger.Information($"{metric.Player.Name}: " +
$"{metric.Over} {metric.OverLine} | " +
$"{metric.Under} {metric.UnderLine}");
}
PlayerUnderOvers.AddRange(tempMetrics);
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://nj.betamerica.com/sports/basketball/nba/";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(20000);
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
var rawMatches = doc.DocumentNode.SelectNodes("/html/body/div[4]/div/div[2]/div/section[7]/div[2]/div[2]/div/sb-comp/div[1]/div/sb-lazy-render/div[*]");
var sourceIds = rawMatches
.Select(rawMatch => rawMatch.SelectSingleNode("div/a"))
.Select(aTag => aTag.GetAttributeValue("href", string.Empty))
.Where(sourceId => !sourceId.Contains("live-betting")).ToList();
await UpdateScrapeStatus(10, "Scraping metric data");
var rangeProgress = sourceIds.Count != 0 ? 90 / sourceIds.Count : 0;
var currentRange = 10;
foreach (var matchUrl in sourceIds.Select(sourceId => $"https://nj.betamerica.com{sourceId}"))
{
chromeDriver.Navigate().GoToUrl(matchUrl);
await Task.Delay(7000);
doc.LoadHtml(chromeDriver.PageSource);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = doc.DocumentNode.SelectNodes("//html/body/div[@class='content-main']/div[contains(@class, 'content-main-inner')]/div[@id='pagesWrapper']/div[@id='panel-center-inner']/section[@id='pre-live-betting']/div/div[@class='event-view-views-switcher']/div/ul");
foreach (var rawMetric in rawMetrics)
{
var liTags = rawMetric.SelectNodes("li");
foreach (var liTag in liTags)
{
var spanItem = liTag.SelectSingleNode("h4/span[2]/span");
var spanText = spanItem.InnerText;
var scoreType =
spanText.Contains("Total Points") ? ScoreType.Point :
spanText.Contains("Total Rebounds") ? ScoreType.Rebound :
spanText.Contains("Total Assists") ? ScoreType.Assist :
spanText.Contains("Pts, Rebs, Asts") ? ScoreType.PointReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(spanText, @"(.*) (Total|\(Pts,)");
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overUnderFirstData = liTag.SelectSingleNode("div/div/div[2]/button");
var overUnderSecondData = liTag.SelectSingleNode("div/div/div[1]/button");
double?over = 0, overLine = 0, under = 0, underLine = 0;
var overUnderFirstLine = overUnderFirstData.SelectSingleNode("span[1]/span[1]").InnerText;
if (overUnderFirstLine.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderFirstLine, @"Over (\d*.\d*) .*"));
var overPriceData = overUnderFirstData.SelectSingleNode("span[2]/span").InnerText;
var priceNumber = ScrapeHelper.ConvertMetric(overPriceData);
over = priceNumber != null
? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) :
Math.Round((double)(100 / (priceNumber * -1) + 1), 2)
: (double?)null;
}
else
{
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderFirstLine, @"Under (\d*.\d*) .*"));
var underPriceData = overUnderFirstData.SelectSingleNode("span[2]/span").InnerText;
var priceNumber = ScrapeHelper.ConvertMetric(underPriceData);
under = priceNumber != null
? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) :
Math.Round((double)(100 / (priceNumber * -1) + 1), 2)
: (double?)null;
}
var overUnderSecondLine = overUnderSecondData.SelectSingleNode("span[1]/span[1]").InnerText;
if (overUnderSecondLine.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderSecondLine, @"Over (\d*.\d*) .*"));
var overPriceData = overUnderSecondData.SelectSingleNode("span[2]/span").InnerText;
var priceNumber = ScrapeHelper.ConvertMetric(overPriceData);
over = priceNumber != null
? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) :
Math.Round((double)(100 / (priceNumber * -1) + 1), 2)
: (double?)null;
}
else
{
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overUnderSecondLine, @"Under (\d*.\d*) .*"));
var underPriceData = overUnderSecondData.SelectSingleNode("span[2]/span").InnerText;
var priceNumber = ScrapeHelper.ConvertMetric(underPriceData);
under = priceNumber != null
? priceNumber > 0 ? Math.Round((double)(priceNumber / 100 + 1), 2) :
Math.Round((double)(100 / (priceNumber * -1) + 1), 2)
: (double?)null;
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
const string url = "https://api.beta.tab.com.au/v1/tab-info-service/sports/Basketball/competitions/NBA/matches?jurisdiction=VIC";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.matches[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$._links.self").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectTokens("$.contestants[?(@.position == 'HOME')].name")?.FirstOrDefault()?.ToString();
var awayTeam = rawMatch.SelectTokens("$.contestants[?(@.position == 'AWAY')].name")?.FirstOrDefault()?.ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scraping metric data");
foreach (var match in foundMatches)
{
doc = await ScrapeHelper.GetDocument(match.SourceId);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMetrics = jDoc.SelectTokens("$.markets[?(@.betOptionSpectrumId =~ /(605)/)]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var rawMetric in rawMetrics)
{
var propositionPlayerA = rawMetric.SelectToken("$.propositions[0]");
var propositionTie = rawMetric.SelectToken("$.propositions[1]");
var propositionPlayerB = rawMetric.SelectToken("$.propositions[2]");
var playerNameA = ScrapeHelper.RegexMappingExpression(propositionPlayerA.SelectToken("$.name").ToString(), @"(.*)\(");
var playerNameB = ScrapeHelper.RegexMappingExpression(propositionPlayerB.SelectToken("$.name").ToString(), @"(.*)\(");
var playerA = ScrapeHelper.FindPlayerInMatch(playerNameA, match);
if (playerA == null)
{
Logger.Warning($"Cannot find any playerA {playerNameA} in match {match.Id}");
continue;
}
var playerB = ScrapeHelper.FindPlayerInMatch(playerNameB, match);
if (playerB == null)
{
Logger.Warning($"Cannot find any playerB {playerNameB} in match {match.Id}");
continue;
}
var priceA = ScrapeHelper.ConvertMetric(propositionPlayerA.SelectToken("$.returnWin").ToString());
var tiePrice = ScrapeHelper.ConvertMetric(propositionTie.SelectToken("$.returnWin").ToString());
var priceB = ScrapeHelper.ConvertMetric(propositionPlayerB.SelectToken("$.returnWin").ToString());
Logger.Information($"{playerA.Name} vs {playerB.Name}: {priceA} vs {priceB} | Tie: {tiePrice}");
var metric = new PlayerHeadToHead
{
PlayerAId = playerA.Id,
PlayerBId = playerB.Id,
IsTieIncluded = tiePrice != null,
PlayerAPrice = priceA,
PlayerBPrice = priceB,
ScrapingInformationId = GetScrapingInformation().Id,
TiePrice = tiePrice,
MatchId = match.Id,
CreatedAt = DateTime.Now
};
PlayerHeadToHeads.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://www.betus.com.pa/sportsbook/nba-basketball-lines.aspx";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(5000);
var matchCount = chromeDriver.FindElementsByClassName("props").Count;
await UpdateScrapeStatus(10, "Scraping metric data");
var rangeProgress = matchCount != 0 ? 90 / matchCount : 0;
var currentRange = 10;
for (var i = 0; i < matchCount; i++)
{
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(3000);
var props = chromeDriver.FindElementsByClassName("props");
props[i].Click();
await Task.Delay(3000);
var select = new SelectElement(chromeDriver.FindElementByClassName("odds-display"));
select.SelectByText("Decimal");
await Task.Delay(2000);
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var matchNode = doc.DocumentNode.SelectSingleNode("//html/body/form/div[@class='sportsbook-visitor']/div[@class='sportsbook-master']/div[@class='rounded-block-white']/span[2]/div[@class='col']/div[@id='game-lines']/div/div[@class='game-block']/div[@class='normal']/div/div[@class='future-lines inline-prop']");
var rawMetrics = matchNode.SelectNodes("table");
foreach (var rawMetric in rawMetrics)
{
var playerNode = rawMetric.SelectSingleNode("tbody/tr[1]/th[2]").InnerText;
var scoreType =
playerNode.Contains("Total Points+Rebounds+Assists") ? ScoreType.PointReboundAssist :
playerNode.Contains("Total Points+Rebounds") ? ScoreType.PointRebound :
playerNode.Contains("Total Points+Assists") ? ScoreType.PointAssist :
playerNode.Contains("Total Rebounds+Assists") ? ScoreType.ReboundAssist :
playerNode.Contains("Total Points") ? ScoreType.Point :
playerNode.Contains("Total Rebounds") ? ScoreType.Rebound :
playerNode.Contains("Total Assists") ? ScoreType.Assist :
playerNode.Contains("Total Made 3") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
playerNode = playerNode
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\t", string.Empty)
.Trim();
var playerName = ScrapeHelper.RegexMappingExpression(playerNode, @"(.*) \(");
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
var overNode = rawMetric.SelectSingleNode("tbody/tr[2]/td[2]").InnerText;
overNode = overNode
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\t", string.Empty)
.Replace(" ", " ")
.Trim();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overNode, @"Over.(\d*.\d*)"));
var overPriceNode = rawMetric.SelectSingleNode("tbody/tr[2]/td[3]/a").InnerText;
var over = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overPriceNode, @"(\d*.\d*).*"));
var underNode = rawMetric.SelectSingleNode("tbody/tr[3]/td[2]").InnerText;
underNode = underNode
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace(" ", " ")
.Replace("\t", string.Empty)
.Trim();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underNode, @"Under.(\d*.\d*)"));
var underPriceNode = rawMetric.SelectSingleNode("tbody/tr[3]/td[3]/a").InnerText;
var under = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underPriceNode, @"(\d*.\d*).*"));
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
private void ProcessMetric(Match match, IReadOnlyCollection <JToken> entrants,
IReadOnlyCollection <JToken> prices, string scoreType, string actualScoreType)
{
var overMetrics = entrants.Where(x =>
{
var name = x.Children().FirstOrDefault()?.SelectToken("$.name").ToString();
if (scoreType == "PR" || scoreType == "RA")
{
return(name != null && name.Contains(actualScoreType) && name.Contains("Over") && !name.Contains("PRA"));
}
return(name != null && name.Contains(actualScoreType) && name.Contains("Over"));
}).ToList();
var underMetrics = entrants.Where(x =>
{
var name = x.Children().FirstOrDefault()?.SelectToken("$.name").ToString();
if (scoreType == "PR" || scoreType == "RA")
{
return(name != null && name.Contains(actualScoreType) && name.Contains("Under") && !name.Contains("PRA"));
}
return(name != null && name.Contains(actualScoreType) && name.Contains("Under"));
}).ToList();
var playerNames = new HashSet <string>();
foreach (var content in overMetrics
.Select(overMetric => overMetric
.Children()
.FirstOrDefault()?
.SelectToken("$.name").ToString() ?? string.Empty))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(content, "(.*) Over"));
}
foreach (var content in underMetrics
.Select(underMetric => underMetric.Children()
.FirstOrDefault()?
.SelectToken("$.name").ToString() ?? string.Empty))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(content, "(.*) Under"));
}
foreach (var playerName in playerNames)
{
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overContent = overMetrics.FirstOrDefault(x =>
x.Children().FirstOrDefault()?.SelectToken("$.name").ToString().Contains(playerName) ?? false)
?.Children().FirstOrDefault();
var underContent = underMetrics.FirstOrDefault(x =>
x.Children().FirstOrDefault()?.SelectToken("$.name").ToString().Contains(playerName) ?? false)
?.Children().FirstOrDefault();
var overLine =
ScrapeHelper.ConvertMetric(ScrapeHelper
.RegexMappingExpression(overContent?.SelectToken("$.name").ToString(), "Over (.*) .*"));
var underLine =
ScrapeHelper.ConvertMetric(ScrapeHelper
.RegexMappingExpression(underContent?.SelectToken("$.name").ToString(), "Under (.*) .*"));
var priceOverId = overContent?.SelectToken("$.id").ToString() ?? string.Empty;
var priceUnderId = underContent?.SelectToken("$.id").ToString() ?? string.Empty;
var priceOverData = prices.FirstOrDefault(x => x.ToString().Contains(priceOverId));
var priceOverContent = priceOverData?.Children().FirstOrDefault();
var numeratorOver = ScrapeHelper.ConvertMetric(priceOverContent?.SelectToken("$.odds.numerator").ToString());
var denominatorOver = ScrapeHelper.ConvertMetric(priceOverContent?.SelectToken("$.odds.denominator").ToString());
var over = numeratorOver != null && denominatorOver != null ? numeratorOver / denominatorOver + 1 : null;
var priceUnderData = prices.FirstOrDefault(x => x.ToString().Contains(priceUnderId));
var priceUnderContent = priceUnderData?.Children().FirstOrDefault();
var numeratorUnder = ScrapeHelper.ConvertMetric(priceUnderContent.SelectToken("$.odds.numerator").ToString());
var denominatorUnder = ScrapeHelper.ConvertMetric(priceUnderContent.SelectToken("$.odds.denominator").ToString());
var under = numeratorUnder != null && denominatorUnder != null ? numeratorUnder / denominatorUnder + 1 : null;
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
}
protected override async Task ScrapeData()
{
const string url = "https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Competitions/6927/Events";
var rawDoc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
var rawMatches = jDoc.SelectTokens("$.[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
if (rawMatch.SelectToken("$.competitionName").ToString() != "NBA Matches")
{
continue;
}
var sourceMatchId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var matchName = rawMatch.SelectToken("$.name").ToString();
var teams = matchName.Split(new[] { " At " }, StringSplitOptions.None);
var homeTeam = teams[1];
var awayTeam = teams[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
var pointsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/567/Markets";
var reboundsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/568/Markets";
var assistsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/569/Markets";
var combinationUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/570/Markets";
currentRange = Math.Min(currentRange + rangeProgress, 90);
var docTasks = new List <Task <string> >
{
ScrapeHelper.GetDocument(pointsUrl),
ScrapeHelper.GetDocument(reboundsUrl),
ScrapeHelper.GetDocument(assistsUrl),
ScrapeHelper.GetDocument(combinationUrl)
};
var results = await Task.WhenAll(docTasks);
var rawMetrics = new List <JToken>();
foreach (var result in results)
{
rawMetrics.AddRange(JsonConvert.DeserializeObject <JToken>(result).SelectTokens("$.[*]"));
}
foreach (var rawMetric in rawMetrics)
{
var metricName = rawMetric.SelectToken("$.name").ToString();
var scoreType =
metricName.Contains("Pts + Reb + Ast") ? ScoreType.PointReboundAssist :
metricName.Contains("Points") ? ScoreType.Point :
metricName.Contains("Assists") ? ScoreType.Assist :
metricName.Contains("Made Threes") ? ScoreType.ThreePoint :
metricName.Contains("Rebounds") ? ScoreType.Rebound :
metricName.Contains("Pts + Reb") ? ScoreType.PointRebound :
metricName.Contains("Pts + Ast") ? ScoreType.PointAssist :
metricName.Contains("Reb + Ast") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = metricName.Split('-')[0];
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var selection = rawMetric.SelectToken("$.selections[0]");
var selectionName = selection.SelectToken("$.name").ToString();
double?over = 0, overLine = 0, under = 0, underLine = 0;
if (selectionName.Contains("Over"))
{
over = ScrapeHelper.ConvertMetric(selection.SelectToken("$.price.winPrice")?.ToString());
overLine = ScrapeHelper.ConvertMetric(selection.SelectToken("$.unformattedHandicap")?.ToString());
}
else if (selectionName.Contains("Under"))
{
under = ScrapeHelper.ConvertMetric(selection.SelectToken("$.price.winPrice")?.ToString());
underLine = ScrapeHelper.ConvertMetric(selection.SelectToken("$.unformattedHandicap")?.ToString());
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://ubet.com/api/sportsViewData/mainevents/false/6698";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.MainEventId").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var mainEventName = rawMatch.SelectToken("$.MainEventName").ToString();
var date = Helper.GetCurrentDate("dd/M");
var teams = mainEventName
.Split(new[] { date }, StringSplitOptions.None)[0]
.Split(new[] { " v " }, StringSplitOptions.None);
var homeTeam = teams[0];
var awayTeam = teams[1];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scraping metric data");
var betTypes = new HashSet <string>();
foreach (var match in foundMatches)
{
var metricUrl = $"https://ubet.com/api/sportsViewData/markets/false/{match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMetrics = jDoc
.SelectTokens("$.All[?(@.GroupId == -1)].SubEvents[?(@.LongDisplayName =~ /(.* (Ttl|Total).*)/)]")
.ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var rawMetric in rawMetrics)
{
var betTypeShortName = rawMetric.SelectToken("$.BetTypeShortName").ToString();
var scoreType =
betTypeShortName.Contains("Total Points Scored") ? ScoreType.Point :
betTypeShortName.Contains("Total Pts+Rebound+Assist") ? ScoreType.PointReboundAssist :
string.Empty;
betTypes.Add(betTypeShortName);
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var longDisplayName = rawMetric.SelectToken("$.LongDisplayName").ToString();
var playerName = ScrapeHelper.RegexMappingExpression(longDisplayName, @"(.*)(?:Ttl|Total)");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
double?over = 0, overLine = 0, under = 0, underLine = 0;
var offers = rawMetric.SelectTokens(@"$.Offers[?(@.OfferName =~ /(.* Over|Under|OV|UN .*)/)]");
foreach (var offer in offers)
{
var offerName = offer.SelectToken("$.OfferName").ToString();
if (offerName.Contains("Over") || offerName.Contains("OV"))
{
over = ScrapeHelper.ConvertMetric(offer.SelectToken("$.WinReturn").ToString());
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(offerName, @".* (\d*.\d)"));
}
else if (offerName.Contains("Under") || offerName.Contains("UN"))
{
under = ScrapeHelper.ConvertMetric(offer.SelectToken("$.WinReturn").ToString());
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(offerName, @".* (\d*.\d)"));
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information($"Unique scoreTypes: {JsonConvert.SerializeObject(betTypes)}"); // notify to update
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://www.nextbet.com/en/sports/227-basketball/22892-usa/44178-nba";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(10000);
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
var matchNodes = doc.DocumentNode.SelectNodes("//span[@class='period-description']");
var rawMatches = matchNodes.Where(x => !x.InnerText.Contains("Tomorrow")).ToList();
Logger.Information("Scrape metric data");
await UpdateScrapeStatus(10, "Scraping metric data");
var rangeProgress = rawMatches.Count != 0 ? 90 / rawMatches.Count : 0;
var currentRange = 10;
for (var i = 0; i < rawMatches.Count; i++)
{
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(10000);
var markets = chromeDriver.FindElementsByClassName("more_markets");
markets[i].Click();
await Task.Delay(10000);
doc.LoadHtml(chromeDriver.PageSource);
var marketContainer = doc.DocumentNode.SelectSingleNode("//div[@id='content']");
var marketGroups = marketContainer.SelectNodes("//div[@class='markets-group-component']");
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var marketItem in marketGroups)
{
string scoreTypeItem;
try
{
scoreTypeItem = marketItem.SelectSingleNode("div/div/h2").InnerText;
}
catch (Exception)
{
continue;
}
var scoreType =
scoreTypeItem.Contains("Points + Assists") ? ScoreType.PointAssist :
scoreTypeItem.Contains("Points + Rebounds") ? ScoreType.PointRebound :
scoreTypeItem.Contains("Assists + Rebounds") ? ScoreType.ReboundAssist :
scoreTypeItem.Contains("Points + Assists + Rebounds") ? ScoreType.PointReboundAssist :
scoreTypeItem.Contains("Points") ? ScoreType.Point :
scoreTypeItem.Contains("Rebounds") ? ScoreType.Rebound :
scoreTypeItem.Contains("Assists") ? ScoreType.Assist :
scoreTypeItem.Contains("Three Made") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType) || scoreTypeItem.Contains("Odd/Even") ||
!scoreTypeItem.Contains("Match"))
{
continue;
}
doc.LoadHtml(marketItem.InnerHtml);
var playerMarkets = doc.DocumentNode.SelectNodes("//div[@class='market-component']");
foreach (var playerMarket in playerMarkets)
{
var playerName = playerMarket.SelectSingleNode("div[@class='player']").InnerText
.Replace("\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\t", string.Empty)
.Trim();
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
var overLineItem = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td/span");
var overLineData = overLineItem.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, "Over (.*)"));
var priceOverData = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td/span[2]/span/span");
var over = ScrapeHelper.ConvertMetric(priceOverData.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim());
var underLineItem = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td[2]/span");
var underLineData = underLineItem.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, "Under (.*)"));
var priceUnderData = playerMarket.SelectSingleNode("div[@class='swish-markets-wrapper']/div/table/tbody/tr/td[2]/span[2]/span/span");
var under = ScrapeHelper.ConvertMetric(priceUnderData.InnerText.Replace("\n", string.Empty).Replace("\r", string.Empty).Replace("\t", string.Empty).Trim());
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / playerMarkets.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
