protected override async Task ScrapeData()
{
const string url = "https://api.neds.com.au/v2/sport/event-request?category_ids=%5B%223c34d075-dc14-436d-bfc4-9272a49c2b39%22%5D";
var doc = await ScrapeHelper.GetDocument(new Uri(url));
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectToken("$.events").ToList();
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var matchContent in rawMatches.Select(rawMatch => rawMatch.Children().FirstOrDefault()))
{
if (matchContent == null)
{
Logger.Warning("Match content is null");
continue;
}
var sourceMatchId = matchContent.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source match id is null");
continue;
}
var matchName = matchContent.SelectToken("$.name").ToString();
var competitionName = matchContent.SelectToken("$.competition.name").ToString();
if (!matchName.Contains(" V ") || !competitionName.Contains("NBA"))
{
continue;
}
var homeTeam = ScrapeHelper.RegexMappingExpression(matchName, "(.*) V");
var awayTeam = ScrapeHelper.RegexMappingExpression(matchName, "V (.*)");
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
Logger.Information("Scraping metric data");
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://api.neds.com.au/v2/sport/event-card?id={match.SourceId}";
doc = await ScrapeHelper.GetDocument(new Uri(metricUrl));
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var entrants = jDoc.SelectToken("$.entrants").ToList();
var prices = jDoc.SelectToken("$.prices").ToList();
ProcessMetric(match, entrants, prices, ScoreType.Point, "Points");
ProcessMetric(match, entrants, prices, ScoreType.Rebound, "Rebounds");
ProcessMetric(match, entrants, prices, ScoreType.Assist, "Assists");
ProcessMetric(match, entrants, prices, ScoreType.PointReboundAssist, "PRA");
ProcessMetric(match, entrants, prices, ScoreType.PointRebound, "PR");
ProcessMetric(match, entrants, prices, ScoreType.PointAssist, "PA");
ProcessMetric(match, entrants, prices, ScoreType.ReboundAssist, "RA");
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + 90 / foundMatches.Count, 90);
await UpdateScrapeStatus(newProgress, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://ubet.com/api/sportsViewData/mainevents/false/6698";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.MainEventId").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var mainEventName = rawMatch.SelectToken("$.MainEventName").ToString();
var date = Helper.GetCurrentDate("dd/M");
var teams = mainEventName
.Split(new[] { date }, StringSplitOptions.None)[0]
.Split(new[] { " v " }, StringSplitOptions.None);
var homeTeam = teams[0];
var awayTeam = teams[1];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scraping metric data");
var betTypes = new HashSet <string>();
foreach (var match in foundMatches)
{
var metricUrl = $"https://ubet.com/api/sportsViewData/markets/false/{match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMetrics = jDoc
.SelectTokens("$.All[?(@.GroupId == -1)].SubEvents[?(@.LongDisplayName =~ /(.* (Ttl|Total).*)/)]")
.ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var rawMetric in rawMetrics)
{
var betTypeShortName = rawMetric.SelectToken("$.BetTypeShortName").ToString();
var scoreType =
betTypeShortName.Contains("Total Points Scored") ? ScoreType.Point :
betTypeShortName.Contains("Total Pts+Rebound+Assist") ? ScoreType.PointReboundAssist :
string.Empty;
betTypes.Add(betTypeShortName);
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var longDisplayName = rawMetric.SelectToken("$.LongDisplayName").ToString();
var playerName = ScrapeHelper.RegexMappingExpression(longDisplayName, @"(.*)(?:Ttl|Total)");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
double?over = 0, overLine = 0, under = 0, underLine = 0;
var offers = rawMetric.SelectTokens(@"$.Offers[?(@.OfferName =~ /(.* Over|Under|OV|UN .*)/)]");
foreach (var offer in offers)
{
var offerName = offer.SelectToken("$.OfferName").ToString();
if (offerName.Contains("Over") || offerName.Contains("OV"))
{
over = ScrapeHelper.ConvertMetric(offer.SelectToken("$.WinReturn").ToString());
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(offerName, @".* (\d*.\d)"));
}
else if (offerName.Contains("Under") || offerName.Contains("UN"))
{
under = ScrapeHelper.ConvertMetric(offer.SelectToken("$.WinReturn").ToString());
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(offerName, @".* (\d*.\d)"));
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information($"Unique scoreTypes: {JsonConvert.SerializeObject(betTypes)}"); // notify to update
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://services.bovada.lv/services/sports/event/coupon/events/A/description/basketball/nba?marketFilterId=def&preMatchOnly=true&lang=en";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.[*].events[*]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.link").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var description = rawMatch.SelectToken("$.description").ToString();
var desSplits = description.Split('@');
var homeTeam = desSplits[1];
var awayTeam = desSplits[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://services.bovada.lv/services/sports/event/coupon/events/A/description{match.SourceId}?lang=en";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc
.SelectTokens("$.[*].events[*].displayGroups[?(@.description == 'Player Props')].markets[*]")
.ToList();
foreach (var rawMetric in rawMetrics)
{
var description = rawMetric.SelectToken("$.description").ToString();
var scoreType =
description.Contains("Total Points, Rebounds and Assists") ? ScoreType.PointReboundAssist :
description.Contains("Total Points and Rebounds") ? ScoreType.PointRebound :
description.Contains("Total Points and Assists") ? ScoreType.PointAssist :
description.Contains("Total Rebounds and Assists") ? ScoreType.ReboundAssist :
description.Contains("Total Points") ? ScoreType.Point :
description.Contains("Total Rebounds") ? ScoreType.Rebound :
description.Contains("Total Assists") ? ScoreType.Assist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(description, @"- (.*) \(");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var outcomes = rawMetric.SelectTokens("$.outcomes[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcome in outcomes)
{
var des = outcome.SelectToken("$.description").ToString();
if (des.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.handicap").ToString());
var overPrice = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.decimal").ToString());
over = overPrice != null?Math.Round((double)overPrice, 2) : (double?)null;
}
else if (des.Contains("Under"))
{
underLine = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.handicap").ToString());
var underPrice = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.decimal").ToString());
under = underPrice != null?Math.Round((double)underPrice, 2) : (double?)null;
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://cds-api.borgataonline.com/bettingoffer/fixtures?x-bwin-accessid=ZTJhZDliYjgtNTdmOC00Njk0LWIxZmItODI3YzhjZGQ5NmIx&lang=en-US&country=VN&userCountry=VN&streamProviders=unas%2Cperform%2Cimgdge&fixtureTypes=Standard&state=Latest&skip=0&take=50&offerMapping=Filtered&offerCategories=Gridable&fixtureCategories=Gridable,NonGridable,Other&sortBy=Tags&sportIds=7®ionIds=9&competitionIds=6004";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.fixtures[*]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var participants = rawMatch.SelectTokens("$.participants[*]").ToList();
var homeTeam = participants.Last().SelectToken("$.name.value").ToString();
var awayTeam = participants.First().SelectToken("$.name.value").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://cds-api.borgataonline.com/bettingoffer/fixture-view?x-bwin-accessid=ZTJhZDliYjgtNTdmOC00Njk0LWIxZmItODI3YzhjZGQ5NmIx&lang=en-US&country=VN&userCountry=VN&streamProviders=unas%2Cperform%2Cimgdge&offerMapping=All&scoreboardMode=Full&fixtureIds={match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc.SelectTokens("$.fixture.games[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var nameData = rawMetric.SelectToken("$.name.value").ToString();
var scoreType =
nameData.Contains("How many points") ? ScoreType.Point :
nameData.Contains("How many assists") ? ScoreType.Assist :
nameData.Contains("How many rebounds") ? ScoreType.Rebound :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(nameData, @".* will (.*) \(");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overData = rawMetric.SelectToken("$.results[0]");
var over = ScrapeHelper.ConvertMetric(overData.SelectToken("$.odds").ToString());
var overLineData = overData.SelectToken("$.name.value").ToString();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, "Over (.*)"));
var underData = rawMetric.SelectToken("$.results[0]");
var under = ScrapeHelper.ConvertMetric(underData.SelectToken("$.odds").ToString());
var underLineData = underData.SelectToken("$.name.value").ToString();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, "Over (.*)"));
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://api.pointsbet.com/api/v2/competitions/7176/events/featured?includeLive=true";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.events[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.key").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.homeTeam").ToString();
var awayTeam = rawMatch.SelectToken("$.awayTeam").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://api.pointsbet.com/api/v2/events/{match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc
.SelectTokens(
@"$.fixedOddsMarkets[?(@.name =~ /(Player Points Over\/Under .*)/ || @.name =~ /(Player Rebounds Over\/Under .*)/ || @.name =~ /(Player Assists Over\/Under .*)/)]")
.ToList();
foreach (var rawMetric in rawMetrics)
{
var metricName = rawMetric.SelectToken("$.name").ToString();
var scoreType =
metricName.Contains("Player Points") ? ScoreType.Point :
metricName.Contains("Player Assists") ? ScoreType.Assist :
metricName.Contains("Player Rebounds") ? ScoreType.Rebound :
metricName.Contains("Player Pts + Rebs + Asts") ? ScoreType.PointReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var outcomeOvers = rawMetric.SelectTokens("$.outcomes[?(@.name =~ /(.* Over .*)/)]").ToList();
var outcomeUnders = rawMetric.SelectTokens("$.outcomes[?(@.name =~ /(.* Under .*)/)]").ToList();
var playerNames = new HashSet <string>();
foreach (var name in outcomeOvers.Select(outcome => outcome.SelectToken("name").ToString()))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(name, "(.*)(?:Over|Under)"));
}
foreach (var name in outcomeUnders.Select(outcome => outcome.SelectToken("name").ToString()))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(name, "(.*)(?:Over|Under)"));
}
foreach (var playerName in playerNames)
{
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var outcomeOver = outcomeOvers.FirstOrDefault(x =>
x.SelectToken("$.name").ToString().Contains(playerName));
var outcomeUnder = outcomeUnders.FirstOrDefault(x =>
x.SelectToken("$.name").ToString().Contains(playerName));
var rawOver = outcomeOver?.SelectToken("price").ToString();
var rawOverLine =
ScrapeHelper.RegexMappingExpression(outcomeOver?.SelectToken("$.name").ToString(),
"(?:Over) (.*)");
var rawUnder = outcomeUnder?.SelectToken("price").ToString();
var rawUnderLine =
ScrapeHelper.RegexMappingExpression(outcomeUnder?.SelectToken("$.name").ToString(),
"(?:Under) (.*)");
Logger.Information($"{player.Name}: {scoreType} - {rawOver} {rawOverLine} | {rawUnder} {rawUnderLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = ScrapeHelper.ConvertMetric(rawOver),
OverLine = ScrapeHelper.ConvertMetric(rawOverLine),
Under = ScrapeHelper.ConvertMetric(rawUnder),
UnderLine = ScrapeHelper.ConvertMetric(rawUnderLine),
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://eu-offering.kambicdn.org/offering/v2018/888/listView/basketball.json?lang=en_GB&market=ZZ&client_id=2&channel_id=1&ncid=1572482733163&useCombined=true";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.events[?(@.event.group == 'NBA')]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.event.id").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.event.homeName").ToString();
var awayTeam = rawMatch.SelectToken("$.event.awayName").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
foreach (var match in foundMatches)
{
var metricUrl = $"https://eu-offering.kambicdn.org/offering/v2018/888/betoffer/event/{match.SourceId}.json?lang=en_GB&market=ZZ&client_id=2&channel_id=1&ncid=1005710980&includeParticipants=true";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc.SelectTokens("$.betOffers[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var labelData = rawMetric.SelectToken("$.criterion.label").ToString();
var scoreType = labelData.Contains("Points scored by the player") ? ScoreType.Point :
labelData.Contains("Rebounds by the player") ? ScoreType.Rebound :
labelData.Contains("Assists by the player") ? ScoreType.Assist :
labelData.Contains("Points, rebounds & assists by the player") ? ScoreType.PointReboundAssist :
labelData.Contains("3-point field goals made by the player") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var overData = rawMetric.SelectToken("$.outcomes[?(@.label =~ /(Over.*)/)]");
var underData = rawMetric.SelectToken("$.outcomes[?(@.label =~ /(Under.*)/)]");
var playerData = overData.SelectToken("$.participant").ToString();
var playerSplits = playerData.Split(',');
var rvPlayer = playerSplits.Reverse().ToList();
var playerName = string.Join(" ", rvPlayer);
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overLineData = overData.SelectToken("$.label").ToString();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, @"Over.* (\d*.\d*)"));
var overOdd = ScrapeHelper.ConvertMetric(overData.SelectToken("$.odds").ToString());
var over = overOdd / 1000;
var underLineData = underData.SelectToken("$.label").ToString();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, @"Under.* (\d*.\d*)"));
var underOdd = ScrapeHelper.ConvertMetric(underData.SelectToken("$.odds").ToString());
var under = underOdd / 1000;
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Competitions/6927/Events";
var rawDoc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
var rawMatches = jDoc.SelectTokens("$.[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
if (rawMatch.SelectToken("$.competitionName").ToString() != "NBA Matches")
{
continue;
}
var sourceMatchId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var matchName = rawMatch.SelectToken("$.name").ToString();
var teams = matchName.Split(new[] { " At " }, StringSplitOptions.None);
var homeTeam = teams[1];
var awayTeam = teams[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
var pointsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/567/Markets";
var reboundsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/568/Markets";
var assistsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/569/Markets";
var combinationUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/570/Markets";
currentRange = Math.Min(currentRange + rangeProgress, 90);
var docTasks = new List <Task <string> >
{
ScrapeHelper.GetDocument(pointsUrl),
ScrapeHelper.GetDocument(reboundsUrl),
ScrapeHelper.GetDocument(assistsUrl),
ScrapeHelper.GetDocument(combinationUrl)
};
var results = await Task.WhenAll(docTasks);
var rawMetrics = new List <JToken>();
foreach (var result in results)
{
rawMetrics.AddRange(JsonConvert.DeserializeObject <JToken>(result).SelectTokens("$.[*]"));
}
foreach (var rawMetric in rawMetrics)
{
var metricName = rawMetric.SelectToken("$.name").ToString();
var scoreType =
metricName.Contains("Pts + Reb + Ast") ? ScoreType.PointReboundAssist :
metricName.Contains("Points") ? ScoreType.Point :
metricName.Contains("Assists") ? ScoreType.Assist :
metricName.Contains("Made Threes") ? ScoreType.ThreePoint :
metricName.Contains("Rebounds") ? ScoreType.Rebound :
metricName.Contains("Pts + Reb") ? ScoreType.PointRebound :
metricName.Contains("Pts + Ast") ? ScoreType.PointAssist :
metricName.Contains("Reb + Ast") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = metricName.Split('-')[0];
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var selection = rawMetric.SelectToken("$.selections[0]");
var selectionName = selection.SelectToken("$.name").ToString();
double?over = 0, overLine = 0, under = 0, underLine = 0;
if (selectionName.Contains("Over"))
{
over = ScrapeHelper.ConvertMetric(selection.SelectToken("$.price.winPrice")?.ToString());
overLine = ScrapeHelper.ConvertMetric(selection.SelectToken("$.unformattedHandicap")?.ToString());
}
else if (selectionName.Contains("Under"))
{
under = ScrapeHelper.ConvertMetric(selection.SelectToken("$.price.winPrice")?.ToString());
underLine = ScrapeHelper.ConvertMetric(selection.SelectToken("$.unformattedHandicap")?.ToString());
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://www.betvictor.com/api/top_bets/601600?number_to_show=20&exclude_in_running=true&event_ids_to_exclude=&sport_ids_to_exclude=";
var rawDoc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
await UpdateScrapeStatus(10, "Scraping match data");
var rawMatches = jDoc.SelectTokens("$.[?(@.meeting_description == 'NBA')]");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.event_id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
if (foundMatches.Select(x => x.SourceId).Contains(sourceMatchId))
{
continue;
}
var game = rawMatch.SelectToken("$.event_description").ToString();
var clubs = game.Split('@');
var homeTeam = clubs[1];
var awayTeam = clubs[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
rawDoc = await ScrapeHelper.GetDocument($"https://www.betvictor.com/bv_event_level/en-gb/1/coupons/{match.SourceId}/4787?t=1573192805056");
var jResult = JsonConvert.DeserializeObject <JToken>(rawDoc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jResult.SelectTokens("$.[*].markets[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var description = rawMetric.SelectToken("$.des").ToString();
var scoreType =
description.Contains("Total combined points, assists & rebounds") ? ScoreType.PointReboundAssist :
description.Contains("Total points") ? ScoreType.Point :
description.Contains("Total assists") ? ScoreType.Assist :
description.Contains("Total three pointers") ? ScoreType.ThreePoint :
description.Contains("Total rebounds") ? ScoreType.Rebound :
description.Contains("Total combined points & rebounds") ? ScoreType.PointRebound :
description.Contains("Total combined points & assists") ? ScoreType.PointAssist :
description.Contains("Total combined assists & rebounds") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = rawMetric.SelectToken("$.opponentDescription").ToString();
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var playerMetrics = rawMetric.SelectTokens("$.o[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var metricItem in playerMetrics)
{
var des = metricItem.SelectToken("$.des").ToString();
if (des.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(des, "Over (.*)"));
over = ScrapeHelper.ConvertMetric(metricItem.SelectToken("$.pr").ToString());
}
else if (des.Contains("Under"))
{
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(des, "Under (.*)"));
under = ScrapeHelper.ConvertMetric(metricItem.SelectToken("$.pr").ToString());
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
// Fetch all teams from web portal
var teams = await WebPortalHelper.GetTeams(Helper.GetSportCode());
if (teams == null || teams.Count == 0)
{
throw new ArgumentNullException(nameof(teams), "Teams from web portal is null");
}
FromDate = Helper.ToMinTime(FromDate);
ToDate = Helper.ToMaxTime(ToDate);
var totalDays = Convert.ToInt32((ToDate - FromDate).TotalDays);
for (var date = FromDate.Date; date <= ToDate.Date; date = date.AddDays(1))
{
Logger.Information($"Scrape matches: {date.ToShortDateString()}");
await UpdateScrapeStatus(null, $"Scrape matches: {date.ToShortDateString()}");
var url = $"http://site.api.espn.com/apis/site/v2/sports/basketball/nba/scoreboard?lang=en®ion=au&calendartype=blacklist&limit=100&dates={Helper.GetDate(date)}&tz=Australia%2FMelbourne";
var rawResult = await ScrapeHelper.GetDocument(url);
var jObject = JObject.Parse(rawResult);
if (jObject == null)
{
throw new ArgumentNullException(nameof(jObject));
}
var sportCode = Helper.GetSportCode();
var teamCodes = new List <string>();
var competitions = jObject.SelectTokens("$.events[*].competitions[*]");
foreach (var competition in competitions)
{
var homeAbbr = competition.SelectToken("$.competitors[0].team.abbreviation").ToString();
var awayAbbr = competition.SelectToken("$.competitors[1].team.abbreviation").ToString();
if (teams.All(x => x.ShortName != homeAbbr))
{
throw new Exception($"Cannot find any home team named '{homeAbbr}' in teams");
}
if (teams.All(x => x.ShortName != awayAbbr))
{
throw new Exception($"Cannot find any away team named '{awayAbbr}' in teams");
}
teamCodes.Add(homeAbbr);
teamCodes.Add(awayAbbr);
var homeTeamId = teams.First(x => x.ShortName.Equals(homeAbbr)).Id;
var awayTeamId = teams.First(x => x.ShortName.Equals(awayAbbr)).Id;
var homeTeamName = competition.SelectToken("$.competitors[0].team.name").ToString();
var awayTeamName = competition.SelectToken("$.competitors[1].team.name").ToString();
homeTeamName = homeTeamName.Substring(0, Math.Min(homeTeamName.Length, 3)).ToUpper();
awayTeamName = awayTeamName.Substring(0, Math.Min(awayTeamName.Length, 3)).ToUpper();
var gameCode = $"{Helper.GetDate(date, "MMddyyyy")}{homeTeamName}{awayTeamName}";
var gameDate = competition.SelectToken("$.date").ToString();
Logger.Information($"Match: {gameCode}, {homeTeamName} vs {awayTeamName}, {gameDate}");
Matches.Add(new Match
{
StartTime = DateTime.Parse(gameDate),
HomeTeamId = homeTeamId,
AwayTeamId = awayTeamId,
GameCode = gameCode,
SportCode = sportCode
});
}
Logger.Information("Scraped matches complete");
Logger.Information("Scrape players from teams");
const string baseTeamsUrl = "https://www.espn.com/nba/team/stats/_/name";
const string xPathToPlayers = "/html/body/div[1]/div/div/div/div/div[5]/div[2]/div[5]/div[1]/div/section/div/section[1]/div[2]/table/tbody/tr[*]/td/span/a";
var playerTasks = new List <Task <HtmlNodeCollection> >();
foreach (var teamCode in teamCodes)
{
url = $"{baseTeamsUrl}/{teamCode}";
playerTasks.Add(ScrapeHelper.GetInnerHtml(url, xPathToPlayers));
}
var nodes = await Task.WhenAll(playerTasks);
for (var i = 0; i < teamCodes.Count; i++)
{
var teamId = teams.First(x => x.ShortName == teamCodes[i]).Id;
Logger.Information($"Scrape player from: {baseTeamsUrl}/{teamCodes[i]}");
ExtractPlayers(nodes[i], teamId);
}
Logger.Information("Scrape players complete");
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + 90 / totalDays, 90);
await UpdateScrapeStatus(newProgress, $"Scrape matches: {date.ToShortDateString()} complete");
}
}
protected override async Task ScrapeData()
{
const string url = "https://api.beta.tab.com.au/v1/tab-info-service/sports/Basketball/competitions/NBA/matches?jurisdiction=VIC";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.matches[*]");
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$._links.self").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var matchName = rawMatch.SelectToken("$.name").ToString();
var homeTeam = ScrapeHelper.RegexMappingExpression(matchName, "(.*) v .*");
var awayTeam = ScrapeHelper.RegexMappingExpression(matchName, ".* v (.*)");
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
doc = await ScrapeHelper.GetDocument($"{match.SourceId}");
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMetrics = jDoc.SelectTokens("$.markets[?(@.betOptionSpectrumId =~ /(837|838|839|1789|1791)/)]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var rawMetric in rawMetrics)
{
var spectrumId = rawMetric.SelectToken("$.betOptionSpectrumId").ToString();
var scoreType = spectrumId == "837" ? ScoreType.Point :
spectrumId == "838" ? ScoreType.Rebound :
spectrumId == "839" ? ScoreType.Assist :
spectrumId == "1789" ? ScoreType.PointReboundAssist :
spectrumId == "1791" ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerOvers = rawMetric.SelectTokens("$.propositions[?(@.name =~ /(.* Over .*)/)]").ToList();
var playerUnders = rawMetric.SelectTokens("$.propositions[?(@.name =~ /(.* Under .*)/)]").ToList();
var playerNames = new HashSet <string>();
foreach (var playerName in playerOvers.Select(item => item.SelectToken("name").ToString()).Select(itemName => ScrapeHelper.RegexMappingExpression(itemName, "(.*)(?:Over|Under)")))
{
playerNames.Add(playerName);
}
foreach (var playerName in playerUnders.Select(item => item.SelectToken("name").ToString()).Select(itemName => ScrapeHelper.RegexMappingExpression(itemName, "(.*)(?:Over|Under)")))
{
playerNames.Add(playerName);
}
foreach (var playerName in playerNames)
{
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overOutcomeItem = playerOvers.FirstOrDefault(x => x.SelectToken("$.name").ToString().Contains(playerName));
var overName = overOutcomeItem?.SelectToken("name").ToString() ?? string.Empty;
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overName, @".* (\d*.\d*) .*"));
var over = ScrapeHelper.ConvertMetric(overOutcomeItem?.SelectToken("returnWin").ToString() ?? string.Empty);
var underOutcomeItem = playerUnders.FirstOrDefault(x => x.SelectToken("$.name").ToString().Contains(playerName));
var underName = underOutcomeItem?.SelectToken("name").ToString() ?? string.Empty;
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underName, @".* (\d*.\d*) .*"));
var under = ScrapeHelper.ConvertMetric(underOutcomeItem?.SelectToken("returnWin").ToString() ?? string.Empty);
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://fixture.palmerbet.online/fixtures/sports/1c2eeb3a-6bab-4ac2-b434-165cc350180f/matches?pageSize=25";
var rawResult = await ScrapeHelper.GetDocument(url);
var matchObj = JsonConvert.DeserializeObject <JToken>(rawResult);
var rawMatches = matchObj.SelectTokens("$.matches[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.eventId").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.homeTeam.title").ToString();
var awayTeam = rawMatch.SelectToken("$.awayTeam.title").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rawMetricsTasks = foundMatches
.Select(match => $"https://fixture.palmerbet.online/fixtures/sports/matches/{match.SourceId}/markets?pageSize=1000")
.Select(metricUrl => ScrapeHelper.GetDocument(metricUrl))
.ToList();
var rawMetrics = await Task.WhenAll(rawMetricsTasks);
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = rawMetrics.Length != 0 ? 90 / rawMetrics.Length : 0;
var currentRange = 20;
Logger.Information($"Total raw metrics count {rawMetrics.Length}");
var totalMetrics = 0;
try
{
for (var i = 0; i < rawMetrics.Length; i++)
{
var rawMetric = rawMetrics[i];
var metricObj = JsonConvert.DeserializeObject <JToken>(rawMetric);
var metricMarkets = metricObj.SelectTokens("$.markets[?(@.title=~ /(.* - .*)/)]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var metricMarket in metricMarkets)
{
var title = metricMarket.SelectToken("$.title").ToString();
var scoreType =
title.Contains("Points") ? ScoreType.Point :
title.Contains("Rebounds") ? ScoreType.Rebound :
title.Contains("Assists") ? ScoreType.Assist :
title.Contains("Made Threes") ? ScoreType.ThreePoint :
title.Contains("Pts + Ast") ? ScoreType.PointAssist :
title.Contains("Pts + Reb + Ast") ? ScoreType.PointReboundAssist :
title.Contains("Pts + Reb") ? ScoreType.PointRebound :
title.Contains("Reb + Ast") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = title.Split('-')[0].Trim();
var player = ScrapeHelper.FindPlayerInMatch(playerName, foundMatches[i]);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {foundMatches[i].Id}");
continue;
}
totalMetrics++;
var playerId = metricMarket.SelectToken("$.id").ToString();
var priceUrl = $"https://fixture.palmerbet.online/fixtures/sports/markets/{playerId}";
var priceDoc = await ScrapeHelper.GetDocument(priceUrl, 3000);
var priceObj = JsonConvert.DeserializeObject <JToken>(priceDoc);
if (priceObj == null)
{
Logger.Warning($"priceObj is null for player: {playerId}");
continue;
}
var outcomeItems = priceObj.SelectTokens("$.market.outcomes[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcomeItem in outcomeItems)
{
var titleItem = outcomeItem.SelectToken("$.title").ToString();
if (titleItem.Contains("Over"))
{
var rawOver = ScrapeHelper.RegexMappingExpression(titleItem, "(?:Over) (.*)");
var rawPrice = outcomeItem.SelectTokens("$.prices[*]").FirstOrDefault()
.SelectToken("$.priceSnapshot.current").ToString();
overLine = ScrapeHelper.ConvertMetric(rawOver);
over = ScrapeHelper.ConvertMetric(rawPrice);
}
else if (titleItem.Contains("Under"))
{
var rawUnder = ScrapeHelper.RegexMappingExpression(titleItem, "(?:Under) (.*)");
var rawPrice = outcomeItem.SelectTokens("$.prices[*]").FirstOrDefault()
.SelectToken("$.priceSnapshot.current").ToString();
underLine = ScrapeHelper.ConvertMetric(rawUnder);
under = ScrapeHelper.ConvertMetric(rawPrice);
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = foundMatches[i].Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / metricMarkets.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
Logger.Information($"Total metric count: {totalMetrics}");
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
throw;
}
}
protected override async Task ScrapeData()
{
const string url = "https://api.beta.tab.com.au/v1/tab-info-service/sports/Basketball/competitions/NBA/matches?jurisdiction=VIC";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.matches[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$._links.self").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectTokens("$.contestants[?(@.position == 'HOME')].name")?.FirstOrDefault()?.ToString();
var awayTeam = rawMatch.SelectTokens("$.contestants[?(@.position == 'AWAY')].name")?.FirstOrDefault()?.ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scraping metric data");
foreach (var match in foundMatches)
{
doc = await ScrapeHelper.GetDocument(match.SourceId);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMetrics = jDoc.SelectTokens("$.markets[?(@.betOptionSpectrumId =~ /(605)/)]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var rawMetric in rawMetrics)
{
var propositionPlayerA = rawMetric.SelectToken("$.propositions[0]");
var propositionTie = rawMetric.SelectToken("$.propositions[1]");
var propositionPlayerB = rawMetric.SelectToken("$.propositions[2]");
var playerNameA = ScrapeHelper.RegexMappingExpression(propositionPlayerA.SelectToken("$.name").ToString(), @"(.*)\(");
var playerNameB = ScrapeHelper.RegexMappingExpression(propositionPlayerB.SelectToken("$.name").ToString(), @"(.*)\(");
var playerA = ScrapeHelper.FindPlayerInMatch(playerNameA, match);
if (playerA == null)
{
Logger.Warning($"Cannot find any playerA {playerNameA} in match {match.Id}");
continue;
}
var playerB = ScrapeHelper.FindPlayerInMatch(playerNameB, match);
if (playerB == null)
{
Logger.Warning($"Cannot find any playerB {playerNameB} in match {match.Id}");
continue;
}
var priceA = ScrapeHelper.ConvertMetric(propositionPlayerA.SelectToken("$.returnWin").ToString());
var tiePrice = ScrapeHelper.ConvertMetric(propositionTie.SelectToken("$.returnWin").ToString());
var priceB = ScrapeHelper.ConvertMetric(propositionPlayerB.SelectToken("$.returnWin").ToString());
Logger.Information($"{playerA.Name} vs {playerB.Name}: {priceA} vs {priceB} | Tie: {tiePrice}");
var metric = new PlayerHeadToHead
{
PlayerAId = playerA.Id,
PlayerBId = playerB.Id,
IsTieIncluded = tiePrice != null,
PlayerAPrice = priceA,
PlayerBPrice = priceB,
ScrapingInformationId = GetScrapingInformation().Id,
TiePrice = tiePrice,
MatchId = match.Id,
CreatedAt = DateTime.Now
};
PlayerHeadToHeads.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://betbuilder.digitalsportstech.com/api/latestGames?leagueId=123&sb=topsport&status=1";
var rawDoc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
var rawMatches = jDoc.SelectTokens("$.data[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.homeTeam.title").ToString();
var awayTeam = rawMatch.SelectToken("$.visitingTeam.title").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scraping metric data");
_rawData = new List <PlayerOverUnder>();
foreach (var match in foundMatches)
{
var metricUrl = $"https://betbuilder.digitalsportstech.com/api/feed?betType=in,18,19&gameId={match.SourceId}&limit=9999&sb=topsport&tz=7";
rawDoc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
var rawMetrics = jDoc.SelectTokens("$.data[*]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 80);
foreach (var rawMetric in rawMetrics)
{
var statistic = rawMetric.SelectToken("$.statistic.title").ToString();
var scoreType =
statistic.Contains("Pts + Reb + Ast") ? ScoreType.PointReboundAssist :
statistic.Contains("Total Rebounds") ? ScoreType.Rebound :
statistic.Contains("Assists") ? ScoreType.Assist :
statistic.Contains("Points") ? ScoreType.Point :
statistic.Contains("Three Point") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = rawMetric.SelectToken("$.player1.name").ToString();
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var betType = rawMetric.SelectToken("$.betType").ToString();
double?over = 0, overLine = 0, under = 0, underLine = 0;
switch (betType)
{
case "18":
over = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.markets[0].odds").ToString());
overLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.markets[0].value").ToString());
break;
case "19":
under = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.markets[0].odds").ToString());
underLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.markets[0].value").ToString());
break;
}
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
Player = player,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
_rawData.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
MergeData();
Logger.Information("Merge data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}