private void ExtractPlayers(HtmlNodeCollection playerElements, int teamId)
{
for (var playerIdx = 0; playerIdx < playerElements.Count; playerIdx++)
{
var item = playerElements[playerIdx];
const int sourceIdInHref = 7;
var sourceId = item.Attributes["href"].Value.Split('/')[sourceIdInHref];
var playerName = item.InnerText;
playerName = ScrapeHelper.FormatPlayerName(playerName);
Logger.Information($"SourceId {sourceId}, PlayerName {playerName}, TeamId {teamId}");
if (string.IsNullOrEmpty(sourceId) || string.IsNullOrEmpty(playerName))
{
Logger.Warning($"SourceId {sourceId} or PlayerName {playerName} " +
$"is null or empty at index {playerIdx}");
continue;
}
Players.Add(new Player
{
SourceId = sourceId,
Name = playerName,
TeamId = teamId
});
}
}
public EspnCompetitionJob(WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper) : base(webPortalHelper, scrapeHelper)
{
Logger = Helper
.GetLoggerConfig($@"{NBAScrapingService.BaseLoggerPath}\Scrape\Competition\espn-.txt")
.CreateLogger();
ProviderType = typeof(EspnCompetition);
}
public PalmerbetPlayerOverUnderJob(WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper) : base(webPortalHelper, scrapeHelper)
{
Logger = Helper
.GetLoggerConfig($@"{NBAScrapingService.BaseLoggerPath}\Scrape\PlayerOverUnder\palmerbet-.txt")
.CreateLogger();
ProviderType = typeof(PalmerbetPlayerOverUnder);
}
// ------------------------------------------------------------ //
public void Update()
{
var api = HotItemController.CurrentApi;
Cookie cookie = null;
if (!String.IsNullOrEmpty(api.ExchangeHost))
{
cookie = new Cookie("s", HotItemController.Config.SessionKey, "/", api.ExchangeHost);
}
ScrapeHelper.Get(api.UriBuyGems(10000), api.ExchangeHost, cookie, api.ExchangeReferer, CompletedGemPrice, api.UserAgent);
ScrapeHelper.Get(api.UriBuyGold(10000), api.ExchangeHost, cookie, api.ExchangeReferer, CompletedGoldPrice, api.UserAgent);
BuyGemsRules.Money = new Money(BuyGemPrice * 100)
{
Name = "Gems for Gold"
};;
BuyGoldRules.Money = new Money(BuyGoldPrice * 100)
{
Name = "Gold for Gems"
};
if (_isBuyGemActive)
{
foreach (NotifierRule r in BuyGemsRules.Rules)
{
if (r.Compare(BuyGemsRules.Money.TotalCopper))
{
HotItemController.Self.AddNotification(this,
new Event.NotificationEventArgs(0, null, r,
Event.NotificationType
.BuyGems)
{
GemRuleViewModel
=
BuyGemsRules
});
}
}
}
if (_isBuyGoldActive)
{
foreach (NotifierRule r in BuyGoldRules.Rules)
{
if (r.Compare(BuyGoldRules.Money.TotalCopper))
{
HotItemController.Self.AddNotification(this,
new Event.NotificationEventArgs(0, null, r,
Event.NotificationType
.BuyGold)
{
GemRuleViewModel
=
BuyGoldRules
});
}
}
}
}
public TabPlayerHeadToHeadJob(WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper)
: base(webPortalHelper, scrapeHelper)
{
Logger = Helper
.GetLoggerConfig($@"{NBAScrapingService.BaseLoggerPath}\Scrape\PlayerHeadToHead\tab-.txt")
.CreateLogger();
ProviderType = typeof(TabPlayerHeadToHead);
}
public BorgataonlinePlayerOverUnderJob(WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper)
: base(webPortalHelper, scrapeHelper)
{
Logger = Helper
.GetLoggerConfig($@"{NBAScrapingService.BaseLoggerPath}\Scrape\PlayerOverUnder\borgataonline-.txt")
.CreateLogger();
ProviderType = typeof(BorgataonlinePlayerOverUnder);
}
// ===========================================================================
// = Private Methods
// ===========================================================================
private static IEnumerable <Beer> LoadFromWeb()
{
var data = ScrapeHelper.FetchParseAsync("http://www.fynefest.com/?page_id=5", transform: TransformHtml).Result;
var next = data.QuerySelector("h1");
string breweryName = "";
while (next != null)
{
if (!next.Name.Equals("p"))
{
next = next.NextSibling;
continue;
}
if (next.Elements().Any(X => X.Name.Equals("strong", StringComparison.OrdinalIgnoreCase)))
{
breweryName = WebUtility.HtmlDecode(next.Element("strong").InnerText).Trim();
if (breweryName.Contains("–"))
{
breweryName = breweryName.Split('–').First().Trim();
}
next = next.NextSibling;
continue;
}
var record = WebUtility.HtmlDecode(next.InnerText);
var lines = record.Split(new[] { '\n' });
var regex = new Regex(@"^(?<beerName>[^–]+?)\s*–?\s*(?<abv>[0-9]+\.?[0-9]*)%?\s*\((?<caskOrKeg>[KC])\)\s*$");
var match = regex.Match(lines.First());
if (!match.Success)
{
throw new ApplicationException("Invalid beer: " + lines.First());
}
var beerName = match.Groups["beerName"].Value.Trim();
var abv = Decimal.Parse(match.Groups["abv"].Value.Trim());
var isCask = match.Groups["caskOrKeg"].Value.Trim() == "C";
var description = lines.Last().Trim();
var beer = new Beer
{
BreweryName = breweryName,
BeerName = beerName,
ABV = abv,
Id = $"{breweryName}$$${beerName}$$${abv}$$${isCask}",
StyleName = description
};
next = next.NextSibling;
yield return(beer);
}
}
protected ScrapeHandler(ILogger logger, WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper)
{
Logger = logger;
Environment = AppEnvironment.Production;
WebPortalHelper = webPortalHelper;
ScrapeHelper = scrapeHelper;
// Inject
WebPortalHelper.Logger = ScrapeHelper.Logger = Logger;
}
public void GetEstimatedBirthDate()
{
var birthDate = ScrapeHelper.GetEstimatedBirthDate("Bill Allen");
Assert.AreEqual(1945, birthDate.Year);
Assert.AreEqual(1, birthDate.Month);
Assert.AreEqual(1, birthDate.Day);
Assert.AreEqual(TimeSpan.Zero, birthDate.TimeOfDay);
Assert.AreEqual(DateTimeKind.Utc, birthDate.Kind);
}
/// <summary>
/// For debugging purpose only
/// </summary>
protected ScrapeHandler()
{
Logger = Helper
.GetLoggerConfig($@"{Constants.BaseLoggerPath}\Debug\debug-.txt")
.CreateLogger();
Environment = AppEnvironment.Debug;
WebPortalHelper = new WebPortalHelper(new HttpClient());
ScrapeHelper = new ScrapeHelper(new HttpClient(), new WebClient());
//ShouldGetTodayMatches = false;
// Inject
WebPortalHelper.Logger = ScrapeHelper.Logger = Logger;
}
public void SearchTransactions(TransactionType transactionType)
{
if (HotItemController.CurrentApi.IsTransactionApiSupported)
{
var f = new TaskFactory();
var t = f.StartNew(() =>
{
//int count = 10;
//int offset = page * 10;
ScrapeHelper transactionScraper = new ScrapeHelper(HotItemController.Config.SessionKey)
{
TransactionType = transactionType
};
transactionScraper.Finished += new EventHandler <ScrapeFinishedEventArgs>(transactionScraper_Finished);
String uri = HotItemController.CurrentApi.UriTransaction(transactionType, 0, HotItemController.Config.TransactionLimit);
transactionScraper.CrawlString(uri, 0, 0, HotItemController.CurrentApi);
});
}
}
protected override async Task ScrapeData()
{
const string url = "https://services.bovada.lv/services/sports/event/coupon/events/A/description/basketball/nba?marketFilterId=def&preMatchOnly=true&lang=en";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.[*].events[*]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.link").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var description = rawMatch.SelectToken("$.description").ToString();
var desSplits = description.Split('@');
var homeTeam = desSplits[1];
var awayTeam = desSplits[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://services.bovada.lv/services/sports/event/coupon/events/A/description{match.SourceId}?lang=en";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc
.SelectTokens("$.[*].events[*].displayGroups[?(@.description == 'Player Props')].markets[*]")
.ToList();
foreach (var rawMetric in rawMetrics)
{
var description = rawMetric.SelectToken("$.description").ToString();
var scoreType =
description.Contains("Total Points, Rebounds and Assists") ? ScoreType.PointReboundAssist :
description.Contains("Total Points and Rebounds") ? ScoreType.PointRebound :
description.Contains("Total Points and Assists") ? ScoreType.PointAssist :
description.Contains("Total Rebounds and Assists") ? ScoreType.ReboundAssist :
description.Contains("Total Points") ? ScoreType.Point :
description.Contains("Total Rebounds") ? ScoreType.Rebound :
description.Contains("Total Assists") ? ScoreType.Assist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(description, @"- (.*) \(");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var outcomes = rawMetric.SelectTokens("$.outcomes[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcome in outcomes)
{
var des = outcome.SelectToken("$.description").ToString();
if (des.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.handicap").ToString());
var overPrice = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.decimal").ToString());
over = overPrice != null?Math.Round((double)overPrice, 2) : (double?)null;
}
else if (des.Contains("Under"))
{
underLine = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.handicap").ToString());
var underPrice = ScrapeHelper.ConvertMetric(outcome.SelectToken("$.price.decimal").ToString());
under = underPrice != null?Math.Round((double)underPrice, 2) : (double?)null;
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
public static async Task <IScrapeHandler> CreateAsync(Type providerType, ILogger logger,
WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper)
{
var instance = (IScrapeHandler)Activator.CreateInstance(providerType, logger, webPortalHelper, scrapeHelper);
await instance.Initialize(providerType);
return(instance);
}
protected override async Task ScrapeData()
{
const string url = "https://eu-offering.kambicdn.org/offering/v2018/888/listView/basketball.json?lang=en_GB&market=ZZ&client_id=2&channel_id=1&ncid=1572482733163&useCombined=true";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.events[?(@.event.group == 'NBA')]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.event.id").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.event.homeName").ToString();
var awayTeam = rawMatch.SelectToken("$.event.awayName").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
foreach (var match in foundMatches)
{
var metricUrl = $"https://eu-offering.kambicdn.org/offering/v2018/888/betoffer/event/{match.SourceId}.json?lang=en_GB&market=ZZ&client_id=2&channel_id=1&ncid=1005710980&includeParticipants=true";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc.SelectTokens("$.betOffers[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var labelData = rawMetric.SelectToken("$.criterion.label").ToString();
var scoreType = labelData.Contains("Points scored by the player") ? ScoreType.Point :
labelData.Contains("Rebounds by the player") ? ScoreType.Rebound :
labelData.Contains("Assists by the player") ? ScoreType.Assist :
labelData.Contains("Points, rebounds & assists by the player") ? ScoreType.PointReboundAssist :
labelData.Contains("3-point field goals made by the player") ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var overData = rawMetric.SelectToken("$.outcomes[?(@.label =~ /(Over.*)/)]");
var underData = rawMetric.SelectToken("$.outcomes[?(@.label =~ /(Under.*)/)]");
var playerData = overData.SelectToken("$.participant").ToString();
var playerSplits = playerData.Split(',');
var rvPlayer = playerSplits.Reverse().ToList();
var playerName = string.Join(" ", rvPlayer);
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overLineData = overData.SelectToken("$.label").ToString();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, @"Over.* (\d*.\d*)"));
var overOdd = ScrapeHelper.ConvertMetric(overData.SelectToken("$.odds").ToString());
var over = overOdd / 1000;
var underLineData = underData.SelectToken("$.label").ToString();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, @"Under.* (\d*.\d*)"));
var underOdd = ScrapeHelper.ConvertMetric(underData.SelectToken("$.odds").ToString());
var under = underOdd / 1000;
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
public UbetPlayerOverUnder(ILogger logger, WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper)
: base(logger, webPortalHelper, scrapeHelper)
{
}
protected override async Task ScrapeData()
{
const string url = "https://fixture.palmerbet.online/fixtures/sports/1c2eeb3a-6bab-4ac2-b434-165cc350180f/matches?pageSize=25";
var rawResult = await ScrapeHelper.GetDocument(url);
var matchObj = JsonConvert.DeserializeObject <JToken>(rawResult);
var rawMatches = matchObj.SelectTokens("$.matches[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.eventId").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.homeTeam.title").ToString();
var awayTeam = rawMatch.SelectToken("$.awayTeam.title").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rawMetricsTasks = foundMatches
.Select(match => $"https://fixture.palmerbet.online/fixtures/sports/matches/{match.SourceId}/markets?pageSize=1000")
.Select(metricUrl => ScrapeHelper.GetDocument(metricUrl))
.ToList();
var rawMetrics = await Task.WhenAll(rawMetricsTasks);
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = rawMetrics.Length != 0 ? 90 / rawMetrics.Length : 0;
var currentRange = 20;
Logger.Information($"Total raw metrics count {rawMetrics.Length}");
var totalMetrics = 0;
try
{
for (var i = 0; i < rawMetrics.Length; i++)
{
var rawMetric = rawMetrics[i];
var metricObj = JsonConvert.DeserializeObject <JToken>(rawMetric);
var metricMarkets = metricObj.SelectTokens("$.markets[?(@.title=~ /(.* - .*)/)]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var metricMarket in metricMarkets)
{
var title = metricMarket.SelectToken("$.title").ToString();
var scoreType =
title.Contains("Points") ? ScoreType.Point :
title.Contains("Rebounds") ? ScoreType.Rebound :
title.Contains("Assists") ? ScoreType.Assist :
title.Contains("Made Threes") ? ScoreType.ThreePoint :
title.Contains("Pts + Ast") ? ScoreType.PointAssist :
title.Contains("Pts + Reb + Ast") ? ScoreType.PointReboundAssist :
title.Contains("Pts + Reb") ? ScoreType.PointRebound :
title.Contains("Reb + Ast") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = title.Split('-')[0].Trim();
var player = ScrapeHelper.FindPlayerInMatch(playerName, foundMatches[i]);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {foundMatches[i].Id}");
continue;
}
totalMetrics++;
var playerId = metricMarket.SelectToken("$.id").ToString();
var priceUrl = $"https://fixture.palmerbet.online/fixtures/sports/markets/{playerId}";
var priceDoc = await ScrapeHelper.GetDocument(priceUrl, 3000);
var priceObj = JsonConvert.DeserializeObject <JToken>(priceDoc);
if (priceObj == null)
{
Logger.Warning($"priceObj is null for player: {playerId}");
continue;
}
var outcomeItems = priceObj.SelectTokens("$.market.outcomes[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcomeItem in outcomeItems)
{
var titleItem = outcomeItem.SelectToken("$.title").ToString();
if (titleItem.Contains("Over"))
{
var rawOver = ScrapeHelper.RegexMappingExpression(titleItem, "(?:Over) (.*)");
var rawPrice = outcomeItem.SelectTokens("$.prices[*]").FirstOrDefault()
.SelectToken("$.priceSnapshot.current").ToString();
overLine = ScrapeHelper.ConvertMetric(rawOver);
over = ScrapeHelper.ConvertMetric(rawPrice);
}
else if (titleItem.Contains("Under"))
{
var rawUnder = ScrapeHelper.RegexMappingExpression(titleItem, "(?:Under) (.*)");
var rawPrice = outcomeItem.SelectTokens("$.prices[*]").FirstOrDefault()
.SelectToken("$.priceSnapshot.current").ToString();
underLine = ScrapeHelper.ConvertMetric(rawUnder);
under = ScrapeHelper.ConvertMetric(rawPrice);
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = foundMatches[i].Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / metricMarkets.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
Logger.Information($"Total metric count: {totalMetrics}");
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
throw;
}
}
public BorgataonlinePlayerOverUnder(ILogger logger, WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper)
: base(logger, webPortalHelper, scrapeHelper)
{
}
private void ProcessMetric(Match match, IReadOnlyCollection <JToken> entrants,
IReadOnlyCollection <JToken> prices, string scoreType, string actualScoreType)
{
var overMetrics = entrants.Where(x =>
{
var name = x.Children().FirstOrDefault()?.SelectToken("$.name").ToString();
if (scoreType == "PR" || scoreType == "RA")
{
return(name != null && name.Contains(actualScoreType) && name.Contains("Over") && !name.Contains("PRA"));
}
return(name != null && name.Contains(actualScoreType) && name.Contains("Over"));
}).ToList();
var underMetrics = entrants.Where(x =>
{
var name = x.Children().FirstOrDefault()?.SelectToken("$.name").ToString();
if (scoreType == "PR" || scoreType == "RA")
{
return(name != null && name.Contains(actualScoreType) && name.Contains("Under") && !name.Contains("PRA"));
}
return(name != null && name.Contains(actualScoreType) && name.Contains("Under"));
}).ToList();
var playerNames = new HashSet <string>();
foreach (var content in overMetrics
.Select(overMetric => overMetric
.Children()
.FirstOrDefault()?
.SelectToken("$.name").ToString() ?? string.Empty))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(content, "(.*) Over"));
}
foreach (var content in underMetrics
.Select(underMetric => underMetric.Children()
.FirstOrDefault()?
.SelectToken("$.name").ToString() ?? string.Empty))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(content, "(.*) Under"));
}
foreach (var playerName in playerNames)
{
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overContent = overMetrics.FirstOrDefault(x =>
x.Children().FirstOrDefault()?.SelectToken("$.name").ToString().Contains(playerName) ?? false)
?.Children().FirstOrDefault();
var underContent = underMetrics.FirstOrDefault(x =>
x.Children().FirstOrDefault()?.SelectToken("$.name").ToString().Contains(playerName) ?? false)
?.Children().FirstOrDefault();
var overLine =
ScrapeHelper.ConvertMetric(ScrapeHelper
.RegexMappingExpression(overContent?.SelectToken("$.name").ToString(), "Over (.*) .*"));
var underLine =
ScrapeHelper.ConvertMetric(ScrapeHelper
.RegexMappingExpression(underContent?.SelectToken("$.name").ToString(), "Under (.*) .*"));
var priceOverId = overContent?.SelectToken("$.id").ToString() ?? string.Empty;
var priceUnderId = underContent?.SelectToken("$.id").ToString() ?? string.Empty;
var priceOverData = prices.FirstOrDefault(x => x.ToString().Contains(priceOverId));
var priceOverContent = priceOverData?.Children().FirstOrDefault();
var numeratorOver = ScrapeHelper.ConvertMetric(priceOverContent?.SelectToken("$.odds.numerator").ToString());
var denominatorOver = ScrapeHelper.ConvertMetric(priceOverContent?.SelectToken("$.odds.denominator").ToString());
var over = numeratorOver != null && denominatorOver != null ? numeratorOver / denominatorOver + 1 : null;
var priceUnderData = prices.FirstOrDefault(x => x.ToString().Contains(priceUnderId));
var priceUnderContent = priceUnderData?.Children().FirstOrDefault();
var numeratorUnder = ScrapeHelper.ConvertMetric(priceUnderContent.SelectToken("$.odds.numerator").ToString());
var denominatorUnder = ScrapeHelper.ConvertMetric(priceUnderContent.SelectToken("$.odds.denominator").ToString());
var under = numeratorUnder != null && denominatorUnder != null ? numeratorUnder / denominatorUnder + 1 : null;
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://www.5dimes.eu/";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(2000);
var userElement = chromeDriver.FindElementById("customerID");
userElement.SendKeys("5D2543319");
var passwordElement = chromeDriver.FindElementByName("password");
passwordElement.SendKeys("qwerty123!");
var loginButton = chromeDriver.FindElementById("submit1");
loginButton.Click();
await Task.Delay(2000);
var nbaCheckbox = chromeDriver.FindElementByName("Basketball_NBA Props");
nbaCheckbox.Click();
var nbaSubmitElement = chromeDriver.FindElementById("btnContinue");
nbaSubmitElement.Click();
var doc = new HtmlDocument();
doc.LoadHtml(chromeDriver.PageSource);
var rawMatches = doc.DocumentNode.SelectNodes("//html/body/form/div[@id='PageElt']/div/div[@id='Middle']/div[@id='contentRight']/div[@id='ManagedContentWrapper']/div[@id='MainContent']/div[@id='mainContentContainer']/div[@id='readingPaneSplitPane']/div[@id='readingPaneContainer']/div[@id='readingPaneContentContainer']/div[@id='contentContainer']/div[@class='linesContainer']/table/tbody/tr");
for (var i = 0; i < rawMatches.Count; i++)
{
var tdScoreName = rawMatches[i].SelectSingleNode("td[3]");
if (tdScoreName == null)
{
continue;
}
var scoreText = tdScoreName.InnerText;
var scoreType =
scoreText.Contains("points") && !scoreText.Contains("first") && !scoreText.Contains("total") && !scoreText.Contains("wins") ? ScoreType.Point :
scoreText.Contains("assists") ? ScoreType.Assist :
scoreText.Contains("rebounds") ? ScoreType.Rebound :
scoreText.Contains("3-Pt") ? ScoreType.ThreePoint :
scoreText.Contains("pts+reb+ast") ? ScoreType.PointReboundAssist :
scoreText.Contains("pts+reb") ? ScoreType.PointRebound :
scoreText.Contains("pts+ast") ? ScoreType.PointAssist :
scoreText.Contains("reb+ast") ? ScoreType.ReboundAssist
: string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var scoreSplits = scoreText.Split();
var playerName = scoreSplits[1].Replace(".", " ");
var match = ScrapeHelper.FindMatchByPlayerName(playerName, TodayMatches);
if (match == null)
{
continue;
}
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
var overText = rawMatches[i].SelectSingleNode("td[6]").InnerText;
var overLine = ScrapeHelper.ConvertMetric($"{ScrapeHelper.RegexMappingExpression(overText, "o(.*)½")}.05");
var over = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overText, @"½ [\-\+](.*)").Trim());
var underText = rawMatches[i + 1].SelectSingleNode("td[6]").InnerText;
var underLine = ScrapeHelper.ConvertMetric($"{ScrapeHelper.RegexMappingExpression(underText, "u(.*)½")}.05");
var under = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underText, @"½ [\-\+](.*)").Trim());
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://beteasy.com.au/api/sports/sports?filters%5BEventTypes%5D=107&filters%5BMasterCategoryID%5D=37";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(10000);
var pageSource = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var jDocument = JsonConvert.DeserializeObject <JToken>(pageSource);
var rawMatches = jDocument.SelectTokens("$.result.sports.events[*]");
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.MasterEventID").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source match id is null");
continue;
}
var eventName = rawMatch.SelectToken("$.MasterEventName").ToString();
if (string.IsNullOrEmpty(eventName))
{
Logger.Warning("Even name is null");
continue;
}
var homeTeam = eventName.Split('@')[1];
var awayTeam = eventName.Split('@')[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
Logger.Information("Scrape matches done");
await UpdateScrapeStatus(20, "Scrape single metrics data");
var rangeProgress = foundMatches.Count != 0 ? 55 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape single metrics data");
foreach (var match in foundMatches)
{
var singleMetricUrl =
$"https://beteasy.com.au/api/sports/event-group/?id={match.SourceId}&ecGroupOrderByIds%5B%5D=18";
chromeDriver.Navigate().GoToUrl(singleMetricUrl);
await Task.Delay(5000);
currentRange = Math.Min(currentRange + rangeProgress, 55);
var metricPageSource = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var jRawMetrics = JsonConvert.DeserializeObject <JToken>(metricPageSource);
var rawMetrics = jRawMetrics.SelectTokens("$.result.*.BettingType[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var eventName = rawMetric.SelectToken("$.EventName").ToString();
var scoreType =
eventName.Contains("Points") ? ScoreType.Point :
eventName.Contains("Rebounds") ? ScoreType.Rebound :
eventName.Contains("Assists") ? ScoreType.Assist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(eventName, "(.*) (Points|Rebounds|Assists)");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var outcomes = rawMetric.SelectTokens("$.Outcomes[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcome in outcomes)
{
var outcomeName = outcome.SelectToken("$.OutcomeName").ToString();
if (outcomeName.Contains("Over"))
{
var rawOver = ScrapeHelper.RegexMappingExpression(outcomeName, "Over (.*)");
var rawPrice = outcome.SelectToken("$.BetTypes[0]").SelectToken("$.Price").ToString();
overLine = ScrapeHelper.ConvertMetric(rawOver);
over = ScrapeHelper.ConvertMetric(rawPrice);
}
else if (outcomeName.Contains("Under"))
{
var rawUnder = ScrapeHelper.RegexMappingExpression(outcomeName, "Under (.*)");
var rawPrice = outcome.SelectToken("$.BetTypes[0]").SelectToken("$.Price").ToString();
underLine = ScrapeHelper.ConvertMetric(rawUnder);
under = ScrapeHelper.ConvertMetric(rawPrice);
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape single metrics data complete");
await UpdateScrapeStatus(55, "Scrape single metrics data complete");
Logger.Information("Scrape combination metrics data");
rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
currentRange = 55;
await UpdateScrapeStatus(55, "Scrape combination metrics data");
foreach (var match in foundMatches)
{
var combinationMetricUrl =
$"https://beteasy.com.au/api/sports/event-group/?id={match.SourceId}&ecGroupOrderByIds%5B%5D=24";
chromeDriver.Navigate().GoToUrl(combinationMetricUrl);
await Task.Delay(5000);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var metricPageSource = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var jRawMetrics = JsonConvert.DeserializeObject <JToken>(metricPageSource);
var rawMetrics = jRawMetrics.SelectTokens("$.result.*.BettingType[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var eventName = rawMetric.SelectToken("$.EventName").ToString();
var scoreType =
eventName.Contains("Points + Rebounds + Assists") ? ScoreType.PointReboundAssist :
eventName.Contains("Points + Rebounds") ? ScoreType.PointRebound :
eventName.Contains("Points + Assists") ? ScoreType.PointAssist :
eventName.Contains("Rebounds + Assists") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var outcomes = rawMetric.SelectTokens("$.Outcomes[*]").ToList();
Player player = null;
foreach (var outcomeName in outcomes.Select(outcome => outcome.SelectToken("$.OutcomeName").ToString()))
{
if (outcomeName.Contains("Over"))
{
var playerName = ScrapeHelper.RegexMappingExpression(outcomeName, "(.*) Over");
player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player != null)
{
continue;
}
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
}
else if (outcomeName.Contains("Under"))
{
var playerName = ScrapeHelper.RegexMappingExpression(outcomeName, "(.*) Under");
player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player != null)
{
continue;
}
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
}
}
if (player == null)
{
continue;
}
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var outcome in outcomes)
{
var outcomeName = outcome.SelectToken("$.OutcomeName").ToString();
if (outcomeName.Contains("Over"))
{
var rawOver = ScrapeHelper.RegexMappingExpression(outcomeName, "Over (.*)");
var rawPrice = outcome.SelectToken("$.BetTypes[0]").SelectToken("$.Price").ToString();
overLine = string.IsNullOrEmpty(rawOver) ? (double?)null : Convert.ToDouble(rawOver);
over = string.IsNullOrEmpty(rawPrice) ? (double?)null : Convert.ToDouble(rawPrice);
}
else if (outcomeName.Contains("Under"))
{
var rawUnder = ScrapeHelper.RegexMappingExpression(outcomeName, "Under (.*)");
var rawPrice = outcome.SelectToken("$.BetTypes[0]").SelectToken("$.Price").ToString();
underLine = string.IsNullOrEmpty(rawUnder) ? (double?)null : Convert.ToDouble(rawUnder);
under = string.IsNullOrEmpty(rawPrice) ? (double?)null : Convert.ToDouble(rawPrice);
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape combination metrics data complete");
await UpdateScrapeStatus(90, "Scrape combination metrics data complete");
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
protected override async Task ScrapeData()
{
var chromeDriver = new ChromeDriver(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
try
{
const string url = "https://nodejs08.tglab.io/cache/20/en/us/11624/prematch-by-tournaments.json";
chromeDriver.Navigate().GoToUrl(url);
await Task.Delay(2000);
var pageSource = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var jDocument = JsonConvert.DeserializeObject <JToken>(pageSource);
var rawMatches = jDocument.SelectTokens("$.events[*]");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.teams.home").ToString();
var awayTeam = rawMatch.SelectToken("$.teams.away").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
var tempMetrics = new List <PlayerOverUnder>();
foreach (var match in foundMatches)
{
var metricUrl = $"https://nodejs08.tglab.io/cache/20/en/us/{match.SourceId}/single-pre-event.json?hidenseek=95183b1a46914705952f351b5bc156fd8e1e7b57be65d492e09186129ff70e8a1ba8";
chromeDriver.Navigate().GoToUrl(metricUrl);
await Task.Delay(2000);
var pageSourceMarket = ScrapeHelper.RegexMappingExpression(chromeDriver.PageSource, "({.*})");
var marketData = JsonConvert.DeserializeObject <JToken>(pageSourceMarket);
var rawMetrics = marketData.SelectTokens("$.odds.*");
foreach (var rawMetric in rawMetrics)
{
var teamName = rawMetric.SelectToken("$.team_name.en").ToString();
if (!teamName.Contains("total points"))
{
continue;
}
var playerSplits = ScrapeHelper.RegexMappingExpression(teamName, "(.*) total").Split('.');
var playerName = $"{playerSplits[0]} {playerSplits[1]}";
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var metric = tempMetrics.FirstOrDefault(x => x.PlayerId == player.Id);
if (teamName.Contains("OVER"))
{
if (metric != null)
{
metric.OverLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.additional_value_raw").ToString());
metric.Over = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.odd_value").ToString());
}
else
{
var newMetric = new PlayerOverUnder
{
MatchId = match.Id,
PlayerId = player.Id,
OverLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.additional_value_raw").ToString()),
Over = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.odd_value").ToString()),
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
tempMetrics.Add(newMetric);
}
}
else if (teamName.Contains("UNDER"))
{
if (metric != null)
{
metric.UnderLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.additional_value_raw").ToString());
metric.Under = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.odd_value").ToString());
}
else
{
var newMetric = new PlayerOverUnder
{
MatchId = match.Id,
PlayerId = player.Id,
UnderLine = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.additional_value_raw").ToString()),
Under = ScrapeHelper.ConvertMetric(rawMetric.SelectToken("$.odd_value").ToString()),
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
tempMetrics.Add(newMetric);
}
}
}
}
foreach (var metric in tempMetrics)
{
Logger.Information($"{metric.Player.Name}: " +
$"{metric.Over} {metric.OverLine} | " +
$"{metric.Under} {metric.UnderLine}");
}
PlayerUnderOvers.AddRange(tempMetrics);
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
chromeDriver.Quit();
throw;
}
chromeDriver.Quit();
}
public void SearchTransactions(TransactionType transactionType)
{
if (HotItemController.CurrentApi.IsTransactionApiSupported)
{
var f = new TaskFactory();
var t = f.StartNew(() =>
{
//int count = 10;
//int offset = page * 10;
ScrapeHelper transactionScraper = new ScrapeHelper(HotItemController.Config.SessionKey) { TransactionType = transactionType };
transactionScraper.Finished += new EventHandler<ScrapeFinishedEventArgs>(transactionScraper_Finished);
String uri = HotItemController.CurrentApi.UriTransaction(transactionType, 0, HotItemController.Config.TransactionLimit);
transactionScraper.CrawlString(uri, 0, 0, HotItemController.CurrentApi);
});
}
}
protected override async Task ScrapeData()
{
const string url = "https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Competitions/6927/Events";
var rawDoc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
var rawMatches = jDoc.SelectTokens("$.[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
if (rawMatch.SelectToken("$.competitionName").ToString() != "NBA Matches")
{
continue;
}
var sourceMatchId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var matchName = rawMatch.SelectToken("$.name").ToString();
var teams = matchName.Split(new[] { " At " }, StringSplitOptions.None);
var homeTeam = teams[1];
var awayTeam = teams[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
var pointsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/567/Markets";
var reboundsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/568/Markets";
var assistsUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/569/Markets";
var combinationUrl = $"https://www.sportsbet.com.au/apigw/sportsbook-sports/Sportsbook/Sports/Events/{match.SourceId}/MarketGroupings/570/Markets";
currentRange = Math.Min(currentRange + rangeProgress, 90);
var docTasks = new List <Task <string> >
{
ScrapeHelper.GetDocument(pointsUrl),
ScrapeHelper.GetDocument(reboundsUrl),
ScrapeHelper.GetDocument(assistsUrl),
ScrapeHelper.GetDocument(combinationUrl)
};
var results = await Task.WhenAll(docTasks);
var rawMetrics = new List <JToken>();
foreach (var result in results)
{
rawMetrics.AddRange(JsonConvert.DeserializeObject <JToken>(result).SelectTokens("$.[*]"));
}
foreach (var rawMetric in rawMetrics)
{
var metricName = rawMetric.SelectToken("$.name").ToString();
var scoreType =
metricName.Contains("Pts + Reb + Ast") ? ScoreType.PointReboundAssist :
metricName.Contains("Points") ? ScoreType.Point :
metricName.Contains("Assists") ? ScoreType.Assist :
metricName.Contains("Made Threes") ? ScoreType.ThreePoint :
metricName.Contains("Rebounds") ? ScoreType.Rebound :
metricName.Contains("Pts + Reb") ? ScoreType.PointRebound :
metricName.Contains("Pts + Ast") ? ScoreType.PointAssist :
metricName.Contains("Reb + Ast") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = metricName.Split('-')[0];
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var selection = rawMetric.SelectToken("$.selections[0]");
var selectionName = selection.SelectToken("$.name").ToString();
double?over = 0, overLine = 0, under = 0, underLine = 0;
if (selectionName.Contains("Over"))
{
over = ScrapeHelper.ConvertMetric(selection.SelectToken("$.price.winPrice")?.ToString());
overLine = ScrapeHelper.ConvertMetric(selection.SelectToken("$.unformattedHandicap")?.ToString());
}
else if (selectionName.Contains("Under"))
{
under = ScrapeHelper.ConvertMetric(selection.SelectToken("$.price.winPrice")?.ToString());
underLine = ScrapeHelper.ConvertMetric(selection.SelectToken("$.unformattedHandicap")?.ToString());
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
// Fetch all teams from web portal
var teams = await WebPortalHelper.GetTeams(Helper.GetSportCode());
if (teams == null || teams.Count == 0)
{
throw new ArgumentNullException(nameof(teams), "Teams from web portal is null");
}
FromDate = Helper.ToMinTime(FromDate);
ToDate = Helper.ToMaxTime(ToDate);
var totalDays = Convert.ToInt32((ToDate - FromDate).TotalDays);
for (var date = FromDate.Date; date <= ToDate.Date; date = date.AddDays(1))
{
Logger.Information($"Scrape matches: {date.ToShortDateString()}");
await UpdateScrapeStatus(null, $"Scrape matches: {date.ToShortDateString()}");
var url = $"http://site.api.espn.com/apis/site/v2/sports/basketball/nba/scoreboard?lang=en®ion=au&calendartype=blacklist&limit=100&dates={Helper.GetDate(date)}&tz=Australia%2FMelbourne";
var rawResult = await ScrapeHelper.GetDocument(url);
var jObject = JObject.Parse(rawResult);
if (jObject == null)
{
throw new ArgumentNullException(nameof(jObject));
}
var sportCode = Helper.GetSportCode();
var teamCodes = new List <string>();
var competitions = jObject.SelectTokens("$.events[*].competitions[*]");
foreach (var competition in competitions)
{
var homeAbbr = competition.SelectToken("$.competitors[0].team.abbreviation").ToString();
var awayAbbr = competition.SelectToken("$.competitors[1].team.abbreviation").ToString();
if (teams.All(x => x.ShortName != homeAbbr))
{
throw new Exception($"Cannot find any home team named '{homeAbbr}' in teams");
}
if (teams.All(x => x.ShortName != awayAbbr))
{
throw new Exception($"Cannot find any away team named '{awayAbbr}' in teams");
}
teamCodes.Add(homeAbbr);
teamCodes.Add(awayAbbr);
var homeTeamId = teams.First(x => x.ShortName.Equals(homeAbbr)).Id;
var awayTeamId = teams.First(x => x.ShortName.Equals(awayAbbr)).Id;
var homeTeamName = competition.SelectToken("$.competitors[0].team.name").ToString();
var awayTeamName = competition.SelectToken("$.competitors[1].team.name").ToString();
homeTeamName = homeTeamName.Substring(0, Math.Min(homeTeamName.Length, 3)).ToUpper();
awayTeamName = awayTeamName.Substring(0, Math.Min(awayTeamName.Length, 3)).ToUpper();
var gameCode = $"{Helper.GetDate(date, "MMddyyyy")}{homeTeamName}{awayTeamName}";
var gameDate = competition.SelectToken("$.date").ToString();
Logger.Information($"Match: {gameCode}, {homeTeamName} vs {awayTeamName}, {gameDate}");
Matches.Add(new Match
{
StartTime = DateTime.Parse(gameDate),
HomeTeamId = homeTeamId,
AwayTeamId = awayTeamId,
GameCode = gameCode,
SportCode = sportCode
});
}
Logger.Information("Scraped matches complete");
Logger.Information("Scrape players from teams");
const string baseTeamsUrl = "https://www.espn.com/nba/team/stats/_/name";
const string xPathToPlayers = "/html/body/div[1]/div/div/div/div/div[5]/div[2]/div[5]/div[1]/div/section/div/section[1]/div[2]/table/tbody/tr[*]/td/span/a";
var playerTasks = new List <Task <HtmlNodeCollection> >();
foreach (var teamCode in teamCodes)
{
url = $"{baseTeamsUrl}/{teamCode}";
playerTasks.Add(ScrapeHelper.GetInnerHtml(url, xPathToPlayers));
}
var nodes = await Task.WhenAll(playerTasks);
for (var i = 0; i < teamCodes.Count; i++)
{
var teamId = teams.First(x => x.ShortName == teamCodes[i]).Id;
Logger.Information($"Scrape player from: {baseTeamsUrl}/{teamCodes[i]}");
ExtractPlayers(nodes[i], teamId);
}
Logger.Information("Scrape players complete");
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + 90 / totalDays, 90);
await UpdateScrapeStatus(newProgress, $"Scrape matches: {date.ToShortDateString()} complete");
}
}
public EspnFutureCompetition(ILogger logger, WebPortalHelper webPortalHelper, ScrapeHelper scrapeHelper)
: base(logger, webPortalHelper, scrapeHelper)
{
}
public void Search(int page, SearchFilters filters)
{
if (SearchFinished != null)
{
this.CurrentSearchFilters = filters;
TaskFactory f = new TaskFactory();
var t = f.StartNew(() =>
{
int count = HotItemController.CurrentApi.ItemsPerPage;
int offset = (page) * HotItemController.CurrentApi.ItemsPerPage;
ScrapeHelper searchScraper = new ScrapeHelper(Config.SessionKey);
searchScraper.Finished += new EventHandler<DataProvider.Event.ScrapeFinishedEventArgs>(searchScraper_Finished);
string sortingMode = filters.SortingMode.ToString();
if (filters.SortingMode == SortingMode.none)
{
sortingMode = null;
}
IsSearchInProgress = true;
String uri = HotItemController.CurrentApi.UriSearch(
filters.QueryString, offset, count, filters.TypeId, filters.SubTypeId, filters.Rarity, filters.LevelMin, filters.LevelMax, sortingMode, filters.DescendingSorting); //h.UseSearchApi().Add("text", query).Add("offset", offset.ToString()).Add("count", count.ToString()).Generate();
searchScraper.CrawlString(uri, 0, filters.QueryString, HotItemController.CurrentApi);
});
}
}
protected override async Task ScrapeData()
{
const string url = "https://www.betvictor.com/api/top_bets/601600?number_to_show=20&exclude_in_running=true&event_ids_to_exclude=&sport_ids_to_exclude=";
var rawDoc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(rawDoc);
await UpdateScrapeStatus(10, "Scraping match data");
var rawMatches = jDoc.SelectTokens("$.[?(@.meeting_description == 'NBA')]");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.event_id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
if (foundMatches.Select(x => x.SourceId).Contains(sourceMatchId))
{
continue;
}
var game = rawMatch.SelectToken("$.event_description").ToString();
var clubs = game.Split('@');
var homeTeam = clubs[1];
var awayTeam = clubs[0];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
rawDoc = await ScrapeHelper.GetDocument($"https://www.betvictor.com/bv_event_level/en-gb/1/coupons/{match.SourceId}/4787?t=1573192805056");
var jResult = JsonConvert.DeserializeObject <JToken>(rawDoc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jResult.SelectTokens("$.[*].markets[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var description = rawMetric.SelectToken("$.des").ToString();
var scoreType =
description.Contains("Total combined points, assists & rebounds") ? ScoreType.PointReboundAssist :
description.Contains("Total points") ? ScoreType.Point :
description.Contains("Total assists") ? ScoreType.Assist :
description.Contains("Total three pointers") ? ScoreType.ThreePoint :
description.Contains("Total rebounds") ? ScoreType.Rebound :
description.Contains("Total combined points & rebounds") ? ScoreType.PointRebound :
description.Contains("Total combined points & assists") ? ScoreType.PointAssist :
description.Contains("Total combined assists & rebounds") ? ScoreType.ReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = rawMetric.SelectToken("$.opponentDescription").ToString();
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var playerMetrics = rawMetric.SelectTokens("$.o[*]");
double?over = 0, overLine = 0, under = 0, underLine = 0;
foreach (var metricItem in playerMetrics)
{
var des = metricItem.SelectToken("$.des").ToString();
if (des.Contains("Over"))
{
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(des, "Over (.*)"));
over = ScrapeHelper.ConvertMetric(metricItem.SelectToken("$.pr").ToString());
}
else if (des.Contains("Under"))
{
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(des, "Under (.*)"));
under = ScrapeHelper.ConvertMetric(metricItem.SelectToken("$.pr").ToString());
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
private void ThreadRunProcessVerifier()
{
try
{
GuildWars2Status = Notifier.GuildWars2Status.NotRunning;
while (IsRunning)
{
if (_scraper == null)
{
_scraper = new ScrapeHelper(Config.SessionKey);
}
_scraper.EatCookie(Config.SessionKey);
if (_scraper.VerifyKey())
{
CurrentApi = tradingPostApi;
GuildWars2Status = Notifier.GuildWars2Status.FoundKey;
IsSessionKeyValid = true;
break;
}
else
{
CheckGuildWars2Process();
if (_keyHelper == null)
{
_keyHelper = new SessionKeyReader.KeyHelper();
}
if (GuildWars2Running && !_keyHelper.Running)
{
GuildWars2Status = Notifier.GuildWars2Status.SearchingKey;
_keyHelper.UriFound -= new EventHandler<SessionKeyReader.UriFoundEventArgs>(keyHelper_UriFound);
_keyHelper.UriFound += new EventHandler<SessionKeyReader.UriFoundEventArgs>(keyHelper_UriFound);
_keyHelper.FindKey();
}
}
Thread.Sleep(10000);
}
}
catch (ThreadInterruptedException)
{
IsRunning = false;
}
}
protected override async Task ScrapeData()
{
const string url = "https://api.pointsbet.com/api/v2/competitions/7176/events/featured?includeLive=true";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.events[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.key").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source Id is null");
continue;
}
var homeTeam = rawMatch.SelectToken("$.homeTeam").ToString();
var awayTeam = rawMatch.SelectToken("$.awayTeam").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://api.pointsbet.com/api/v2/events/{match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc
.SelectTokens(
@"$.fixedOddsMarkets[?(@.name =~ /(Player Points Over\/Under .*)/ || @.name =~ /(Player Rebounds Over\/Under .*)/ || @.name =~ /(Player Assists Over\/Under .*)/)]")
.ToList();
foreach (var rawMetric in rawMetrics)
{
var metricName = rawMetric.SelectToken("$.name").ToString();
var scoreType =
metricName.Contains("Player Points") ? ScoreType.Point :
metricName.Contains("Player Assists") ? ScoreType.Assist :
metricName.Contains("Player Rebounds") ? ScoreType.Rebound :
metricName.Contains("Player Pts + Rebs + Asts") ? ScoreType.PointReboundAssist :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var outcomeOvers = rawMetric.SelectTokens("$.outcomes[?(@.name =~ /(.* Over .*)/)]").ToList();
var outcomeUnders = rawMetric.SelectTokens("$.outcomes[?(@.name =~ /(.* Under .*)/)]").ToList();
var playerNames = new HashSet <string>();
foreach (var name in outcomeOvers.Select(outcome => outcome.SelectToken("name").ToString()))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(name, "(.*)(?:Over|Under)"));
}
foreach (var name in outcomeUnders.Select(outcome => outcome.SelectToken("name").ToString()))
{
playerNames.Add(ScrapeHelper.RegexMappingExpression(name, "(.*)(?:Over|Under)"));
}
foreach (var playerName in playerNames)
{
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var outcomeOver = outcomeOvers.FirstOrDefault(x =>
x.SelectToken("$.name").ToString().Contains(playerName));
var outcomeUnder = outcomeUnders.FirstOrDefault(x =>
x.SelectToken("$.name").ToString().Contains(playerName));
var rawOver = outcomeOver?.SelectToken("price").ToString();
var rawOverLine =
ScrapeHelper.RegexMappingExpression(outcomeOver?.SelectToken("$.name").ToString(),
"(?:Over) (.*)");
var rawUnder = outcomeUnder?.SelectToken("price").ToString();
var rawUnderLine =
ScrapeHelper.RegexMappingExpression(outcomeUnder?.SelectToken("$.name").ToString(),
"(?:Under) (.*)");
Logger.Information($"{player.Name}: {scoreType} - {rawOver} {rawOverLine} | {rawUnder} {rawUnderLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = ScrapeHelper.ConvertMetric(rawOver),
OverLine = ScrapeHelper.ConvertMetric(rawOverLine),
Under = ScrapeHelper.ConvertMetric(rawUnder),
UnderLine = ScrapeHelper.ConvertMetric(rawUnderLine),
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://cds-api.borgataonline.com/bettingoffer/fixtures?x-bwin-accessid=ZTJhZDliYjgtNTdmOC00Njk0LWIxZmItODI3YzhjZGQ5NmIx&lang=en-US&country=VN&userCountry=VN&streamProviders=unas%2Cperform%2Cimgdge&fixtureTypes=Standard&state=Latest&skip=0&take=50&offerMapping=Filtered&offerCategories=Gridable&fixtureCategories=Gridable,NonGridable,Other&sortBy=Tags&sportIds=7®ionIds=9&competitionIds=6004";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.fixtures[*]");
await UpdateScrapeStatus(10, "Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var participants = rawMatch.SelectTokens("$.participants[*]").ToList();
var homeTeam = participants.Last().SelectToken("$.name.value").ToString();
var awayTeam = participants.First().SelectToken("$.name.value").ToString();
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://cds-api.borgataonline.com/bettingoffer/fixture-view?x-bwin-accessid=ZTJhZDliYjgtNTdmOC00Njk0LWIxZmItODI3YzhjZGQ5NmIx&lang=en-US&country=VN&userCountry=VN&streamProviders=unas%2Cperform%2Cimgdge&offerMapping=All&scoreboardMode=Full&fixtureIds={match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
currentRange = Math.Min(currentRange + rangeProgress, 90);
var rawMetrics = jDoc.SelectTokens("$.fixture.games[*]").ToList();
foreach (var rawMetric in rawMetrics)
{
var nameData = rawMetric.SelectToken("$.name.value").ToString();
var scoreType =
nameData.Contains("How many points") ? ScoreType.Point :
nameData.Contains("How many assists") ? ScoreType.Assist :
nameData.Contains("How many rebounds") ? ScoreType.Rebound :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerName = ScrapeHelper.RegexMappingExpression(nameData, @".* will (.*) \(");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overData = rawMetric.SelectToken("$.results[0]");
var over = ScrapeHelper.ConvertMetric(overData.SelectToken("$.odds").ToString());
var overLineData = overData.SelectToken("$.name.value").ToString();
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overLineData, "Over (.*)"));
var underData = rawMetric.SelectToken("$.results[0]");
var under = ScrapeHelper.ConvertMetric(underData.SelectToken("$.odds").ToString());
var underLineData = underData.SelectToken("$.name.value").ToString();
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underLineData, "Over (.*)"));
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
public virtual void BuildItem(bool force)
{
if (!ItemBuildDone || force)
{
using (var s = new ScrapeHelper(HotItemController.Config.SessionKey))
{
ItemBuildDone = true;
String uri = HotItemController.CurrentApi.UriBuildItem(DataId); //new UriHelper().UseSearchApi().Add("ids", DataId.ToString()).Generate();
s.Finished -= new EventHandler<DataProvider.Event.ScrapeFinishedEventArgs>(sSearch_Finished);
s.Finished += new EventHandler<DataProvider.Event.ScrapeFinishedEventArgs>(sSearch_Finished);
s.CrawlString(uri, DataId, DataId, HotItemController.CurrentApi);
}
}
}
protected override async Task ScrapeData()
{
const string url = "https://api.neds.com.au/v2/sport/event-request?category_ids=%5B%223c34d075-dc14-436d-bfc4-9272a49c2b39%22%5D";
var doc = await ScrapeHelper.GetDocument(new Uri(url));
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectToken("$.events").ToList();
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var matchContent in rawMatches.Select(rawMatch => rawMatch.Children().FirstOrDefault()))
{
if (matchContent == null)
{
Logger.Warning("Match content is null");
continue;
}
var sourceMatchId = matchContent.SelectToken("$.id").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source match id is null");
continue;
}
var matchName = matchContent.SelectToken("$.name").ToString();
var competitionName = matchContent.SelectToken("$.competition.name").ToString();
if (!matchName.Contains(" V ") || !competitionName.Contains("NBA"))
{
continue;
}
var homeTeam = ScrapeHelper.RegexMappingExpression(matchName, "(.*) V");
var awayTeam = ScrapeHelper.RegexMappingExpression(matchName, "V (.*)");
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
Logger.Information("Scraping metric data");
await UpdateScrapeStatus(20, "Scraping metric data");
foreach (var match in foundMatches)
{
var metricUrl = $"https://api.neds.com.au/v2/sport/event-card?id={match.SourceId}";
doc = await ScrapeHelper.GetDocument(new Uri(metricUrl));
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var entrants = jDoc.SelectToken("$.entrants").ToList();
var prices = jDoc.SelectToken("$.prices").ToList();
ProcessMetric(match, entrants, prices, ScoreType.Point, "Points");
ProcessMetric(match, entrants, prices, ScoreType.Rebound, "Rebounds");
ProcessMetric(match, entrants, prices, ScoreType.Assist, "Assists");
ProcessMetric(match, entrants, prices, ScoreType.PointReboundAssist, "PRA");
ProcessMetric(match, entrants, prices, ScoreType.PointRebound, "PR");
ProcessMetric(match, entrants, prices, ScoreType.PointAssist, "PA");
ProcessMetric(match, entrants, prices, ScoreType.ReboundAssist, "RA");
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + 90 / foundMatches.Count, 90);
await UpdateScrapeStatus(newProgress, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
public virtual void Crawl()
{
DataId = DataId;
using (var s = new ScrapeHelper(HotItemController.Config.SessionKey))
{
String uri = HotItemController.CurrentApi.UriListingItem(DataId); //new UriHelper().UseListingApi().AddId(DataId.ToString()).Generate();
s.Finished -= new EventHandler<DataProvider.Event.ScrapeFinishedEventArgs>(s_Finished);
s.Finished += new EventHandler<DataProvider.Event.ScrapeFinishedEventArgs>(s_Finished);
s.CrawlString(uri, DataId, DataId, HotItemController.CurrentApi);
}
}
protected override async Task ScrapeData()
{
const string url = "https://ubet.com/api/sportsViewData/mainevents/false/6698";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.[*]");
var foundMatches = new List <Match>();
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
foreach (var rawMatch in rawMatches)
{
var sourceMatchId = rawMatch.SelectToken("$.MainEventId").ToString();
if (string.IsNullOrEmpty(sourceMatchId))
{
Logger.Warning("Source Id is null");
continue;
}
var mainEventName = rawMatch.SelectToken("$.MainEventName").ToString();
var date = Helper.GetCurrentDate("dd/M");
var teams = mainEventName
.Split(new[] { date }, StringSplitOptions.None)[0]
.Split(new[] { " v " }, StringSplitOptions.None);
var homeTeam = teams[0];
var awayTeam = teams[1];
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceMatchId;
foundMatches.Add(match);
}
Logger.Information("Scrape match data complete");
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scraping metric data");
var betTypes = new HashSet <string>();
foreach (var match in foundMatches)
{
var metricUrl = $"https://ubet.com/api/sportsViewData/markets/false/{match.SourceId}";
doc = await ScrapeHelper.GetDocument(metricUrl);
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMetrics = jDoc
.SelectTokens("$.All[?(@.GroupId == -1)].SubEvents[?(@.LongDisplayName =~ /(.* (Ttl|Total).*)/)]")
.ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var rawMetric in rawMetrics)
{
var betTypeShortName = rawMetric.SelectToken("$.BetTypeShortName").ToString();
var scoreType =
betTypeShortName.Contains("Total Points Scored") ? ScoreType.Point :
betTypeShortName.Contains("Total Pts+Rebound+Assist") ? ScoreType.PointReboundAssist :
string.Empty;
betTypes.Add(betTypeShortName);
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var longDisplayName = rawMetric.SelectToken("$.LongDisplayName").ToString();
var playerName = ScrapeHelper.RegexMappingExpression(longDisplayName, @"(.*)(?:Ttl|Total)");
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
double?over = 0, overLine = 0, under = 0, underLine = 0;
var offers = rawMetric.SelectTokens(@"$.Offers[?(@.OfferName =~ /(.* Over|Under|OV|UN .*)/)]");
foreach (var offer in offers)
{
var offerName = offer.SelectToken("$.OfferName").ToString();
if (offerName.Contains("Over") || offerName.Contains("OV"))
{
over = ScrapeHelper.ConvertMetric(offer.SelectToken("$.WinReturn").ToString());
overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(offerName, @".* (\d*.\d)"));
}
else if (offerName.Contains("Under") || offerName.Contains("UN"))
{
under = ScrapeHelper.ConvertMetric(offer.SelectToken("$.WinReturn").ToString());
underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(offerName, @".* (\d*.\d)"));
}
}
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information($"Unique scoreTypes: {JsonConvert.SerializeObject(betTypes)}"); // notify to update
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
protected override async Task ScrapeData()
{
const string url = "https://api.beta.tab.com.au/v1/tab-info-service/sports/Basketball/competitions/NBA/matches?jurisdiction=VIC";
var doc = await ScrapeHelper.GetDocument(url);
var jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMatches = jDoc.SelectTokens("$.matches[*]");
await UpdateScrapeStatus(10, "Scraping match data");
Logger.Information("Scraping match data");
var foundMatches = new List <Match>();
foreach (var rawMatch in rawMatches)
{
var sourceId = rawMatch.SelectToken("$._links.self").ToString();
if (string.IsNullOrEmpty(sourceId))
{
Logger.Warning("Source match id is null");
continue;
}
var matchName = rawMatch.SelectToken("$.name").ToString();
var homeTeam = ScrapeHelper.RegexMappingExpression(matchName, "(.*) v .*");
var awayTeam = ScrapeHelper.RegexMappingExpression(matchName, ".* v (.*)");
var match = ScrapeHelper.FindMatchByHomeAndAwayTeam(TodayMatches, homeTeam, awayTeam);
if (match == null)
{
Logger.Warning($"Cannot find match with Home team: {homeTeam} and Away team: {awayTeam}");
continue;
}
match.SourceId = sourceId;
foundMatches.Add(match);
}
await UpdateScrapeStatus(20, "Scrape match data complete");
await UpdateScrapeStatus(20, "Scraping metric data");
var rangeProgress = foundMatches.Count != 0 ? 90 / foundMatches.Count : 0;
var currentRange = 20;
Logger.Information("Scrape metric data");
foreach (var match in foundMatches)
{
doc = await ScrapeHelper.GetDocument($"{match.SourceId}");
jDoc = JsonConvert.DeserializeObject <JToken>(doc);
var rawMetrics = jDoc.SelectTokens("$.markets[?(@.betOptionSpectrumId =~ /(837|838|839|1789|1791)/)]").ToList();
currentRange = Math.Min(currentRange + rangeProgress, 90);
foreach (var rawMetric in rawMetrics)
{
var spectrumId = rawMetric.SelectToken("$.betOptionSpectrumId").ToString();
var scoreType = spectrumId == "837" ? ScoreType.Point :
spectrumId == "838" ? ScoreType.Rebound :
spectrumId == "839" ? ScoreType.Assist :
spectrumId == "1789" ? ScoreType.PointReboundAssist :
spectrumId == "1791" ? ScoreType.ThreePoint :
string.Empty;
if (string.IsNullOrEmpty(scoreType))
{
continue;
}
var playerOvers = rawMetric.SelectTokens("$.propositions[?(@.name =~ /(.* Over .*)/)]").ToList();
var playerUnders = rawMetric.SelectTokens("$.propositions[?(@.name =~ /(.* Under .*)/)]").ToList();
var playerNames = new HashSet <string>();
foreach (var playerName in playerOvers.Select(item => item.SelectToken("name").ToString()).Select(itemName => ScrapeHelper.RegexMappingExpression(itemName, "(.*)(?:Over|Under)")))
{
playerNames.Add(playerName);
}
foreach (var playerName in playerUnders.Select(item => item.SelectToken("name").ToString()).Select(itemName => ScrapeHelper.RegexMappingExpression(itemName, "(.*)(?:Over|Under)")))
{
playerNames.Add(playerName);
}
foreach (var playerName in playerNames)
{
var player = ScrapeHelper.FindPlayerInMatch(playerName, match);
if (player == null)
{
Logger.Warning($"Cannot find any player {playerName} in match {match.Id}");
continue;
}
var overOutcomeItem = playerOvers.FirstOrDefault(x => x.SelectToken("$.name").ToString().Contains(playerName));
var overName = overOutcomeItem?.SelectToken("name").ToString() ?? string.Empty;
var overLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(overName, @".* (\d*.\d*) .*"));
var over = ScrapeHelper.ConvertMetric(overOutcomeItem?.SelectToken("returnWin").ToString() ?? string.Empty);
var underOutcomeItem = playerUnders.FirstOrDefault(x => x.SelectToken("$.name").ToString().Contains(playerName));
var underName = underOutcomeItem?.SelectToken("name").ToString() ?? string.Empty;
var underLine = ScrapeHelper.ConvertMetric(ScrapeHelper.RegexMappingExpression(underName, @".* (\d*.\d*) .*"));
var under = ScrapeHelper.ConvertMetric(underOutcomeItem?.SelectToken("returnWin").ToString() ?? string.Empty);
Logger.Information($"{player.Name}: {scoreType} - {over} {overLine} | {under} {underLine}");
var metric = new PlayerOverUnder
{
MatchId = match.Id,
Over = over,
OverLine = overLine,
Under = under,
UnderLine = underLine,
PlayerId = player.Id,
ScoreType = scoreType,
ScrapingInformationId = GetScrapingInformation().Id,
CreatedAt = DateTime.Now
};
PlayerUnderOvers.Add(metric);
}
var newProgress = GetScrapingInformation().Progress;
newProgress = Math.Min(newProgress + currentRange / rawMetrics.Count, currentRange);
await UpdateScrapeStatus(newProgress, null);
}
await UpdateScrapeStatus(currentRange, null);
}
Logger.Information("Scrape metric data complete");
await UpdateScrapeStatus(90, "Scrape metric data complete");
}
