public async Task Run(int fixtureId)
{
try
{
_logger.LogTrace($"Entering Run({fixtureId})");
FixtureRosters rosters = null;
FixtureShots shots = null;
FixtureMatchInfo matchInfo = null;
using (var conn = await _connectionProvider.GetOpenConnectionAsync())
using (var trans = conn.BeginTransaction())
{
var browser = await Puppeteer.ConnectAsync(await _chromeHelper.GetConnectOptionsAsync());
try
{
using (var page = await browser.NewPageAsync())
{
var url = $"https://understat.com/match/{fixtureId}";
_logger.LogInformation("Loading " + url);
var response = await page.GoToAsync(url, new NavigationOptions { WaitUntil = new WaitUntilNavigation[] { WaitUntilNavigation.Networkidle2 } });
if (!response.Ok)
{
_logger.LogError($"Unable to load {url}. Http Status {response.Status}");
return;
}
_logger.LogInformation($"Success loading {url}");
var sd = await page.EvaluateExpressionAsync("shotsData");
var rd = await page.EvaluateExpressionAsync("rostersData");
var mi = await page.EvaluateExpressionAsync("match_info");
shots = sd.ToObject <FixtureShots>();
rosters = rd.ToObject <FixtureRosters>();
matchInfo = mi.ToObject <FixtureMatchInfo>();
}
}
finally
{
browser.Disconnect();
}
await _fixtureDetailsManager.ProcessRosters(rosters, shots, matchInfo, conn);
await _fixtureDetailsManager.ProcessShots(shots, rosters, matchInfo, conn);
await _fixtureDetailsManager.ProcessMatchInfo(matchInfo, conn);
await _fixtureDetailsManager.ConfirmDetailsSaved(fixtureId, conn);
trans.Commit();
_logger.LogTrace($"Exiting Run({fixtureId})");
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error running FixtureDetailsScraper for fixture {fixtureId}");
throw;
}
}
public async Task Run(int competitionId, int?season = null)
{
try
{
_logger.LogTrace($"Entering Run for competition {competitionId}");
List <Models.Player> players = null;
Dictionary <string, Models.Team> teams = null;
List <Models.Fixture> fixtures = null;
int seasonFromPage;
Entities.Competition competition;
using (var conn = await _connectionProvider.GetOpenConnectionAsync())
{
var browser = await Puppeteer.ConnectAsync(await _chromeHelper.GetConnectOptionsAsync());
try
{
using (var page = await browser.NewPageAsync())
{
competition = await _competitionRepository.GetByIdAsync(competitionId, conn);
if (competition == null)
{
throw new Exception($"Competition not found with id {competitionId}.");
}
var url = season != null ? $"https://understat.com/league/{competition.InternalName}/{season}" :
$"https://understat.com/league/{competition.InternalName}";
_logger.LogInformation("Loading " + url);
var response = await page.GoToAsync(url, new NavigationOptions { WaitUntil = new WaitUntilNavigation[] { WaitUntilNavigation.Networkidle2 } });
if (!response.Ok)
{
_logger.LogError($"Unable to load {url}. Http Status {response.Status}");
return;
}
_logger.LogInformation("Attempting to parse season from page");
// Use the select dropdown on the page to decide which season it is
seasonFromPage = await page.EvaluateExpressionAsync <int>("+document.querySelector('select[name=season] > option[selected]').value");
_logger.LogInformation($"Season {seasonFromPage} found");
var pd = await page.EvaluateExpressionAsync("playersData");
var td = await page.EvaluateExpressionAsync("teamsData");
var dd = await page.EvaluateExpressionAsync("datesData");
players = pd.ToObject <List <Models.Player> >();
teams = td.ToObject <Dictionary <string, Models.Team> >();
fixtures = dd.ToObject <List <Models.Fixture> >();
}
}
finally
{
browser.Disconnect();
}
await _leagueSummaryManager.ProcessTeams(teams.Values.ToList(), seasonFromPage, competition, conn);
await _leagueSummaryManager.ProcessPlayers(players, seasonFromPage, competition, conn);
await _leagueSummaryManager.ProcessFixtures(fixtures, seasonFromPage, competition, conn);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error running LeagueSummaryScraper");
throw;
}
}
