public async Task Run(int fixtureId) { try { _logger.LogTrace($"Entering Run({fixtureId})"); FixtureRosters rosters = null; FixtureShots shots = null; FixtureMatchInfo matchInfo = null; using (var conn = await _connectionProvider.GetOpenConnectionAsync()) using (var trans = conn.BeginTransaction()) { var browser = await Puppeteer.ConnectAsync(await _chromeHelper.GetConnectOptionsAsync()); try { using (var page = await browser.NewPageAsync()) { var url = $"https://understat.com/match/{fixtureId}"; _logger.LogInformation("Loading " + url); var response = await page.GoToAsync(url, new NavigationOptions { WaitUntil = new WaitUntilNavigation[] { WaitUntilNavigation.Networkidle2 } }); if (!response.Ok) { _logger.LogError($"Unable to load {url}. Http Status {response.Status}"); return; } _logger.LogInformation($"Success loading {url}"); var sd = await page.EvaluateExpressionAsync("shotsData"); var rd = await page.EvaluateExpressionAsync("rostersData"); var mi = await page.EvaluateExpressionAsync("match_info"); shots = sd.ToObject <FixtureShots>(); rosters = rd.ToObject <FixtureRosters>(); matchInfo = mi.ToObject <FixtureMatchInfo>(); } } finally { browser.Disconnect(); } await _fixtureDetailsManager.ProcessRosters(rosters, shots, matchInfo, conn); await _fixtureDetailsManager.ProcessShots(shots, rosters, matchInfo, conn); await _fixtureDetailsManager.ProcessMatchInfo(matchInfo, conn); await _fixtureDetailsManager.ConfirmDetailsSaved(fixtureId, conn); trans.Commit(); _logger.LogTrace($"Exiting Run({fixtureId})"); } } catch (Exception ex) { _logger.LogError(ex, $"Error running FixtureDetailsScraper for fixture {fixtureId}"); throw; } }
public async Task Run(int competitionId, int?season = null) { try { _logger.LogTrace($"Entering Run for competition {competitionId}"); List <Models.Player> players = null; Dictionary <string, Models.Team> teams = null; List <Models.Fixture> fixtures = null; int seasonFromPage; Entities.Competition competition; using (var conn = await _connectionProvider.GetOpenConnectionAsync()) { var browser = await Puppeteer.ConnectAsync(await _chromeHelper.GetConnectOptionsAsync()); try { using (var page = await browser.NewPageAsync()) { competition = await _competitionRepository.GetByIdAsync(competitionId, conn); if (competition == null) { throw new Exception($"Competition not found with id {competitionId}."); } var url = season != null ? $"https://understat.com/league/{competition.InternalName}/{season}" : $"https://understat.com/league/{competition.InternalName}"; _logger.LogInformation("Loading " + url); var response = await page.GoToAsync(url, new NavigationOptions { WaitUntil = new WaitUntilNavigation[] { WaitUntilNavigation.Networkidle2 } }); if (!response.Ok) { _logger.LogError($"Unable to load {url}. Http Status {response.Status}"); return; } _logger.LogInformation("Attempting to parse season from page"); // Use the select dropdown on the page to decide which season it is seasonFromPage = await page.EvaluateExpressionAsync <int>("+document.querySelector('select[name=season] > option[selected]').value"); _logger.LogInformation($"Season {seasonFromPage} found"); var pd = await page.EvaluateExpressionAsync("playersData"); var td = await page.EvaluateExpressionAsync("teamsData"); var dd = await page.EvaluateExpressionAsync("datesData"); players = pd.ToObject <List <Models.Player> >(); teams = td.ToObject <Dictionary <string, Models.Team> >(); fixtures = dd.ToObject <List <Models.Fixture> >(); } } finally { browser.Disconnect(); } await _leagueSummaryManager.ProcessTeams(teams.Values.ToList(), seasonFromPage, competition, conn); await _leagueSummaryManager.ProcessPlayers(players, seasonFromPage, competition, conn); await _leagueSummaryManager.ProcessFixtures(fixtures, seasonFromPage, competition, conn); } } catch (Exception ex) { _logger.LogError(ex, "Error running LeagueSummaryScraper"); throw; } }