public Task UpdateGamesTable(GamesSyncResult syncResult) { if (syncResult.DefunctGameRows .Concat(syncResult.NewGameRows) .Concat(syncResult.UpdatedGameRows) .Select(r => r.PlayerID).Distinct().Count() > 1) { throw new SyncException("Can only update games for one player at a time."); } return(_gamesTable.ExecuteBatchesAsync(syncResult.DefunctGameRows.Select(TableOperation.Delete) .Concat(syncResult.NewGameRows.Select(TableOperation.Insert)) .Concat(syncResult.UpdatedGameRows.Select(TableOperation.Replace)))); }
public async Task UpdateGamesTable_WhenDefunctGamesExist() { var player = new Player { ID = "1", BirthDate = DateTime.UtcNow }; var games = Enumerable.Range(1, 200) .Select(i => new Game { ID = $"1-{i}", PlayerID = "1", Season = 2000, Date = new DateTime(2000, 1, 1).AddDays(i).AsUtc(), Points = i, TotalRebounds = 5 }).ToArray(); var syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.Points <= 120)); Assert.AreEqual(0, syncResult.DefunctGameRows.Count); Assert.AreEqual(0, syncResult.UpdatedGameRows.Count); Assert.AreEqual(120, syncResult.NewGameRows.Count); await _tableService.UpdateGamesTable(syncResult); var gameRows = await _tableService.GetGameRows(player, 2000); Assert.AreEqual(120, gameRows.Count); Assert.AreEqual(20, gameRows.Count(r => r.Points > 100)); games[50].TotalRebounds = 6; games[60].TotalRebounds = 7; syncResult = new GamesSyncResult(gameRows, games.Where(g => g.Points <= 100 || g.Points >= 151)); Assert.AreEqual(20, syncResult.DefunctGameRows.Count); Assert.AreEqual(2, syncResult.UpdatedGameRows.Count); Assert.AreEqual(50, syncResult.NewGameRows.Count); await _tableService.UpdateGamesTable(syncResult); gameRows = await _tableService.GetGameRows(player, 2000); Assert.AreEqual(150, gameRows.Count); Assert.AreEqual(1, gameRows.Count(r => r.TotalRebounds == 6)); Assert.AreEqual(1, gameRows.Count(r => r.TotalRebounds == 7)); Assert.AreEqual(148, gameRows.Count(r => r.TotalRebounds == 5)); Assert.AreEqual(50, gameRows.Count(r => r.Points > 100)); }
public async Task GetGameRows() { var players = Enumerable.Range(1, 10) .Select(i => new Player { ID = i.ToString(), FirstSeason = 2000, LastSeason = 2010 }) .ToArray(); var gameRows = await _tableService.GetGameRows(players[0], 2000); Assert.AreEqual(0, gameRows.Count); var games = Enumerable.Range(1, 10) .SelectMany(i => Enumerable.Range(1, 110) .Select(j => new Game { ID = $"{players[i - 1].ID}-{j}", PlayerID = players[i - 1].ID, Season = 2000 + i - 1, Date = new DateTime(2000 + i - 1, 1, 1).AddDays(j).AsUtc() })) .ToArray(); var syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.PlayerID == "1")); await _tableService.UpdateGamesTable(syncResult); syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.PlayerID == "5")); await _tableService.UpdateGamesTable(syncResult); gameRows = await _tableService.GetGameRows(players[0], 2000); Assert.AreEqual(110, gameRows.Count); CollectionAssert.AreEquivalent( Enumerable.Range(1, 110).Select(i => $"1-{i}").ToArray(), gameRows.Select(r => r.ID).ToArray()); gameRows = await _tableService.GetGameRows(players[4], 2004); Assert.AreEqual(110, gameRows.Count); CollectionAssert.AreEquivalent( Enumerable.Range(1, 110).Select(i => $"5-{i}").ToArray(), gameRows.Select(r => r.ID).ToArray()); gameRows = await _tableService.GetGameRows(players[4], 2005); Assert.AreEqual(0, gameRows.Count); }
public async Task UpdateGamesTable() { var players = Enumerable.Range(1, 10) .Select(i => new Player { ID = i.ToString(), BirthDate = DateTime.UtcNow }) .ToArray(); var games = Enumerable.Range(1, 10) .SelectMany(i => Enumerable.Range(1, 110) .Select(j => new Game { ID = $"{players[i - 1].ID}-{j}", PlayerID = players[i - 1].ID, Season = 2000 + i - 1, Date = new DateTime(2000 + i - 1, 1, 1).AddDays(j).AsUtc(), Points = 10, TotalRebounds = 5 })).ToArray(); var syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.PlayerID == "1")); await _tableService.UpdateGamesTable(syncResult); syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.PlayerID == "5")); await _tableService.UpdateGamesTable(syncResult); var gameRowsPlayer1Season2000 = await _tableService.GetGameRows(players[0], 2000); Assert.AreEqual(110, gameRowsPlayer1Season2000.Count); var updatedGamesPlayer1 = gameRowsPlayer1Season2000.Select(r => new Game { ID = r.ID, PlayerID = r.PlayerID, Season = r.Season, Date = r.Date, Points = r.Points + 15, TotalRebounds = r.TotalRebounds + 10 }).Concat(Enumerable.Range(1, 80) .Select(i => new Game { ID = $"{players[0].ID}-{110 + i}", PlayerID = players[0].ID, Season = 2001, Date = new DateTime(2001, 1, 1).AddDays(i).AsUtc(), Points = 15, TotalRebounds = 10 })).ToArray(); syncResult = new GamesSyncResult(gameRowsPlayer1Season2000, updatedGamesPlayer1); Assert.AreEqual(0, syncResult.DefunctGameRows.Count); Assert.AreEqual(110, syncResult.UpdatedGameRows.Count); Assert.AreEqual(80, syncResult.NewGameRows.Count); await _tableService.UpdateGamesTable(syncResult); gameRowsPlayer1Season2000 = await _tableService.GetGameRows(players[0], 2000); var gameRowsPlayer1Season2001 = await _tableService.GetGameRows(players[0], 2001); Assert.AreEqual(110, gameRowsPlayer1Season2000.Count); Assert.AreEqual(80, gameRowsPlayer1Season2001.Count); Assert.IsTrue(gameRowsPlayer1Season2000.Zip(updatedGamesPlayer1.Where(g => g.Season == 2000), (r, g) => r.Matches(g)).All(m => m)); Assert.IsTrue(gameRowsPlayer1Season2001.Zip(updatedGamesPlayer1.Where(g => g.Season == 2001), (r, g) => r.Matches(g)).All(m => m)); syncResult = new GamesSyncResult(gameRowsPlayer1Season2000.Concat(gameRowsPlayer1Season2001), updatedGamesPlayer1); Assert.AreEqual(0, syncResult.DefunctGameRows.Count); Assert.AreEqual(0, syncResult.UpdatedGameRows.Count); Assert.AreEqual(0, syncResult.NewGameRows.Count); var gameRowsPlayer4Season2000 = await _tableService.GetGameRows(players[3], 2000); Assert.AreEqual(0, gameRowsPlayer4Season2000.Count); var gameRowsPlayer5Season2000 = await _tableService.GetGameRows(players[4], 2000); Assert.AreEqual(0, gameRowsPlayer5Season2000.Count); var gameRowsPlayer5Season2004 = await _tableService.GetGameRows(players[4], 2004); Assert.AreEqual(110, gameRowsPlayer5Season2004.Count); var updatedGamesPlayer5 = gameRowsPlayer5Season2004.Select((r, i) => new Game { ID = r.ID, PlayerID = r.PlayerID, Season = r.Season, Date = r.Date, Points = r.Points + (i % 2 == 0 ? 15 : 0), TotalRebounds = r.TotalRebounds + (i % 2 == 0 ? 10 : 0) }).Concat(Enumerable.Range(1, 80) .Select(i => new Game { ID = $"{players[4].ID}-{110 + i}", PlayerID = players[4].ID, Season = 2005, Date = new DateTime(2005, 1, 1).AddDays(i).AsUtc(), Points = 15, TotalRebounds = 10 })).ToArray(); syncResult = new GamesSyncResult(gameRowsPlayer5Season2004, updatedGamesPlayer5); Assert.AreEqual(0, syncResult.DefunctGameRows.Count); Assert.AreEqual(55, syncResult.UpdatedGameRows.Count); Assert.AreEqual(80, syncResult.NewGameRows.Count); await _tableService.UpdateGamesTable(syncResult); gameRowsPlayer1Season2000 = await _tableService.GetGameRows(players[0], 2000); gameRowsPlayer1Season2001 = await _tableService.GetGameRows(players[0], 2001); gameRowsPlayer5Season2004 = await _tableService.GetGameRows(players[4], 2004); var gameRowsPlayer5Season2005 = await _tableService.GetGameRows(players[4], 2005); Assert.AreEqual(110, gameRowsPlayer1Season2000.Count); Assert.AreEqual(80, gameRowsPlayer1Season2001.Count); Assert.AreEqual(110, gameRowsPlayer5Season2004.Count); Assert.AreEqual(80, gameRowsPlayer5Season2005.Count); Assert.IsTrue(gameRowsPlayer1Season2000.Zip(updatedGamesPlayer1.Where(g => g.Season == 2000), (r, g) => r.Matches(g)).All(m => m)); Assert.IsTrue(gameRowsPlayer1Season2001.Zip(updatedGamesPlayer1.Where(g => g.Season == 2001), (r, g) => r.Matches(g)).All(m => m)); Assert.IsTrue(gameRowsPlayer5Season2004.Zip(updatedGamesPlayer5.Where(g => g.Season == 2004), (r, g) => r.Matches(g)).All(m => m)); Assert.IsTrue(gameRowsPlayer5Season2005.Zip(updatedGamesPlayer5.Where(g => g.Season == 2005), (r, g) => r.Matches(g)).All(m => m)); Assert.AreEqual(syncResult.UpdatedGameRows.OrderBy(r => r.ID).First().RowKey, gameRowsPlayer5Season2004.OrderBy(r => r.ID).First().RowKey); var gameRowsPlayer1 = await _tableService.GetGameRows(players[0]); var gameRowsPlayer5 = await _tableService.GetGameRows(players[4]); Assert.AreEqual(190, gameRowsPlayer1.Count); Assert.AreEqual(190, gameRowsPlayer5.Count); syncResult = new GamesSyncResult(gameRowsPlayer5Season2004.Concat(gameRowsPlayer5Season2005), updatedGamesPlayer5); Assert.AreEqual(0, syncResult.DefunctGameRows.Count); Assert.AreEqual(0, syncResult.UpdatedGameRows.Count); Assert.AreEqual(0, syncResult.NewGameRows.Count); }
public static async Task Run( // Every 3 minutes, between 7:00 PM and 7:59 AM every day except from 4 AM to 5 AM // when site maintenance seems like it might be happening. Trying to go very easy on them. [TimerTrigger("0 */3 0-3,5-7,19-23 * * *")] TimerInfo timer, ILogger log) { var tableService = new TableService(Environment.GetEnvironmentVariable("AzureWebJobsStorage")); var playerRow = (await tableService.GetNextPlayerRows( rowLimit: 1, minimumTimeSinceLastSync: TimeSpan.FromHours(12))) .SingleOrDefault(); log.LogInformation($"Queried players table for next player: {playerRow?.ToString() ?? "N/A"}."); if (playerRow == null) { return; } int syncSeason = playerRow.GetNextSyncSeason(); var gameRows = await tableService.GetGameRows(playerRow, syncSeason); log.LogInformation($"Queried games table for {playerRow.Name}'s {syncSeason} season: {gameRows.Count} rows returned."); var scraper = new Scraper(Environment.GetEnvironmentVariable("TransparentUserAgent")); var games = await scraper.GetGames(playerRow, syncSeason); log.LogInformation($"Scraped games for {playerRow.Name}'s {syncSeason} season: {games.Count} found."); var syncResult = new GamesSyncResult(gameRows, games); if (syncResult.DefunctGameRows.Count() > gameRows.Count() / 2 && !bool.Parse(Environment.GetEnvironmentVariable("AllowDefunctGameRowsMajority"))) { throw new SyncException("Defunct game rows majority found, manual intervention required: " + $"{string.Join(", ", syncResult.DefunctGameRows.Select(r => r.ID))}", playerRow.GetGameLogUrl(syncSeason)); } // Games from the current season get updated relatively frequently. For safety, manually verify // some older updates. Except for plus-minus, which seems to be getting backfilled a bit right now. if (syncResult.UpdatedGameRows.Count(r => r.Season < DateTime.UtcNow.Year) > games.Count() / 2 && (syncResult.UpdatedFields.Count > 1 || !syncResult.UpdatedFields.Contains(nameof(IGame.PlusMinus))) && !bool.Parse(Environment.GetEnvironmentVariable("AllowUpdatedHistoricalGameRowsMajority"))) { throw new SyncException($"Updated historical game rows majority found, manual intervention required: " + $"{string.Join(", ", syncResult.UpdatedGameRows.Select(r => r.ID))}{Environment.NewLine}" + $"Updated fields: {string.Join(", ", syncResult.UpdatedFields)}", playerRow.GetGameLogUrl(syncSeason)); } log.LogInformation(syncResult.DefunctGameRows.Any() ? $"Defunct games found for {playerRow.Name}'s {syncSeason} season: {string.Join(", ", syncResult.DefunctGameRows.Select(r => r.ID))}" : $"No defunct games found for {playerRow.Name}'s {syncSeason} season."); log.LogInformation(syncResult.NewGameRows.Any() ? $"New games found for {playerRow.Name}'s {syncSeason} season: {string.Join(", ", syncResult.NewGameRows.Select(r => r.ID))}" : $"No new games found for {playerRow.Name}'s {syncSeason} season."); log.LogInformation(syncResult.UpdatedGameRows.Any() ? $"Updated games found for {playerRow.Name}'s {syncSeason} season: {string.Join(", ", syncResult.UpdatedGameRows.Select(r => r.ID))}" : $"No updated games found for {playerRow.Name}'s {syncSeason} season."); if (syncResult.FoundChanges) { var queueService = new QueueService(Environment.GetEnvironmentVariable("AzureWebJobsStorage")); await queueService.AddPlayerToBuildGamesBlobQueue(playerRow.ID); log.LogInformation($"Added {playerRow.Name} ({playerRow.ID}) to BuildGamesBlob queue."); await tableService.UpdateGamesTable(syncResult); log.LogInformation("Games table updated."); } await tableService.RequeuePlayerRow(playerRow, syncSeason, syncResult.FoundChanges); log.LogInformation("Player row requeued."); }