public Task UpdateGamesTable(GamesSyncResult syncResult)
{
if (syncResult.DefunctGameRows
.Concat(syncResult.NewGameRows)
.Concat(syncResult.UpdatedGameRows)
.Select(r => r.PlayerID).Distinct().Count() > 1)
{
throw new SyncException("Can only update games for one player at a time.");
}
return(_gamesTable.ExecuteBatchesAsync(syncResult.DefunctGameRows.Select(TableOperation.Delete)
.Concat(syncResult.NewGameRows.Select(TableOperation.Insert))
.Concat(syncResult.UpdatedGameRows.Select(TableOperation.Replace))));
}
public async Task UpdateGamesTable_WhenDefunctGamesExist()
{
var player = new Player {
ID = "1", BirthDate = DateTime.UtcNow
};
var games = Enumerable.Range(1, 200)
.Select(i => new Game
{
ID = $"1-{i}",
PlayerID = "1",
Season = 2000,
Date = new DateTime(2000, 1, 1).AddDays(i).AsUtc(),
Points = i,
TotalRebounds = 5
}).ToArray();
var syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.Points <= 120));
Assert.AreEqual(0, syncResult.DefunctGameRows.Count);
Assert.AreEqual(0, syncResult.UpdatedGameRows.Count);
Assert.AreEqual(120, syncResult.NewGameRows.Count);
await _tableService.UpdateGamesTable(syncResult);
var gameRows = await _tableService.GetGameRows(player, 2000);
Assert.AreEqual(120, gameRows.Count);
Assert.AreEqual(20, gameRows.Count(r => r.Points > 100));
games[50].TotalRebounds = 6;
games[60].TotalRebounds = 7;
syncResult = new GamesSyncResult(gameRows, games.Where(g => g.Points <= 100 || g.Points >= 151));
Assert.AreEqual(20, syncResult.DefunctGameRows.Count);
Assert.AreEqual(2, syncResult.UpdatedGameRows.Count);
Assert.AreEqual(50, syncResult.NewGameRows.Count);
await _tableService.UpdateGamesTable(syncResult);
gameRows = await _tableService.GetGameRows(player, 2000);
Assert.AreEqual(150, gameRows.Count);
Assert.AreEqual(1, gameRows.Count(r => r.TotalRebounds == 6));
Assert.AreEqual(1, gameRows.Count(r => r.TotalRebounds == 7));
Assert.AreEqual(148, gameRows.Count(r => r.TotalRebounds == 5));
Assert.AreEqual(50, gameRows.Count(r => r.Points > 100));
}
public async Task GetGameRows()
{
var players = Enumerable.Range(1, 10)
.Select(i => new Player {
ID = i.ToString(), FirstSeason = 2000, LastSeason = 2010
})
.ToArray();
var gameRows = await _tableService.GetGameRows(players[0], 2000);
Assert.AreEqual(0, gameRows.Count);
var games = Enumerable.Range(1, 10)
.SelectMany(i => Enumerable.Range(1, 110)
.Select(j => new Game
{
ID = $"{players[i - 1].ID}-{j}",
PlayerID = players[i - 1].ID,
Season = 2000 + i - 1,
Date = new DateTime(2000 + i - 1, 1, 1).AddDays(j).AsUtc()
}))
.ToArray();
var syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.PlayerID == "1"));
await _tableService.UpdateGamesTable(syncResult);
syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.PlayerID == "5"));
await _tableService.UpdateGamesTable(syncResult);
gameRows = await _tableService.GetGameRows(players[0], 2000);
Assert.AreEqual(110, gameRows.Count);
CollectionAssert.AreEquivalent(
Enumerable.Range(1, 110).Select(i => $"1-{i}").ToArray(),
gameRows.Select(r => r.ID).ToArray());
gameRows = await _tableService.GetGameRows(players[4], 2004);
Assert.AreEqual(110, gameRows.Count);
CollectionAssert.AreEquivalent(
Enumerable.Range(1, 110).Select(i => $"5-{i}").ToArray(),
gameRows.Select(r => r.ID).ToArray());
gameRows = await _tableService.GetGameRows(players[4], 2005);
Assert.AreEqual(0, gameRows.Count);
}
public async Task UpdateGamesTable()
{
var players = Enumerable.Range(1, 10)
.Select(i => new Player {
ID = i.ToString(), BirthDate = DateTime.UtcNow
})
.ToArray();
var games = Enumerable.Range(1, 10)
.SelectMany(i => Enumerable.Range(1, 110)
.Select(j => new Game
{
ID = $"{players[i - 1].ID}-{j}",
PlayerID = players[i - 1].ID,
Season = 2000 + i - 1,
Date = new DateTime(2000 + i - 1, 1, 1).AddDays(j).AsUtc(),
Points = 10,
TotalRebounds = 5
})).ToArray();
var syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.PlayerID == "1"));
await _tableService.UpdateGamesTable(syncResult);
syncResult = new GamesSyncResult(Enumerable.Empty <GameRow>(), games.Where(g => g.PlayerID == "5"));
await _tableService.UpdateGamesTable(syncResult);
var gameRowsPlayer1Season2000 = await _tableService.GetGameRows(players[0], 2000);
Assert.AreEqual(110, gameRowsPlayer1Season2000.Count);
var updatedGamesPlayer1 = gameRowsPlayer1Season2000.Select(r => new Game
{
ID = r.ID,
PlayerID = r.PlayerID,
Season = r.Season,
Date = r.Date,
Points = r.Points + 15,
TotalRebounds = r.TotalRebounds + 10
}).Concat(Enumerable.Range(1, 80)
.Select(i => new Game
{
ID = $"{players[0].ID}-{110 + i}",
PlayerID = players[0].ID,
Season = 2001,
Date = new DateTime(2001, 1, 1).AddDays(i).AsUtc(),
Points = 15,
TotalRebounds = 10
})).ToArray();
syncResult = new GamesSyncResult(gameRowsPlayer1Season2000, updatedGamesPlayer1);
Assert.AreEqual(0, syncResult.DefunctGameRows.Count);
Assert.AreEqual(110, syncResult.UpdatedGameRows.Count);
Assert.AreEqual(80, syncResult.NewGameRows.Count);
await _tableService.UpdateGamesTable(syncResult);
gameRowsPlayer1Season2000 = await _tableService.GetGameRows(players[0], 2000);
var gameRowsPlayer1Season2001 = await _tableService.GetGameRows(players[0], 2001);
Assert.AreEqual(110, gameRowsPlayer1Season2000.Count);
Assert.AreEqual(80, gameRowsPlayer1Season2001.Count);
Assert.IsTrue(gameRowsPlayer1Season2000.Zip(updatedGamesPlayer1.Where(g => g.Season == 2000), (r, g) => r.Matches(g)).All(m => m));
Assert.IsTrue(gameRowsPlayer1Season2001.Zip(updatedGamesPlayer1.Where(g => g.Season == 2001), (r, g) => r.Matches(g)).All(m => m));
syncResult = new GamesSyncResult(gameRowsPlayer1Season2000.Concat(gameRowsPlayer1Season2001), updatedGamesPlayer1);
Assert.AreEqual(0, syncResult.DefunctGameRows.Count);
Assert.AreEqual(0, syncResult.UpdatedGameRows.Count);
Assert.AreEqual(0, syncResult.NewGameRows.Count);
var gameRowsPlayer4Season2000 = await _tableService.GetGameRows(players[3], 2000);
Assert.AreEqual(0, gameRowsPlayer4Season2000.Count);
var gameRowsPlayer5Season2000 = await _tableService.GetGameRows(players[4], 2000);
Assert.AreEqual(0, gameRowsPlayer5Season2000.Count);
var gameRowsPlayer5Season2004 = await _tableService.GetGameRows(players[4], 2004);
Assert.AreEqual(110, gameRowsPlayer5Season2004.Count);
var updatedGamesPlayer5 = gameRowsPlayer5Season2004.Select((r, i) => new Game
{
ID = r.ID,
PlayerID = r.PlayerID,
Season = r.Season,
Date = r.Date,
Points = r.Points + (i % 2 == 0 ? 15 : 0),
TotalRebounds = r.TotalRebounds + (i % 2 == 0 ? 10 : 0)
}).Concat(Enumerable.Range(1, 80)
.Select(i => new Game
{
ID = $"{players[4].ID}-{110 + i}",
PlayerID = players[4].ID,
Season = 2005,
Date = new DateTime(2005, 1, 1).AddDays(i).AsUtc(),
Points = 15,
TotalRebounds = 10
})).ToArray();
syncResult = new GamesSyncResult(gameRowsPlayer5Season2004, updatedGamesPlayer5);
Assert.AreEqual(0, syncResult.DefunctGameRows.Count);
Assert.AreEqual(55, syncResult.UpdatedGameRows.Count);
Assert.AreEqual(80, syncResult.NewGameRows.Count);
await _tableService.UpdateGamesTable(syncResult);
gameRowsPlayer1Season2000 = await _tableService.GetGameRows(players[0], 2000);
gameRowsPlayer1Season2001 = await _tableService.GetGameRows(players[0], 2001);
gameRowsPlayer5Season2004 = await _tableService.GetGameRows(players[4], 2004);
var gameRowsPlayer5Season2005 = await _tableService.GetGameRows(players[4], 2005);
Assert.AreEqual(110, gameRowsPlayer1Season2000.Count);
Assert.AreEqual(80, gameRowsPlayer1Season2001.Count);
Assert.AreEqual(110, gameRowsPlayer5Season2004.Count);
Assert.AreEqual(80, gameRowsPlayer5Season2005.Count);
Assert.IsTrue(gameRowsPlayer1Season2000.Zip(updatedGamesPlayer1.Where(g => g.Season == 2000), (r, g) => r.Matches(g)).All(m => m));
Assert.IsTrue(gameRowsPlayer1Season2001.Zip(updatedGamesPlayer1.Where(g => g.Season == 2001), (r, g) => r.Matches(g)).All(m => m));
Assert.IsTrue(gameRowsPlayer5Season2004.Zip(updatedGamesPlayer5.Where(g => g.Season == 2004), (r, g) => r.Matches(g)).All(m => m));
Assert.IsTrue(gameRowsPlayer5Season2005.Zip(updatedGamesPlayer5.Where(g => g.Season == 2005), (r, g) => r.Matches(g)).All(m => m));
Assert.AreEqual(syncResult.UpdatedGameRows.OrderBy(r => r.ID).First().RowKey, gameRowsPlayer5Season2004.OrderBy(r => r.ID).First().RowKey);
var gameRowsPlayer1 = await _tableService.GetGameRows(players[0]);
var gameRowsPlayer5 = await _tableService.GetGameRows(players[4]);
Assert.AreEqual(190, gameRowsPlayer1.Count);
Assert.AreEqual(190, gameRowsPlayer5.Count);
syncResult = new GamesSyncResult(gameRowsPlayer5Season2004.Concat(gameRowsPlayer5Season2005), updatedGamesPlayer5);
Assert.AreEqual(0, syncResult.DefunctGameRows.Count);
Assert.AreEqual(0, syncResult.UpdatedGameRows.Count);
Assert.AreEqual(0, syncResult.NewGameRows.Count);
}
public static async Task Run(
// Every 3 minutes, between 7:00 PM and 7:59 AM every day except from 4 AM to 5 AM
// when site maintenance seems like it might be happening. Trying to go very easy on them.
[TimerTrigger("0 */3 0-3,5-7,19-23 * * *")] TimerInfo timer,
ILogger log)
{
var tableService = new TableService(Environment.GetEnvironmentVariable("AzureWebJobsStorage"));
var playerRow = (await tableService.GetNextPlayerRows(
rowLimit: 1, minimumTimeSinceLastSync: TimeSpan.FromHours(12)))
.SingleOrDefault();
log.LogInformation($"Queried players table for next player: {playerRow?.ToString() ?? "N/A"}.");
if (playerRow == null)
{
return;
}
int syncSeason = playerRow.GetNextSyncSeason();
var gameRows = await tableService.GetGameRows(playerRow, syncSeason);
log.LogInformation($"Queried games table for {playerRow.Name}'s {syncSeason} season: {gameRows.Count} rows returned.");
var scraper = new Scraper(Environment.GetEnvironmentVariable("TransparentUserAgent"));
var games = await scraper.GetGames(playerRow, syncSeason);
log.LogInformation($"Scraped games for {playerRow.Name}'s {syncSeason} season: {games.Count} found.");
var syncResult = new GamesSyncResult(gameRows, games);
if (syncResult.DefunctGameRows.Count() > gameRows.Count() / 2 &&
!bool.Parse(Environment.GetEnvironmentVariable("AllowDefunctGameRowsMajority")))
{
throw new SyncException("Defunct game rows majority found, manual intervention required: " +
$"{string.Join(", ", syncResult.DefunctGameRows.Select(r => r.ID))}", playerRow.GetGameLogUrl(syncSeason));
}
// Games from the current season get updated relatively frequently. For safety, manually verify
// some older updates. Except for plus-minus, which seems to be getting backfilled a bit right now.
if (syncResult.UpdatedGameRows.Count(r => r.Season < DateTime.UtcNow.Year) > games.Count() / 2 &&
(syncResult.UpdatedFields.Count > 1 || !syncResult.UpdatedFields.Contains(nameof(IGame.PlusMinus))) &&
!bool.Parse(Environment.GetEnvironmentVariable("AllowUpdatedHistoricalGameRowsMajority")))
{
throw new SyncException($"Updated historical game rows majority found, manual intervention required: " +
$"{string.Join(", ", syncResult.UpdatedGameRows.Select(r => r.ID))}{Environment.NewLine}" +
$"Updated fields: {string.Join(", ", syncResult.UpdatedFields)}", playerRow.GetGameLogUrl(syncSeason));
}
log.LogInformation(syncResult.DefunctGameRows.Any()
? $"Defunct games found for {playerRow.Name}'s {syncSeason} season: {string.Join(", ", syncResult.DefunctGameRows.Select(r => r.ID))}"
: $"No defunct games found for {playerRow.Name}'s {syncSeason} season.");
log.LogInformation(syncResult.NewGameRows.Any()
? $"New games found for {playerRow.Name}'s {syncSeason} season: {string.Join(", ", syncResult.NewGameRows.Select(r => r.ID))}"
: $"No new games found for {playerRow.Name}'s {syncSeason} season.");
log.LogInformation(syncResult.UpdatedGameRows.Any()
? $"Updated games found for {playerRow.Name}'s {syncSeason} season: {string.Join(", ", syncResult.UpdatedGameRows.Select(r => r.ID))}"
: $"No updated games found for {playerRow.Name}'s {syncSeason} season.");
if (syncResult.FoundChanges)
{
var queueService = new QueueService(Environment.GetEnvironmentVariable("AzureWebJobsStorage"));
await queueService.AddPlayerToBuildGamesBlobQueue(playerRow.ID);
log.LogInformation($"Added {playerRow.Name} ({playerRow.ID}) to BuildGamesBlob queue.");
await tableService.UpdateGamesTable(syncResult);
log.LogInformation("Games table updated.");
}
await tableService.RequeuePlayerRow(playerRow, syncSeason, syncResult.FoundChanges);
log.LogInformation("Player row requeued.");
}
