public static async Task Run(
// Every 4 minutes, between 6:00 PM and 7:59 PM every day.
[TimerTrigger("0 */4 18-19 * * *")] TimerInfo timer,
ILogger log)
{
var tableService = new TableService(Environment.GetEnvironmentVariable("AzureWebJobsStorage"));
var playerFeedRow = (await tableService.GetNextPlayerFeedRows(
rowLimit: 1, minimumTimeSinceLastSync: TimeSpan.FromHours(12)))
.SingleOrDefault();
log.LogInformation($"Queried player feeds table for next feed: {playerFeedRow?.ToString() ?? "N/A"}.");
if (playerFeedRow == null)
{
return;
}
var playerRows = await tableService.GetPlayerRows(playerFeedRow);
log.LogInformation($"Queried players table: {playerRows.Count} rows returned.");
var scraper = new Scraper(Environment.GetEnvironmentVariable("TransparentUserAgent"));
var players = await scraper.GetPlayers(playerFeedRow);
log.LogInformation($"Scraped players: {players.Count} found.");
var syncResult = new PlayersSyncResult(playerRows, players);
if (syncResult.DefunctPlayerRows.Any() &&
!bool.Parse(Environment.GetEnvironmentVariable("AllowDefunctPlayerRows")))
{
throw new SyncException("Defunct player rows found, manual intervention required: " +
$"{string.Join(", ", syncResult.DefunctPlayerRows.Select(r => r.ID))}", playerFeedRow.Url);
}
log.LogInformation(syncResult.DefunctPlayerRows.Any()
? $"Defunct players found: {string.Join(", ", syncResult.DefunctPlayerRows.Select(r => r.ID))}"
: "No defunct players found.");
log.LogInformation(syncResult.NewPlayerRows.Any()
? $"New players found: {string.Join(", ", syncResult.NewPlayerRows.Select(r => r.ID))}"
: "No new players found.");
log.LogInformation(syncResult.UpdatedPlayerRows.Any()
? $"Updated players found: {string.Join(", ", syncResult.UpdatedPlayerRows.Select(r => r.ID))}"
: "No updated players found.");
if (syncResult.FoundChanges)
{
await tableService.UpdatePlayersTable(syncResult);
log.LogInformation("Players table updated.");
}
await tableService.RequeuePlayerFeedRow(playerFeedRow, syncResult.FoundChanges);
log.LogInformation("Player feed row requeued.");
}
public async Task GetPlayerRows()
{
var playerFeeds = Enumerable.Range(1, 50)
.Select(i => new PlayerFeed {
Url = i.ToString()
})
.ToArray();
var playerRows = await _tableService.GetPlayerRows(playerFeeds[0]);
Assert.AreEqual(0, playerRows.Count);
var players = Enumerable.Range(1, 50)
.SelectMany(i => Enumerable.Range(1, 5)
.Select(j => new Player
{
ID = $"{i}-{j}",
Name = $"{i}-{j}",
BirthDate = DateTime.UtcNow,
FeedUrl = playerFeeds[i - 1].Url,
}))
.ToArray();
var syncResult = new PlayersSyncResult(Enumerable.Empty <PlayerRow>(), players);
await _tableService.UpdatePlayersTable(syncResult);
playerRows = await _tableService.GetPlayerRows(playerFeeds[0]);
Assert.AreEqual(5, playerRows.Count);
CollectionAssert.AreEquivalent(
new[] { "1-1", "1-2", "1-3", "1-4", "1-5" },
playerRows.Select(r => r.ID).ToArray());
playerRows = await _tableService.GetPlayerRows(playerFeeds[1]);
Assert.AreEqual(5, playerRows.Count);
CollectionAssert.AreEquivalent(
new[] { "2-1", "2-2", "2-3", "2-4", "2-5" },
playerRows.Select(r => r.ID).ToArray());
playerRows = await _tableService.GetPlayerRows(playerFeeds[49]);
Assert.AreEqual(5, playerRows.Count);
CollectionAssert.AreEquivalent(
new[] { "50-1", "50-2", "50-3", "50-4", "50-5" },
playerRows.Select(r => r.ID).ToArray());
playerRows = await _tableService.GetPlayerRows(new PlayerFeed { Url = "51" });
Assert.AreEqual(0, playerRows.Count);
}
public async Task GetNextPlayerRows()
{
var players = Enumerable.Range(1, 10)
.Select(i => new Player {
ID = i.ToString(), BirthDate = DateTime.UtcNow
})
.ToArray();
var syncResult = new PlayersSyncResult(Enumerable.Empty <PlayerRow>(), players);
await _tableService.UpdatePlayersTable(syncResult);
var nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.IsTrue(players[0].Matches(nextPlayerRow));
var nextPlayerRows = await _tableService.GetNextPlayerRows(2, TimeSpan.Zero);
Assert.IsTrue(players[0].Matches(nextPlayerRows[0]));
Assert.IsTrue(players[1].Matches(nextPlayerRows[1]));
Assert.AreEqual(2, nextPlayerRows.Count);
nextPlayerRows = await _tableService.GetNextPlayerRows(2, TimeSpan.FromDays(1));
Assert.AreEqual(0, nextPlayerRows.Count);
}
public Task UpdatePlayersTable(PlayersSyncResult syncResult)
=> _playersTable.ExecuteBatchesAsync(syncResult.DefunctPlayerRows.Select(TableOperation.Delete)
.Concat(syncResult.NewPlayerRows.Select(TableOperation.Insert))
.Concat(syncResult.UpdatedPlayerRows.Select(TableOperation.Replace)));
public async Task RequeuePlayerRow()
{
var playerFeed = new PlayerFeed {
Url = "1"
};
var players = Enumerable.Range(1, 3)
.Select(i => new Player
{
ID = i.ToString(),
FeedUrl = playerFeed.Url,
FirstSeason = DateTime.UtcNow.Year - 5,
LastSeason = DateTime.UtcNow.Year,
BirthDate = DateTime.UtcNow.AddYears(-25)
})
.ToArray();
var syncResult = new PlayersSyncResult(Enumerable.Empty <PlayerRow>(), players);
await _tableService.UpdatePlayersTable(syncResult);
var nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("1", nextPlayerRow.ID);
Assert.IsNull(nextPlayerRow.LastSyncSeason);
Assert.IsNull(nextPlayerRow.LastSyncTimeUtc);
Assert.IsNull(nextPlayerRow.LastSyncWithChangesTimeUtc);
Assert.AreEqual(nextPlayerRow.FirstSeason, nextPlayerRow.GetNextSyncSeason());
await _tableService.RequeuePlayerRow(nextPlayerRow, nextPlayerRow.GetNextSyncSeason(), syncFoundChanges : false);
nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("2", nextPlayerRow.ID);
Assert.IsNull(nextPlayerRow.LastSyncSeason);
Assert.IsNull(nextPlayerRow.LastSyncTimeUtc);
Assert.IsNull(nextPlayerRow.LastSyncWithChangesTimeUtc);
Assert.AreEqual(nextPlayerRow.FirstSeason, nextPlayerRow.GetNextSyncSeason());
await _tableService.RequeuePlayerRow(nextPlayerRow, nextPlayerRow.GetNextSyncSeason(), syncFoundChanges : true);
nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("3", nextPlayerRow.ID);
Assert.IsNull(nextPlayerRow.LastSyncSeason);
Assert.IsNull(nextPlayerRow.LastSyncTimeUtc);
Assert.IsNull(nextPlayerRow.LastSyncWithChangesTimeUtc);
Assert.AreEqual(nextPlayerRow.FirstSeason, nextPlayerRow.GetNextSyncSeason());
await _tableService.RequeuePlayerRow(nextPlayerRow, nextPlayerRow.GetNextSyncSeason(), syncFoundChanges : true);
nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("1", nextPlayerRow.ID);
Assert.AreEqual(nextPlayerRow.FirstSeason, nextPlayerRow.LastSyncSeason);
Assert.IsNotNull(nextPlayerRow.LastSyncTimeUtc);
Assert.IsNull(nextPlayerRow.LastSyncWithChangesTimeUtc);
Assert.AreEqual(nextPlayerRow.FirstSeason + 1, nextPlayerRow.GetNextSyncSeason());
await _tableService.RequeuePlayerRow(nextPlayerRow, nextPlayerRow.GetNextSyncSeason(), syncFoundChanges : true);
nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("2", nextPlayerRow.ID);
Assert.AreEqual(nextPlayerRow.FirstSeason, nextPlayerRow.LastSyncSeason);
Assert.IsNotNull(nextPlayerRow.LastSyncTimeUtc);
Assert.IsNotNull(nextPlayerRow.LastSyncWithChangesTimeUtc);
Assert.AreEqual(nextPlayerRow.FirstSeason + 1, nextPlayerRow.GetNextSyncSeason());
await _tableService.RequeuePlayerRow(nextPlayerRow, nextPlayerRow.GetNextSyncSeason(), syncFoundChanges : true);
var nextPlayerRows = await _tableService.GetNextPlayerRows(3, TimeSpan.Zero);
CollectionAssert.AreEqual(
new[] { "3", "1", "2" },
nextPlayerRows.Select(r => r.ID).ToArray());
nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("3", nextPlayerRow.ID);
nextPlayerRow.FirstSeason = 2000;
nextPlayerRow.LastSeason = 2010;
await _tableService.RequeuePlayerRow(nextPlayerRow, 2010, syncFoundChanges : true);
var playerRows = await _tableService.GetPlayerRows(playerFeed);
Assert.AreEqual("1", playerRows[0].ID);
Assert.AreEqual("2", playerRows[1].ID);
Assert.AreEqual("3", playerRows[2].ID);
Assert.AreEqual(playerRows[0].FirstSeason + 2, playerRows[0].GetNextSyncSeason());
Assert.AreEqual(playerRows[1].FirstSeason + 2, playerRows[1].GetNextSyncSeason());
Assert.AreEqual(playerRows[2].FirstSeason, playerRows[2].GetNextSyncSeason());
nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("1", nextPlayerRow.ID);
await _tableService.RequeuePlayerRow(nextPlayerRow, nextPlayerRow.GetNextSyncSeason(), syncFoundChanges : true);
nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("2", nextPlayerRow.ID);
await _tableService.RequeuePlayerRow(nextPlayerRow, nextPlayerRow.GetNextSyncSeason(), syncFoundChanges : true);
nextPlayerRow = (await _tableService.GetNextPlayerRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("1", nextPlayerRow.ID); // 3 has been deprioritized.
}
public async Task UpdatePlayersTable()
{
var playerFeeds = Enumerable.Range(1, 50)
.Select(i => new PlayerFeed {
Url = i.ToString()
})
.ToArray();
var players = Enumerable.Range(1, 50)
.SelectMany(i => Enumerable.Range(1, 5)
.Select(j => new Player
{
ID = $"{i}-{j}",
Name = $"{i}-{j}",
BirthDate = DateTime.UtcNow,
FeedUrl = playerFeeds[i - 1].Url,
})).ToArray();
var syncResult = new PlayersSyncResult(Enumerable.Empty <PlayerRow>(), players);
await _tableService.UpdatePlayersTable(syncResult);
var playerRowsFeed1 = await _tableService.GetPlayerRows(playerFeeds[0]);
Assert.AreEqual(5, playerRowsFeed1.Count);
var updatedPlayersFeed1 = playerRowsFeed1.Select(r => new Player
{
ID = r.ID,
Name = r.Name + "-updated",
FirstSeason = r.FirstSeason + 1,
LastSeason = r.LastSeason + 1,
BirthDate = r.BirthDate.AddYears(1),
FeedUrl = r.FeedUrl,
}).Concat(Enumerable.Range(6, 5)
.Select(i => new Player
{
ID = $"1-{i}",
Name = $"1-{i}",
BirthDate = DateTime.UtcNow,
FeedUrl = playerFeeds[0].Url,
})).ToArray();
syncResult = new PlayersSyncResult(playerRowsFeed1, updatedPlayersFeed1);
Assert.AreEqual(0, syncResult.DefunctPlayerRows.Count);
Assert.AreEqual(5, syncResult.UpdatedPlayerRows.Count);
Assert.AreEqual(5, syncResult.NewPlayerRows.Count);
await _tableService.UpdatePlayersTable(syncResult);
playerRowsFeed1 = await _tableService.GetPlayerRows(playerFeeds[0]);
Assert.AreEqual(10, playerRowsFeed1.Count);
Assert.IsTrue(playerRowsFeed1.Zip(updatedPlayersFeed1, (r, p) => r.Matches(p)).All(m => m));
syncResult = new PlayersSyncResult(playerRowsFeed1, updatedPlayersFeed1);
Assert.AreEqual(0, syncResult.DefunctPlayerRows.Count);
Assert.AreEqual(0, syncResult.UpdatedPlayerRows.Count);
Assert.AreEqual(0, syncResult.NewPlayerRows.Count);
var playerRowsFeed5 = await _tableService.GetPlayerRows(playerFeeds[4]);
Assert.AreEqual(5, playerRowsFeed5.Count);
var updatedPlayersFeed5 = playerRowsFeed5.Select(r => new Player
{
ID = r.ID,
Name = r.Name,
BirthDate = r.BirthDate,
FeedUrl = r.FeedUrl,
}).Concat(Enumerable.Range(6, 1)
.Select(i => new Player
{
ID = $"5-{i}",
Name = $"5-{i}",
BirthDate = DateTime.UtcNow,
FeedUrl = playerFeeds[4].Url,
})).ToArray();
updatedPlayersFeed5[2].Name += "-updated";
syncResult = new PlayersSyncResult(playerRowsFeed5, updatedPlayersFeed5);
Assert.AreEqual(0, syncResult.DefunctPlayerRows.Count);
Assert.AreEqual(1, syncResult.UpdatedPlayerRows.Count);
Assert.AreEqual(1, syncResult.NewPlayerRows.Count);
await _tableService.UpdatePlayersTable(syncResult);
playerRowsFeed1 = await _tableService.GetPlayerRows(playerFeeds[0]);
playerRowsFeed5 = await _tableService.GetPlayerRows(playerFeeds[4]);
Assert.AreEqual(10, playerRowsFeed1.Count);
Assert.IsTrue(playerRowsFeed1.Zip(updatedPlayersFeed1, (r, p) => r.Matches(p)).All(m => m));
Assert.AreEqual(6, playerRowsFeed5.Count);
Assert.IsTrue(playerRowsFeed5.Zip(updatedPlayersFeed5, (r, p) => r.Matches(p)).All(m => m));
Assert.AreEqual(syncResult.UpdatedPlayerRows.Single().RowKey, playerRowsFeed5[2].RowKey);
}
