public static async Task Run(
// At 5:00 PM every day.
[TimerTrigger("0 0 17 * * *")] TimerInfo timer,
ILogger log)
{
var tableService = new TableService(Environment.GetEnvironmentVariable("AzureWebJobsStorage"));
var playerFeedRows = await tableService.GetPlayerFeedRows();
log.LogInformation($"Queried player feeds table: {playerFeedRows.Count} rows returned.");
var scraper = new Scraper(Environment.GetEnvironmentVariable("TransparentUserAgent"));
var playerFeeds = await scraper.GetPlayerFeeds();
log.LogInformation($"Scraped player feeds: {playerFeeds.Count} found.");
var syncResult = new PlayerFeedsSyncResult(playerFeedRows, playerFeeds);
if (syncResult.DefunctPlayerFeedRows.Any())
{
throw new SyncException("Defunct player feed rows found, manual intervention required: " +
$"{string.Join(", ", syncResult.DefunctPlayerFeedRows.Select(r => r.Url))}");
}
log.LogInformation(syncResult.NewPlayerFeedRows.Any()
? $"New player feeds found: {string.Join(", ", syncResult.NewPlayerFeedRows.Select(r => r.Url))}"
: "No new player feeds found.");
if (syncResult.FoundChanges)
{
await tableService.UpdatePlayerFeedsTable(syncResult);
log.LogInformation("Player feeds table updated.");
}
}
public async Task UpdatePlayerFeedsTable()
{
var playerFeeds = Enumerable.Range(1, 110)
.Select(i => new PlayerFeed {
Url = i.ToString()
});
var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds);
await _tableService.UpdatePlayerFeedsTable(syncResult);
var playerFeedRows = await _tableService.GetPlayerFeedRows();
CollectionAssert.AreEquivalent(
playerFeeds.Select(f => f.Url).ToArray(),
playerFeedRows.Select(r => r.Url).ToArray());
Assert.IsTrue(playerFeedRows.All(r => r.PartitionKey == "0"));
Assert.IsTrue(playerFeedRows.All(r => !r.LastSyncTimeUtc.HasValue));
playerFeeds = Enumerable.Range(1, 112)
.Select(i => new PlayerFeed {
Url = i.ToString()
});
syncResult = new PlayerFeedsSyncResult(playerFeedRows, playerFeeds);
await _tableService.UpdatePlayerFeedsTable(syncResult);
playerFeedRows = await _tableService.GetPlayerFeedRows();
CollectionAssert.AreEquivalent(
playerFeeds.Select(f => f.Url).ToArray(),
playerFeedRows.Select(r => r.Url).ToArray());
Assert.IsTrue(playerFeedRows.All(r => r.PartitionKey == "0"));
Assert.IsTrue(playerFeedRows.All(r => !r.LastSyncTimeUtc.HasValue));
Assert.AreEqual($"112", playerFeedRows.OrderBy(r => r.RowKey).Last().Url);
Assert.IsTrue(playerFeedRows.Zip(playerFeeds, (r, f) => r.Matches(f)).All(m => m));
}
public void DoesntFindChangesWhenAllEmpty()
{
var syncResult = new PlayerFeedsSyncResult(
Enumerable.Empty <PlayerFeedRow>(), Enumerable.Empty <PlayerFeed>());
Assert.AreEqual(0, syncResult.DefunctPlayerFeedRows.Count);
Assert.AreEqual(0, syncResult.NewPlayerFeedRows.Count);
Assert.IsFalse(syncResult.FoundChanges);
}
public async Task RequeuePlayerFeedRow()
{
var playerFeeds = Enumerable.Range(1, 3)
.Select(i => new PlayerFeed {
Url = i.ToString()
})
.ToArray();
var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds);
await _tableService.UpdatePlayerFeedsTable(syncResult);
var nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("1", nextPlayerFeedRow.Url);
Assert.IsNull(nextPlayerFeedRow.LastSyncTimeUtc);
Assert.IsNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc);
await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : false);
nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("2", nextPlayerFeedRow.Url);
Assert.IsNull(nextPlayerFeedRow.LastSyncTimeUtc);
Assert.IsNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc);
await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : true);
nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("3", nextPlayerFeedRow.Url);
Assert.IsNull(nextPlayerFeedRow.LastSyncTimeUtc);
Assert.IsNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc);
await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : true);
nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("1", nextPlayerFeedRow.Url);
Assert.IsNotNull(nextPlayerFeedRow.LastSyncTimeUtc);
Assert.IsNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc);
await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : true);
nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single();
Assert.AreEqual("2", nextPlayerFeedRow.Url);
Assert.IsNotNull(nextPlayerFeedRow.LastSyncTimeUtc);
Assert.IsNotNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc);
await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : true);
var nextPlayerFeedRows = await _tableService.GetNextPlayerFeedRows(3, TimeSpan.Zero);
CollectionAssert.AreEqual(
new[] { "3", "1", "2" },
nextPlayerFeedRows.Select(r => r.Url).ToArray());
}
public void FindsChangesWhenAllNew()
{
var playerFeeds = new[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }
.Select(i => new PlayerFeed {
Url = i.ToString()
})
.ToArray();
var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds);
Assert.IsTrue(playerFeeds.Zip(syncResult.NewPlayerFeedRows).All(p => p.First.Matches(p.Second)));
Assert.AreEqual(0, syncResult.DefunctPlayerFeedRows.Count);
Assert.IsTrue(syncResult.FoundChanges);
}
public void FindsChangesWhenAllDefunct()
{
var playerFeedRows = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }
.Select(i => new PlayerFeedRow {
Url = i.ToString()
})
.ToArray();
var syncResult = new PlayerFeedsSyncResult(playerFeedRows, Enumerable.Empty <PlayerFeed>());
CollectionAssert.AreEqual(playerFeedRows, syncResult.DefunctPlayerFeedRows.ToArray());
Assert.AreEqual(0, syncResult.NewPlayerFeedRows.Count);
Assert.IsTrue(syncResult.FoundChanges);
}
public async Task GetPlayerFeedRows()
{
var playerFeedRows = await _tableService.GetPlayerFeedRows();
Assert.AreEqual(0, playerFeedRows.Count);
var playerFeeds = Enumerable.Range(1, 50)
.Select(i => new PlayerFeed {
Url = i.ToString()
});
var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds);
await _tableService.UpdatePlayerFeedsTable(syncResult);
playerFeedRows = await _tableService.GetPlayerFeedRows();
Assert.AreEqual(50, playerFeedRows.Count);
Assert.IsTrue(playerFeedRows.Zip(playerFeeds, (r, f) => r.Matches(f)).All(m => m));
}
public void DoesntFindChanges()
{
var playerFeedRows = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }
.Select(i => new PlayerFeedRow {
Url = i.ToString()
})
.ToArray();
var playerFeeds = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }
.Select(i => new PlayerFeed {
Url = i.ToString()
})
.ToArray();
var syncResult = new PlayerFeedsSyncResult(playerFeedRows, playerFeeds);
Assert.AreEqual(0, syncResult.DefunctPlayerFeedRows.Count);
Assert.AreEqual(0, syncResult.NewPlayerFeedRows.Count);
Assert.IsFalse(syncResult.FoundChanges);
}
public void FindsChanges()
{
var playerFeedRows = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }
.Select(i => new PlayerFeedRow {
Url = i.ToString()
})
.ToArray();
var playerFeeds = new[] { 0, 1, 2, 3, 6, 7, 8, 10, 11 }
.Select(i => new PlayerFeed {
Url = i.ToString()
})
.ToArray();
var syncResult = new PlayerFeedsSyncResult(playerFeedRows, playerFeeds);
CollectionAssert.AreEquivalent(
playerFeedRows.Where(r => r.Url == "4" || r.Url == "5" || r.Url == "9").ToArray(),
syncResult.DefunctPlayerFeedRows.ToArray());
Assert.IsTrue(playerFeeds.Where(f => f.Url == "0" || f.Url == "10" || f.Url == "11")
.Zip(syncResult.NewPlayerFeedRows).All(p => p.First.Matches(p.Second)));
Assert.IsTrue(syncResult.FoundChanges);
}
public async Task GetNextPlayerFeedRows()
{
var playerFeeds = Enumerable.Range(1, 50)
.Select(i => new PlayerFeed {
Url = i.ToString()
})
.ToArray();
var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds);
await _tableService.UpdatePlayerFeedsTable(syncResult);
var nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single();
Assert.IsTrue(playerFeeds[0].Matches(nextPlayerFeedRow));
var nextPlayerFeedRows = await _tableService.GetNextPlayerFeedRows(2, TimeSpan.Zero);
Assert.IsTrue(playerFeeds[0].Matches(nextPlayerFeedRows[0]));
Assert.IsTrue(playerFeeds[1].Matches(nextPlayerFeedRows[1]));
Assert.AreEqual(2, nextPlayerFeedRows.Count);
nextPlayerFeedRows = await _tableService.GetNextPlayerFeedRows(2, TimeSpan.FromDays(1));
Assert.AreEqual(0, nextPlayerFeedRows.Count);
}
public Task UpdatePlayerFeedsTable(PlayerFeedsSyncResult syncResult)
=> _playerFeedsTable.ExecuteBatchesAsync(syncResult.DefunctPlayerFeedRows.Select(TableOperation.Delete)
.Concat(syncResult.NewPlayerFeedRows.Select(TableOperation.Insert)));
