public static async Task Run( // At 5:00 PM every day. [TimerTrigger("0 0 17 * * *")] TimerInfo timer, ILogger log) { var tableService = new TableService(Environment.GetEnvironmentVariable("AzureWebJobsStorage")); var playerFeedRows = await tableService.GetPlayerFeedRows(); log.LogInformation($"Queried player feeds table: {playerFeedRows.Count} rows returned."); var scraper = new Scraper(Environment.GetEnvironmentVariable("TransparentUserAgent")); var playerFeeds = await scraper.GetPlayerFeeds(); log.LogInformation($"Scraped player feeds: {playerFeeds.Count} found."); var syncResult = new PlayerFeedsSyncResult(playerFeedRows, playerFeeds); if (syncResult.DefunctPlayerFeedRows.Any()) { throw new SyncException("Defunct player feed rows found, manual intervention required: " + $"{string.Join(", ", syncResult.DefunctPlayerFeedRows.Select(r => r.Url))}"); } log.LogInformation(syncResult.NewPlayerFeedRows.Any() ? $"New player feeds found: {string.Join(", ", syncResult.NewPlayerFeedRows.Select(r => r.Url))}" : "No new player feeds found."); if (syncResult.FoundChanges) { await tableService.UpdatePlayerFeedsTable(syncResult); log.LogInformation("Player feeds table updated."); } }
public async Task UpdatePlayerFeedsTable() { var playerFeeds = Enumerable.Range(1, 110) .Select(i => new PlayerFeed { Url = i.ToString() }); var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds); await _tableService.UpdatePlayerFeedsTable(syncResult); var playerFeedRows = await _tableService.GetPlayerFeedRows(); CollectionAssert.AreEquivalent( playerFeeds.Select(f => f.Url).ToArray(), playerFeedRows.Select(r => r.Url).ToArray()); Assert.IsTrue(playerFeedRows.All(r => r.PartitionKey == "0")); Assert.IsTrue(playerFeedRows.All(r => !r.LastSyncTimeUtc.HasValue)); playerFeeds = Enumerable.Range(1, 112) .Select(i => new PlayerFeed { Url = i.ToString() }); syncResult = new PlayerFeedsSyncResult(playerFeedRows, playerFeeds); await _tableService.UpdatePlayerFeedsTable(syncResult); playerFeedRows = await _tableService.GetPlayerFeedRows(); CollectionAssert.AreEquivalent( playerFeeds.Select(f => f.Url).ToArray(), playerFeedRows.Select(r => r.Url).ToArray()); Assert.IsTrue(playerFeedRows.All(r => r.PartitionKey == "0")); Assert.IsTrue(playerFeedRows.All(r => !r.LastSyncTimeUtc.HasValue)); Assert.AreEqual($"112", playerFeedRows.OrderBy(r => r.RowKey).Last().Url); Assert.IsTrue(playerFeedRows.Zip(playerFeeds, (r, f) => r.Matches(f)).All(m => m)); }
public void DoesntFindChangesWhenAllEmpty() { var syncResult = new PlayerFeedsSyncResult( Enumerable.Empty <PlayerFeedRow>(), Enumerable.Empty <PlayerFeed>()); Assert.AreEqual(0, syncResult.DefunctPlayerFeedRows.Count); Assert.AreEqual(0, syncResult.NewPlayerFeedRows.Count); Assert.IsFalse(syncResult.FoundChanges); }
public async Task RequeuePlayerFeedRow() { var playerFeeds = Enumerable.Range(1, 3) .Select(i => new PlayerFeed { Url = i.ToString() }) .ToArray(); var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds); await _tableService.UpdatePlayerFeedsTable(syncResult); var nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single(); Assert.AreEqual("1", nextPlayerFeedRow.Url); Assert.IsNull(nextPlayerFeedRow.LastSyncTimeUtc); Assert.IsNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc); await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : false); nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single(); Assert.AreEqual("2", nextPlayerFeedRow.Url); Assert.IsNull(nextPlayerFeedRow.LastSyncTimeUtc); Assert.IsNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc); await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : true); nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single(); Assert.AreEqual("3", nextPlayerFeedRow.Url); Assert.IsNull(nextPlayerFeedRow.LastSyncTimeUtc); Assert.IsNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc); await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : true); nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single(); Assert.AreEqual("1", nextPlayerFeedRow.Url); Assert.IsNotNull(nextPlayerFeedRow.LastSyncTimeUtc); Assert.IsNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc); await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : true); nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single(); Assert.AreEqual("2", nextPlayerFeedRow.Url); Assert.IsNotNull(nextPlayerFeedRow.LastSyncTimeUtc); Assert.IsNotNull(nextPlayerFeedRow.LastSyncWithChangesTimeUtc); await _tableService.RequeuePlayerFeedRow(nextPlayerFeedRow, syncFoundChanges : true); var nextPlayerFeedRows = await _tableService.GetNextPlayerFeedRows(3, TimeSpan.Zero); CollectionAssert.AreEqual( new[] { "3", "1", "2" }, nextPlayerFeedRows.Select(r => r.Url).ToArray()); }
public void FindsChangesWhenAllNew() { var playerFeeds = new[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 } .Select(i => new PlayerFeed { Url = i.ToString() }) .ToArray(); var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds); Assert.IsTrue(playerFeeds.Zip(syncResult.NewPlayerFeedRows).All(p => p.First.Matches(p.Second))); Assert.AreEqual(0, syncResult.DefunctPlayerFeedRows.Count); Assert.IsTrue(syncResult.FoundChanges); }
public void FindsChangesWhenAllDefunct() { var playerFeedRows = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 } .Select(i => new PlayerFeedRow { Url = i.ToString() }) .ToArray(); var syncResult = new PlayerFeedsSyncResult(playerFeedRows, Enumerable.Empty <PlayerFeed>()); CollectionAssert.AreEqual(playerFeedRows, syncResult.DefunctPlayerFeedRows.ToArray()); Assert.AreEqual(0, syncResult.NewPlayerFeedRows.Count); Assert.IsTrue(syncResult.FoundChanges); }
public async Task GetPlayerFeedRows() { var playerFeedRows = await _tableService.GetPlayerFeedRows(); Assert.AreEqual(0, playerFeedRows.Count); var playerFeeds = Enumerable.Range(1, 50) .Select(i => new PlayerFeed { Url = i.ToString() }); var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds); await _tableService.UpdatePlayerFeedsTable(syncResult); playerFeedRows = await _tableService.GetPlayerFeedRows(); Assert.AreEqual(50, playerFeedRows.Count); Assert.IsTrue(playerFeedRows.Zip(playerFeeds, (r, f) => r.Matches(f)).All(m => m)); }
public void DoesntFindChanges() { var playerFeedRows = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 } .Select(i => new PlayerFeedRow { Url = i.ToString() }) .ToArray(); var playerFeeds = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 } .Select(i => new PlayerFeed { Url = i.ToString() }) .ToArray(); var syncResult = new PlayerFeedsSyncResult(playerFeedRows, playerFeeds); Assert.AreEqual(0, syncResult.DefunctPlayerFeedRows.Count); Assert.AreEqual(0, syncResult.NewPlayerFeedRows.Count); Assert.IsFalse(syncResult.FoundChanges); }
public void FindsChanges() { var playerFeedRows = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 } .Select(i => new PlayerFeedRow { Url = i.ToString() }) .ToArray(); var playerFeeds = new[] { 0, 1, 2, 3, 6, 7, 8, 10, 11 } .Select(i => new PlayerFeed { Url = i.ToString() }) .ToArray(); var syncResult = new PlayerFeedsSyncResult(playerFeedRows, playerFeeds); CollectionAssert.AreEquivalent( playerFeedRows.Where(r => r.Url == "4" || r.Url == "5" || r.Url == "9").ToArray(), syncResult.DefunctPlayerFeedRows.ToArray()); Assert.IsTrue(playerFeeds.Where(f => f.Url == "0" || f.Url == "10" || f.Url == "11") .Zip(syncResult.NewPlayerFeedRows).All(p => p.First.Matches(p.Second))); Assert.IsTrue(syncResult.FoundChanges); }
public async Task GetNextPlayerFeedRows() { var playerFeeds = Enumerable.Range(1, 50) .Select(i => new PlayerFeed { Url = i.ToString() }) .ToArray(); var syncResult = new PlayerFeedsSyncResult(Enumerable.Empty <PlayerFeedRow>(), playerFeeds); await _tableService.UpdatePlayerFeedsTable(syncResult); var nextPlayerFeedRow = (await _tableService.GetNextPlayerFeedRows(1, TimeSpan.Zero)).Single(); Assert.IsTrue(playerFeeds[0].Matches(nextPlayerFeedRow)); var nextPlayerFeedRows = await _tableService.GetNextPlayerFeedRows(2, TimeSpan.Zero); Assert.IsTrue(playerFeeds[0].Matches(nextPlayerFeedRows[0])); Assert.IsTrue(playerFeeds[1].Matches(nextPlayerFeedRows[1])); Assert.AreEqual(2, nextPlayerFeedRows.Count); nextPlayerFeedRows = await _tableService.GetNextPlayerFeedRows(2, TimeSpan.FromDays(1)); Assert.AreEqual(0, nextPlayerFeedRows.Count); }
public Task UpdatePlayerFeedsTable(PlayerFeedsSyncResult syncResult) => _playerFeedsTable.ExecuteBatchesAsync(syncResult.DefunctPlayerFeedRows.Select(TableOperation.Delete) .Concat(syncResult.NewPlayerFeedRows.Select(TableOperation.Insert)));