public async Task GetNewsFromCollection_ShouldLoadNewsFromCache_IfRequestIsCached()
{
// Arrange
var feed = Create.RSSFeed();
FeedlerContext.Feeds.Add(feed);
var collection = Create.Collection(feed);
FeedlerContext.Collections.Add(collection);
await FeedlerContext.SaveChangesAsync();
var items = Create.Many(Create.Item).Take(5).ToArray();
FeedLoader.LoadItemsAsync(feed).Returns(items);
// Act
var response1 = await Client.GetAsync($"collections/{collection.Id}/news");
await HttpAssert.IsSuccessResponseAsync(response1);
FeedLoader.ClearReceivedCalls();
var response2 = await Client.GetAsync($"collections/{collection.Id}/news");
await HttpAssert.IsSuccessResponseAsync(response2);
// Assert: calls
await FeedLoader.DidNotReceive().LoadItemsAsync(feed);
}
public async Task GetNewsFromCollection_ShouldFetchCollectionNews()
{
// Arrange
var feeds = Create.Many(Create.RSSFeed).Cast <Feed>().Take(2).ToArray();
FeedlerContext.Feeds.AddRange(feeds);
var collection = Create.Collection(feeds);
FeedlerContext.Collections.Add(collection);
await FeedlerContext.SaveChangesAsync();
var itemsByFeed = new Dictionary <Feed, Item[]>();
foreach (var feed in feeds)
{
itemsByFeed[feed] = Create.Many(Create.Item).Take(3).ToArray();
FeedLoader.LoadItemsAsync(feed).Returns(itemsByFeed[feed]);
}
// Act
var response = await Client.GetAsync($"collections/{collection.Id}/news");
// Assert: response
await HttpAssert.IsSuccessResponseAsync(response);
var responseJson = await response.Content.ReadAsAsync <JArray>();
Assert.Equal(
expected: itemsByFeed.Values.Flatten().Select(i => new { Title = i.Title }).ToHashSet(),
actual: responseJson.Select(j => new { Title = j.String("title") }).ToHashSet()
);
}
public void TimeZonesTest()
{
FeedLoader feedLoader = new FeedLoader();
var result1 = feedLoader.LoadData(DateTime.Now, "ru", 3);
var result2 = feedLoader.LoadData(DateTime.Now, "ru", 0);
var a = from l in result1.Leagues
where
l.Value.IsTop
from e in l.Value.Events
select e.Value;
var b = from l in result2.Leagues
where
l.Value.IsTop
from e in l.Value.Events
select e.Value;
var r1 = a.FirstOrDefault();
var r2 = b.FirstOrDefault();
Console.WriteLine(r1);
Console.WriteLine(r2);
// var keyValuePairs = result.Leagues.Where(pair => string.Equals(pair.Value.CountryName ,"АРГЕНТИНА", StringComparison.InvariantCultureIgnoreCase)).Select(pair => pair.Value.Title).ToList();
}
public void TestMethod1()
{
FeedLoader feedLoader = new FeedLoader();
var result = feedLoader.LoadData(DateTime.Now, "ru", 6);
Assert.IsTrue(result.Leagues.Count > 0, "Leagues are not empty");
// Assert.IsTrue(result.Participants.Count > 0, "Participants are not empty");
string query = "Словения";
// var a = from l in result.Leagues
// from e in l.Value.Events
// where
// string.Equals(e.Value.HomeName, query, StringComparison.CurrentCultureIgnoreCase) ||
// string.Equals(e.Value.AwayName, query, StringComparison.CurrentCultureIgnoreCase)
// select new {League = l.Value, Event = e.Value};
var a = from l in result.Leagues
where
l.Value.IsTop
from e in l.Value.Events
select e.Value;
var r = a.FirstOrDefault();
// var keyValuePairs = result.Leagues.Where(pair => string.Equals(pair.Value.CountryName ,"АРГЕНТИНА", StringComparison.InvariantCultureIgnoreCase)).Select(pair => pair.Value.Title).ToList();
}
public Task StartAsync(CancellationToken cancellationToken)
{
_scheduler = ScheduledTask.Run(async() =>
{
_logger.LogInformation("Initiating update...");
try
{
foreach (var source in await _sources.All().ToListAsync())
{
_logger.LogInformation($"Updating {source.Name}...");
var addedArticleCount = 0;
var articles = await FeedLoader.ArticlesFrom(source);
var urls = source.Articles
.OrderByDescending(article => article.PublishDate)
.Take(articles.Count * 2)
.Select(article => article.Url)
.ToHashSet();
foreach (var article in articles)
{
if (!urls.Contains(article.Url))
{
source.Articles.Add(article);
addedArticleCount++;
}
}
await _sources.Update(source);
_logger.LogInformation($"{source.Name} finished. {addedArticleCount} added.");
}
_logger.LogInformation("Update finished.");
}
catch (Exception ex)
{
_logger.LogError(ex, "Update failed.");
}
}, TimeSpan.FromMinutes(5), _cancellationTokenSource.Token);
return(Task.CompletedTask);
}
public RssFeedController(FeedLoader feedLoader)
{
_feedLoader = feedLoader;
}
public static async Task <string[]> Run(
[TimerTrigger("0 */15 * * * *")] TimerInfo myTimer,
[Table("CveRefreshState", "state", "recent-cves")] CveRefreshState refreshState,
[Table("CveRefreshState")] CloudTable stateUpdater,
ILogger log)
{
var correlationId = Guid.NewGuid().ToString("D").ToLower();
using (var scope = log.BeginScope(correlationId))
{
log.LogInformation($"CheckNistFeed function triggered at: {DateTimeOffset.UtcNow}");
var feedSourceUrl = GetEnvironmentVariable("NistFeedBaseUrl");
var feedName = GetEnvironmentVariable("NistFeedName");
var batchSize = Int32.Parse(GetEnvironmentVariable("CveBatchSize"));
var lastChecked = refreshState != null ?
refreshState.LastChecked :
DateTimeOffset.Now.Subtract(TimeSpan.FromMinutes(240));
var lastFeedUpdate = refreshState != null ?
refreshState.FeedLastUpdated :
DateTime.MinValue;
var minPublishDate = refreshState != null ?
refreshState.LastPublishDate :
lastChecked;
var now = DateTimeOffset.UtcNow;
var loader = new FeedLoader(feedSourceUrl, log);
var meta = await loader.GetLatestFeedMeta(feedName);
var entries = new CveSummary[0];
if (meta.LastUpdated > lastFeedUpdate)
{
entries = await loader.LoadFeed(feedName, lastChecked, minPublishDate);
}
else
{
log.LogInformation($"No changes since last check.");
}
var batches = entries
.Select((item, index) => new { item, index })
.GroupBy(x => x.index / batchSize)
.Select(g => g.Select(x => x.item))
.Select(b => JsonConvert.SerializeObject(b.ToArray()))
.ToArray();
if (refreshState == null)
{
refreshState = new CveRefreshState()
{
PartitionKey = "state",
RowKey = "recent-cves",
ETag = "*",
LastChecked = now,
FeedLastUpdated = meta.LastUpdated,
LastPublishDate = entries.Length > 0 ?
entries.Max(x => x.PublishDate) :
minPublishDate
};
var operation = TableOperation.InsertOrReplace(refreshState);
await stateUpdater.ExecuteAsync(operation);
}
else
{
refreshState.LastChecked = now;
refreshState.FeedLastUpdated = meta.LastUpdated;
refreshState.LastPublishDate = entries.Length > 0 ?
entries.Max(x => x.PublishDate) :
refreshState.LastPublishDate;
}
log.LogInformation($"CheckNistFeed function completed at: {DateTimeOffset.UtcNow}");
return(batches);
}
}
