public async Task PreLoadsLatestFundingItemDocumentsByPageSize_ExampleOne()
{
int pageSize = 20;
int preloadCount = 50;
SearchFeedV3 <PublishedFundingIndex> pageOne = NewV3SearchFeed(_ => _.WithFeedItems(NewPublishedFundingIndex(),
NewPublishedFundingIndex(),
NewPublishedFundingIndex(),
NewPublishedFundingIndex()
));
SearchFeedV3 <PublishedFundingIndex> pageTwo = NewV3SearchFeed(_ => _.WithFeedItems(NewPublishedFundingIndex(),
NewPublishedFundingIndex(),
NewPublishedFundingIndex(),
NewPublishedFundingIndex()
));
SearchFeedV3 <PublishedFundingIndex> pageThree = NewV3SearchFeed(_ => _.WithFeedItems(NewPublishedFundingIndex(),
NewPublishedFundingIndex(),
NewPublishedFundingIndex(),
NewPublishedFundingIndex()
));
GivenTheSettings(pageSize, preloadCount, true);
AndTheSearchFeedPage(1, pageSize, pageOne);
AndTheSearchFeedPage(2, pageSize, pageTwo);
AndTheSearchFeedPage(3, pageSize, pageThree);
await WhenThePreloadIsRun();
ThenTheFeedItemDocumentsWerePreLoaded(pageOne.Entries.Select(_ => _.DocumentPath)
.Concat(pageTwo.Entries.Select(_ => _.DocumentPath)
.Concat(pageThree.Entries.Select(_ => _.DocumentPath))));
}
public void Getters_WhenOnSubscriptionPage_ShouldReturnCorrectGeneratedValues()
{
// arrange
SearchFeedV3 <string> SearchFeedV3 = new SearchFeedV3 <string>
{
Top = 10,
TotalCount = 45,
PageRef = 5
};
// assert & act
SearchFeedV3
.Current
.Should().BeNull();
SearchFeedV3
.PreviousArchive
.Should().Be(4);
SearchFeedV3
.NextArchive
.Should().BeNull();
SearchFeedV3
.Current
.Should().BeNull();
SearchFeedV3
.TotalPages
.Should().Be(5);
SearchFeedV3
.IsArchivePage
.Should().BeFalse();
}
private void AndTheSearchFeedPage(int page, int pageSize, SearchFeedV3 <PublishedFundingIndex> result)
{
_searchService.GetFeedsV3(page,
pageSize,
null, null, null)
.Returns(result);
}
public void Getters_WhenOnAMiddlePage_ShouldReturnCorrectGeneratedValues()
{
// arrange
SearchFeedV3 <string> SearchFeedV3 = new SearchFeedV3 <string>
{
Top = 10,
TotalCount = 50,
PageRef = 3
};
// assert & act
SearchFeedV3
.Current
.Should().Be(3);
SearchFeedV3
.PreviousArchive
.Should().Be(2);
SearchFeedV3
.NextArchive
.Should().Be(4);
SearchFeedV3
.TotalPages
.Should().Be(5);
SearchFeedV3
.IsArchivePage
.Should().BeTrue();
}
public async Task RetrievesAndReversesTheLastPageIfNoPageRefSupplied()
{
int totalCount = 257;
int top = 50;
IEnumerable <PublishedFundingIndex> sourceResults = NewResults(
NewPublishedFundingIndex(),
NewPublishedFundingIndex(),
NewPublishedFundingIndex());
GivenTheTotalCount(totalCount);
AndTheFundingFeedResults(top, null, totalCount, sourceResults);
SearchFeedV3 <PublishedFundingIndex> fundingFeedResults = await WhenTheFeedIsRequested(null, top);
fundingFeedResults
.Should()
.NotBeNull();
fundingFeedResults
.Top
.Should()
.Be(top);
fundingFeedResults
.TotalCount
.Should()
.Be(totalCount);
fundingFeedResults
.Entries
.Should()
.BeEquivalentTo(sourceResults.Reverse(),
opt => opt.WithStrictOrdering());
}
public async Task GetNotifications_GivenSearchFeedReturnsNoResultsForTheGivenPage_ReturnsNotFoundResult()
{
//Arrange
SearchFeedV3 <PublishedFundingIndex> feeds = new SearchFeedV3 <PublishedFundingIndex>()
{
TotalCount = 1000,
Entries = Enumerable.Empty <PublishedFundingIndex>()
};
IFundingFeedSearchService feedsSearchService = CreateSearchService();
feedsSearchService
.GetFeedsV3(Arg.Is(3), Arg.Is(500))
.Returns(feeds);
Mock <IExternalEngineOptions> externalEngineOptions = new Mock <IExternalEngineOptions>();
externalEngineOptions
.Setup(_ => _.BlobLookupConcurrencyCount)
.Returns(10);
FundingFeedService service = CreateService(feedsSearchService,
externalEngineOptions: externalEngineOptions.Object);
HttpRequest request = Substitute.For <HttpRequest>();
//Act
IActionResult result = await service.GetFunding(request, pageRef : 3);
//Assert
result
.Should()
.BeOfType <NotFoundResult>();
}
public async Task <IActionResult> GetFunding(HttpRequest request,
int?pageRef,
IEnumerable <string> fundingStreamIds = null,
IEnumerable <string> fundingPeriodIds = null,
IEnumerable <Models.GroupingReason> groupingReasons = null,
IEnumerable <Models.VariationReason> variationReasons = null,
int?pageSize = MaxRecords)
{
pageSize = pageSize ?? MaxRecords;
if (pageRef < 1)
{
return(new BadRequestObjectResult("Page ref should be at least 1"));
}
if (pageSize < 1 || pageSize > 500)
{
return(new BadRequestObjectResult($"Page size should be more that zero and less than or equal to {MaxRecords}"));
}
SearchFeedV3 <PublishedFundingIndex> searchFeed = await _feedService.GetFeedsV3(
pageRef, pageSize.Value, fundingStreamIds, fundingPeriodIds,
groupingReasons?.Select(x => x.ToString()),
variationReasons?.Select(x => x.ToString()));
if (searchFeed == null || searchFeed.TotalCount == 0 || searchFeed.Entries.IsNullOrEmpty())
{
return(new NotFoundResult());
}
AtomFeed <AtomEntry> atomFeed = await CreateAtomFeed(searchFeed, request);
return(new OkObjectResult(atomFeed));
}
private async Task PreLoadPage(PageNumber pageNumber, SemaphoreSlim pageThrottle)
{
try
{
SearchFeedV3 <PublishedFundingIndex> resultsPage = await _searchService.GetFeedsV3(pageNumber.Value, _settings.PageSize,
null,
null,
null);
List <Task> cacheDocumentTasks = new List <Task>();
SemaphoreSlim cacheThrottle = new SemaphoreSlim(10, 10);
foreach (PublishedFundingIndex publishedFundingIndex in resultsPage.Entries)
{
await cacheThrottle.WaitAsync();
cacheDocumentTasks.Add(Task.Run(() => CachePublishedFundingDocument(publishedFundingIndex, cacheThrottle)));
}
await TaskHelper.WhenAllAndThrow(cacheDocumentTasks.ToArray());
}
finally
{
pageThrottle.Release();
}
}
public void GenerateAtomLinksForResultGivenBaseUrl_WhenOnAMiddlePage_ShouldReturnCorrectGeneratedValues()
{
// arrange
SearchFeedV3 <string> SearchFeedV3 = new SearchFeedV3 <string>
{
Top = 10,
TotalCount = 50,
PageRef = 3
};
// act
IList <AtomLink> atomLinksForFeed = SearchFeedV3.GenerateAtomLinksForResultGivenBaseUrl("https://localhost:5009/api/v2/allocations/notifications{0}");
// assert
atomLinksForFeed
.Count
.Should().Be(4);
var atomLink1 = atomLinksForFeed.First();
atomLink1
.Href
.Should().Be("https://localhost:5009/api/v2/allocations/notifications/2");
atomLink1
.Rel
.Should().Be("prev-archive");
var atomLink2 = atomLinksForFeed.ElementAt(1);
atomLink2
.Href
.Should().Be("https://localhost:5009/api/v2/allocations/notifications/4");
atomLink2
.Rel
.Should().Be("next-archive");
var atomLink3 = atomLinksForFeed.ElementAt(2);
atomLink3
.Href
.Should().Be("https://localhost:5009/api/v2/allocations/notifications/3");
atomLink3
.Rel
.Should().Be("current");
var atomLink4 = atomLinksForFeed.ElementAt(3);
atomLink4
.Href
.Should().Be("https://localhost:5009/api/v2/allocations/notifications");
atomLink4
.Rel
.Should().Be("self");
}
private async Task <AtomFeed <AtomEntry> > CreateAtomFeed(SearchFeedV3 <PublishedFundingIndex> searchFeed, HttpRequest request)
{
const string fundingEndpointName = "notifications";
string baseRequestPath = request.Path.Value.Substring(0, request.Path.Value.IndexOf(fundingEndpointName, StringComparison.Ordinal) + fundingEndpointName.Length);
string fundingTrimmedRequestPath = baseRequestPath.Replace(fundingEndpointName, string.Empty).TrimEnd('/');
string queryString = request.QueryString.Value;
string fundingUrl = $"{request.Scheme}://{request.Host.Value}{baseRequestPath}{{0}}{(!string.IsNullOrWhiteSpace(queryString) ? queryString : "")}";
AtomFeed <AtomEntry> atomFeed = CreateAtomFeedEntry(searchFeed, fundingUrl);
ConcurrentDictionary <string, object> feedContentResults = new ConcurrentDictionary <string, object>();
List <Task> allTasks = new List <Task>();
SemaphoreSlim throttler = new SemaphoreSlim(initialCount: _externalEngineOptions.BlobLookupConcurrencyCount);
foreach (PublishedFundingIndex feedIndex in searchFeed.Entries)
{
await throttler.WaitAsync();
allTasks.Add(
Task.Run(async() =>
{
try
{
//TODO; sort out the full document url as just the blob name is no good
string contents = await _publishedFundingRetrievalService.GetFundingFeedDocument(feedIndex.DocumentPath);
// Need to convert to an object, so JSON.NET can reserialise the contents, otherwise the string is escaped.
// Future TODO: change whole feed to output via text, instead of objects
object contentsObject = JsonConvert.DeserializeObject(contents);
feedContentResults.TryAdd(feedIndex.Id, contentsObject);
}
finally
{
throttler.Release();
}
}));
}
await TaskHelper.WhenAllAndThrow(allTasks.ToArray());
foreach (PublishedFundingIndex feedIndex in searchFeed.Entries)
{
AddAtomEntry(request, fundingTrimmedRequestPath, feedIndex, feedContentResults, atomFeed);
}
return(atomFeed);
}
private static SearchFeedV3 <PublishedFundingIndex> CreateSearchFeedResult(int pageRef,
int top,
int totalCount,
bool pageRefRequested,
IEnumerable <PublishedFundingIndex> searchResults)
{
PublishedFundingIndex[] fundingFeedResults = pageRefRequested ? searchResults.ToArray() : searchResults.Reverse().ToArray();
SearchFeedV3 <PublishedFundingIndex> searchFeedResult = new SearchFeedV3 <PublishedFundingIndex>
{
PageRef = pageRef,
Top = top,
TotalCount = totalCount,
Entries = fundingFeedResults
};
return(searchFeedResult);
}
private AtomFeed <AtomEntry> CreateAtomFeedEntry(SearchFeedV3 <PublishedFundingIndex> searchFeed, string fundingUrl)
{
AtomFeed <AtomEntry> atomFeed = new AtomFeed <AtomEntry>
{
Id = Guid.NewGuid().ToString("N"),
Title = "Calculate Funding Service Funding Feed",
Author = new CalculateFunding.Models.External.AtomItems.AtomAuthor
{
Name = "Calculate Funding Service",
Email = "*****@*****.**"
},
Updated = DateTimeOffset.Now,
Rights = "Copyright (C) 2019 Department for Education",
Link = searchFeed.GenerateAtomLinksForResultGivenBaseUrl(fundingUrl).ToList(),
AtomEntry = new List <AtomEntry>(),
IsArchived = searchFeed.IsArchivePage
};
return(atomFeed);
}
public async Task RetrievesRequestPageIfPageRefSupplied()
{
int totalCount = 257;
int top = 50;
int pageRef = 2;
IEnumerable <PublishedFundingIndex> sourceResults = NewResults(
NewPublishedFundingIndex(),
NewPublishedFundingIndex(),
NewPublishedFundingIndex());
GivenTheTotalCount(totalCount);
AndTheFundingFeedResults(top, pageRef, totalCount, sourceResults);
SearchFeedV3 <PublishedFundingIndex> fundingFeedResults = await WhenTheFeedIsRequested(pageRef, top);
fundingFeedResults
.Should()
.NotBeNull();
fundingFeedResults
.PageRef
.Should()
.Be(pageRef); //we only set this if they ask for page
fundingFeedResults
.Top
.Should()
.Be(top);
fundingFeedResults
.TotalCount
.Should()
.Be(totalCount);
fundingFeedResults
.Entries
.Should()
.BeEquivalentTo(sourceResults,
opt => opt.WithStrictOrdering());
}
public async Task GetNotifications_GivenSearchFeedReturnsResultsWhenNoQueryParameters_EnsuresAtomLinksCorrect()
{
//Arrange
SearchFeedV3 <PublishedFundingIndex> feeds = new SearchFeedV3 <PublishedFundingIndex>
{
PageRef = 2,
Top = 2,
TotalCount = 8,
Entries = CreateFeedIndexes()
};
Mock <IFundingFeedSearchService> feedsSearchService = new Mock <IFundingFeedSearchService>();
feedsSearchService.Setup(_ => _
.GetFeedsV3(It.IsAny <int?>(),
It.IsAny <int>(),
It.IsAny <IEnumerable <string> >(),
It.IsAny <IEnumerable <string> >(),
It.IsAny <IEnumerable <string> >(),
It.IsAny <IEnumerable <string> >()))
.ReturnsAsync(feeds);
Mock <IPublishedFundingRetrievalService> fundingRetrievalService = new Mock <IPublishedFundingRetrievalService>();
fundingRetrievalService.Setup(_ => _.GetFundingFeedDocument(It.IsAny <string>(), false))
.ReturnsAsync(string.Empty);
Mock <IExternalEngineOptions> externalEngineOptions = new Mock <IExternalEngineOptions>();
externalEngineOptions
.Setup(_ => _.BlobLookupConcurrencyCount)
.Returns(10);
FundingFeedService service = CreateService(feedsSearchService.Object,
fundingRetrievalService.Object, externalEngineOptions.Object);
IHeaderDictionary headerDictionary = new HeaderDictionary
{
{ "Accept", new StringValues("application/json") }
};
HttpRequest request = Substitute.For <HttpRequest>();
request.Scheme.Returns("https");
request.Path.Returns(new PathString("/api/v3/funding/notifications/2"));
request.Host.Returns(new HostString("wherever.naf:12345"));
request.Headers.Returns(headerDictionary);
//Act
IActionResult result = await service.GetFunding(request, pageSize : 2, pageRef : 2);
//Assert
result
.Should()
.BeOfType <OkObjectResult>();
OkObjectResult contentResult = result as OkObjectResult;
Models.External.V3.AtomItems.AtomFeed <AtomEntry> atomFeed = contentResult.Value as Models.External.V3.AtomItems.AtomFeed <AtomEntry>;
atomFeed
.Should()
.NotBeNull();
atomFeed.Id.Should().NotBeEmpty();
atomFeed.Title.Should().Be("Calculate Funding Service Funding Feed");
atomFeed.Author.Name.Should().Be("Calculate Funding Service");
atomFeed.Link.First(m => m.Rel == "prev-archive").Href.Should().Be("https://wherever.naf:12345/api/v3/funding/notifications/1");
atomFeed.Link.First(m => m.Rel == "next-archive").Href.Should().Be("https://wherever.naf:12345/api/v3/funding/notifications/3");
atomFeed.Link.First(m => m.Rel == "current").Href.Should().Be("https://wherever.naf:12345/api/v3/funding/notifications/2");
atomFeed.Link.First(m => m.Rel == "self").Href.Should().Be("https://wherever.naf:12345/api/v3/funding/notifications");
}
public async Task GetNotifications_GivenAQueryStringForWhichThereAreResults_ReturnsAtomFeedWithCorrectLinks()
{
//Arrange
string fundingFeedDocument = JsonConvert.SerializeObject(new
{
FundingStreamId = "PES"
});
int pageRef = 2;
int pageSize = 3;
List <PublishedFundingIndex> searchFeedEntries = CreateFeedIndexes().ToList();
SearchFeedV3 <PublishedFundingIndex> feeds = new SearchFeedV3 <PublishedFundingIndex>
{
PageRef = pageRef,
Top = 2,
TotalCount = 8,
Entries = searchFeedEntries
};
PublishedFundingIndex firstFeedItem = feeds.Entries.ElementAt(0);
IFundingFeedSearchService feedsSearchService = CreateSearchService();
feedsSearchService
.GetFeedsV3(Arg.Is(pageRef), Arg.Is(pageSize))
.ReturnsForAnyArgs(feeds);
IPublishedFundingRetrievalService publishedFundingRetrievalService = Substitute.For <IPublishedFundingRetrievalService>();
publishedFundingRetrievalService
.GetFundingFeedDocument(Arg.Any <string>())
.Returns(fundingFeedDocument);
Mock <IExternalEngineOptions> externalEngineOptions = new Mock <IExternalEngineOptions>();
externalEngineOptions
.Setup(_ => _.BlobLookupConcurrencyCount)
.Returns(10);
FundingFeedService service = CreateService(
searchService: feedsSearchService,
publishedFundingRetrievalService: publishedFundingRetrievalService,
externalEngineOptions.Object);
IHeaderDictionary headerDictionary = new HeaderDictionary
{
{ "Accept", new StringValues("application/json") }
};
IQueryCollection queryStringValues = new QueryCollection(new Dictionary <string, StringValues>
{
{ "pageRef", new StringValues(pageRef.ToString()) },
{ "allocationStatuses", new StringValues("Published,Approved") },
{ "pageSize", new StringValues(pageSize.ToString()) }
});
string scheme = "https";
string path = "/api/v3/fundings/notifications";
string host = "wherever.naf:12345";
string queryString = "?pageSize=2";
HttpRequest request = Substitute.For <HttpRequest>();
request.Scheme.Returns(scheme);
request.Path.Returns(new PathString(path));
request.Host.Returns(new HostString(host));
request.QueryString.Returns(new QueryString(queryString));
request.Headers.Returns(headerDictionary);
request.Query.Returns(queryStringValues);
//Act
IActionResult result = await service.GetFunding(request, pageRef : pageRef, pageSize : pageSize);
//Assert
result
.Should()
.BeOfType <OkObjectResult>();
OkObjectResult contentResult = result as OkObjectResult;
Models.External.V3.AtomItems.AtomFeed <AtomEntry> atomFeed = contentResult.Value as Models.External.V3.AtomItems.AtomFeed <AtomEntry>;
atomFeed
.Should()
.NotBeNull();
atomFeed.Id.Should().NotBeEmpty();
atomFeed.Title.Should().Be("Calculate Funding Service Funding Feed");
atomFeed.Author.Name.Should().Be("Calculate Funding Service");
atomFeed.Link.First(m => m.Rel == "next-archive").Href.Should().Be($"{scheme}://{host}{path}/3{queryString}");
atomFeed.Link.First(m => m.Rel == "prev-archive").Href.Should().Be($"{scheme}://{host}{path}/1{queryString}");
atomFeed.Link.First(m => m.Rel == "self").Href.Should().Be($"{scheme}://{host}{path}{queryString}");
atomFeed.Link.First(m => m.Rel == "current").Href.Should().Be($"{scheme}://{host}{path}/2{queryString}");
atomFeed.AtomEntry.Count.Should().Be(3);
for (int i = 0; i < 3; i++)
{
string text = $"id-{i + 1}";
atomFeed.AtomEntry.ElementAt(i).Id.Should().Be($"{scheme}://{host}/api/v3/fundings/byId/{text}");
atomFeed.AtomEntry.ElementAt(i).Title.Should().Be(text);
atomFeed.AtomEntry.ElementAt(i).Summary.Should().Be(text);
atomFeed.AtomEntry.ElementAt(i).Content.Should().NotBeNull();
}
JObject content = atomFeed.AtomEntry.ElementAt(0).Content as JObject;
content.TryGetValue("FundingStreamId", out JToken token);
((JValue)token).Value <string>().Should().Be("PES");
await feedsSearchService
.Received(1)
.GetFeedsV3(pageRef, pageSize, null, null, null);
await publishedFundingRetrievalService
.Received(searchFeedEntries.Count)
.GetFundingFeedDocument(Arg.Any <string>());
foreach (PublishedFundingIndex index in searchFeedEntries)
{
await publishedFundingRetrievalService
.Received(1)
.GetFundingFeedDocument(index.DocumentPath);
}
}
