public async Task <int> PopulateProviderSummariesForSpecification(string specificationId)
{
IEnumerable <ProviderSummary> allCachedProviders = Enumerable.Empty <ProviderSummary>();
string cacheKey = $"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}";
string providerCount = await _cacheProvider.GetAsync <string>(CacheKeys.AllProviderSummaryCount);
int allSummariesCount = 0;
if (!string.IsNullOrWhiteSpace(providerCount) && !int.TryParse(providerCount, out allSummariesCount))
{
throw new Exception("Invalid provider count in cache");
}
if (allSummariesCount > 0)
{
allCachedProviders = await _cacheProvider.ListRangeAsync <ProviderSummary>(CacheKeys.AllProviderSummaries, 0, allSummariesCount);
}
if (allSummariesCount < 1 || allCachedProviders.IsNullOrEmpty())
{
allCachedProviders = await LoadAllProvidersFromSearch();
allSummariesCount = allCachedProviders.Count();
}
if (allSummariesCount < 1 || allCachedProviders.IsNullOrEmpty())
{
throw new InvalidOperationException($"No provider summaries exist in cache or search");
}
IEnumerable <string> providerIds = await GetScopedProviderIds(specificationId);
int providerIdCount = providerIds.Count();
IList <ProviderSummary> providerSummaries = new List <ProviderSummary>(providerIdCount);
foreach (string providerId in providerIds)
{
ProviderSummary cachedProvider = allCachedProviders.FirstOrDefault(m => m.Id == providerId);
if (cachedProvider != null)
{
providerSummaries.Add(cachedProvider);
}
}
await _cacheProvider.KeyDeleteAsync <ProviderSummary>(cacheKey);
await _cacheProvider.CreateListAsync <ProviderSummary>(providerSummaries, cacheKey);
return(providerIdCount);
}
public async Task <IEnumerable <ProviderSummary> > FetchCoreProviderData()
{
int totalCount = await DetermineTotalCountFromSearch();
string currentProviderCount = await _cacheProvider.GetAsync <string>(CacheKeys.AllProviderSummaryCount);
long totalProviderListCount = await _cacheProvider.ListLengthAsync <ProviderSummary>(CacheKeys.AllProviderSummaries);
if (string.IsNullOrWhiteSpace(currentProviderCount) || int.Parse(currentProviderCount) != totalCount || totalProviderListCount != totalCount)
{
int pageCount = HowManyPagesForTotalCount(totalCount);
await _cacheProvider.KeyDeleteAsync <ProviderSummary>(CacheKeys.AllProviderSummaries);
Dictionary <string, ProviderSummary> providersFromSearch = new Dictionary <string, ProviderSummary>();
for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++)
{
IEnumerable <ProviderSummary> summaries = await FetchProviderSummariesFromSearch(pageNumber, MaxResultsCount);
foreach (ProviderSummary providerSummary in summaries)
{
if (providersFromSearch.ContainsKey(providerSummary.Id))
{
await _cacheProvider.KeyDeleteAsync <ProviderSummary>(CacheKeys.AllProviderSummaries);
throw new NonRetriableException($"Duplicate provider from search with ID '{providerSummary.Id}'");
}
else
{
providersFromSearch.Add(providerSummary.Id, providerSummary);
}
}
await _cacheProvider.CreateListAsync(summaries.ToList(), CacheKeys.AllProviderSummaries);
}
await _cacheProvider.SetAsync(CacheKeys.AllProviderSummaryCount, totalCount.ToString(), TimeSpan.FromDays(365), true);
return(providersFromSearch.Values);
}
else
{
return(await _cacheProvider.ListRangeAsync <ProviderSummary>(CacheKeys.AllProviderSummaries, 0, totalCount));
}
}
async Task PopulateProviderSummariesForSpecification(string specificationId, IEnumerable <ProviderSummary> allCachedProviders)
{
string cacheKey = $"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}";
IEnumerable <string> providerIdsAll = await _providerResultsRepositoryPolicy.ExecuteAsync(() =>
_resultsRepository.GetAllProviderIdsForSpecificationId(specificationId));
IList <ProviderSummary> providerSummaries = new List <ProviderSummary>();
foreach (string providerId in providerIdsAll)
{
ProviderSummary cachedProvider = allCachedProviders.FirstOrDefault(m => m.Id == providerId);
if (cachedProvider != null)
{
providerSummaries.Add(cachedProvider);
}
}
await _cacheProvider.KeyDeleteAsync <ProviderSummary>(cacheKey);
await _cacheProvider.CreateListAsync <ProviderSummary>(providerSummaries, cacheKey);
}
public override async Task Process(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
string specificationId = message.GetUserProperty <string>(SpecificationId);
string scopedProviderSummariesCountCacheKey = $"{CacheKeys.ScopedProviderSummariesCount}{specificationId}";
string cacheKeyScopedListCacheKey = $"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}";
SpecificationSummary specificationSummary = await GetSpecificationSummary(specificationId);
string providerVersionId = specificationSummary?.ProviderVersionId;
if (string.IsNullOrWhiteSpace(providerVersionId))
{
return;
}
ProviderVersion providerVersion = await _providerVersionService.GetProvidersByVersion(providerVersionId);
if (providerVersion == null)
{
return;
}
IEnumerable <Provider> sourceProviders = providerVersion.Providers;
if (specificationSummary.ProviderSource == ProviderSource.CFS)
{
HashSet <string> scopedProviderIds = (await GetScopedProviderIdsBySpecification(specificationId)).ToHashSet();
sourceProviders = sourceProviders.Where(_ => scopedProviderIds.Contains(_.ProviderId));
}
IEnumerable <ProviderSummary> providerSummaries = sourceProviders
.Select(x => new ProviderSummary
{
Name = x.Name,
Id = x.ProviderId,
ProviderProfileIdType = x.ProviderProfileIdType,
UKPRN = x.UKPRN,
URN = x.URN,
Authority = x.Authority,
UPIN = x.UPIN,
ProviderSubType = x.ProviderSubType,
EstablishmentNumber = x.EstablishmentNumber,
ProviderType = x.ProviderType,
DateOpened = x.DateOpened,
DateClosed = x.DateClosed,
LACode = x.LACode,
CrmAccountId = x.CrmAccountId,
LegalName = x.LegalName,
NavVendorNo = x.NavVendorNo,
DfeEstablishmentNumber = x.DfeEstablishmentNumber,
Status = x.Status,
PhaseOfEducation = x.PhaseOfEducation,
ReasonEstablishmentClosed = x.ReasonEstablishmentClosed,
ReasonEstablishmentOpened = x.ReasonEstablishmentOpened,
Successor = x.Successor,
TrustStatus = x.TrustStatus,
TrustName = x.TrustName,
TrustCode = x.TrustCode,
Town = x.Town,
Postcode = x.Postcode,
CompaniesHouseNumber = x.CompaniesHouseNumber,
GroupIdNumber = x.GroupIdNumber,
RscRegionName = x.RscRegionName,
RscRegionCode = x.RscRegionCode,
GovernmentOfficeRegionName = x.GovernmentOfficeRegionName,
GovernmentOfficeRegionCode = x.GovernmentOfficeRegionCode,
DistrictCode = x.DistrictCode,
DistrictName = x.DistrictName,
WardName = x.WardName,
WardCode = x.WardCode,
CensusWardCode = x.CensusWardCode,
CensusWardName = x.CensusWardName,
MiddleSuperOutputAreaCode = x.MiddleSuperOutputAreaCode,
MiddleSuperOutputAreaName = x.MiddleSuperOutputAreaName,
LowerSuperOutputAreaCode = x.LowerSuperOutputAreaCode,
LowerSuperOutputAreaName = x.LowerSuperOutputAreaName,
ParliamentaryConstituencyCode = x.ParliamentaryConstituencyCode,
ParliamentaryConstituencyName = x.ParliamentaryConstituencyName,
CountryCode = x.CountryCode,
CountryName = x.CountryName,
LocalGovernmentGroupTypeCode = x.LocalGovernmentGroupTypeCode,
LocalGovernmentGroupTypeName = x.LocalGovernmentGroupTypeName,
Street = x.Street,
Locality = x.Locality,
Address3 = x.Address3,
PaymentOrganisationIdentifier = x.PaymentOrganisationIdentifier,
PaymentOrganisationName = x.PaymentOrganisationName
});
await _cachePolicy.ExecuteAsync(() => _cacheProvider.KeyDeleteAsync <ProviderSummary>(cacheKeyScopedListCacheKey));
// Batch to get around redis timeouts
foreach (IEnumerable <ProviderSummary> batch in providerSummaries.ToBatches(1000))
{
// Create list is an upsert into the redis list
await _cachePolicy.ExecuteAsync(() => _cacheProvider.CreateListAsync(batch, cacheKeyScopedListCacheKey));
await _cachePolicy.ExecuteAsync(() => _cacheProvider.SetExpiry <ProviderSummary>(cacheKeyScopedListCacheKey, DateTime.UtcNow.AddDays(7)));
}
await _cachePolicy.ExecuteAsync(() =>
_cacheProvider.SetAsync(scopedProviderSummariesCountCacheKey,
providerSummaries.Count().ToString(), TimeSpan.FromDays(7), true));
string filesystemCacheKey = $"{CacheKeys.ScopedProviderSummariesFilesystemKeyPrefix}{specificationId}";
await _cachePolicy.ExecuteAsync(() => _cacheProvider.KeyDeleteAsync <string>(filesystemCacheKey));
}
