public async Task <int> PopulateProviderSummariesForSpecification(string specificationId) { IEnumerable <ProviderSummary> allCachedProviders = Enumerable.Empty <ProviderSummary>(); string cacheKey = $"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}"; string providerCount = await _cacheProvider.GetAsync <string>(CacheKeys.AllProviderSummaryCount); int allSummariesCount = 0; if (!string.IsNullOrWhiteSpace(providerCount) && !int.TryParse(providerCount, out allSummariesCount)) { throw new Exception("Invalid provider count in cache"); } if (allSummariesCount > 0) { allCachedProviders = await _cacheProvider.ListRangeAsync <ProviderSummary>(CacheKeys.AllProviderSummaries, 0, allSummariesCount); } if (allSummariesCount < 1 || allCachedProviders.IsNullOrEmpty()) { allCachedProviders = await LoadAllProvidersFromSearch(); allSummariesCount = allCachedProviders.Count(); } if (allSummariesCount < 1 || allCachedProviders.IsNullOrEmpty()) { throw new InvalidOperationException($"No provider summaries exist in cache or search"); } IEnumerable <string> providerIds = await GetScopedProviderIds(specificationId); int providerIdCount = providerIds.Count(); IList <ProviderSummary> providerSummaries = new List <ProviderSummary>(providerIdCount); foreach (string providerId in providerIds) { ProviderSummary cachedProvider = allCachedProviders.FirstOrDefault(m => m.Id == providerId); if (cachedProvider != null) { providerSummaries.Add(cachedProvider); } } await _cacheProvider.KeyDeleteAsync <ProviderSummary>(cacheKey); await _cacheProvider.CreateListAsync <ProviderSummary>(providerSummaries, cacheKey); return(providerIdCount); }
public async Task <IEnumerable <ProviderSummary> > FetchCoreProviderData() { int totalCount = await DetermineTotalCountFromSearch(); string currentProviderCount = await _cacheProvider.GetAsync <string>(CacheKeys.AllProviderSummaryCount); long totalProviderListCount = await _cacheProvider.ListLengthAsync <ProviderSummary>(CacheKeys.AllProviderSummaries); if (string.IsNullOrWhiteSpace(currentProviderCount) || int.Parse(currentProviderCount) != totalCount || totalProviderListCount != totalCount) { int pageCount = HowManyPagesForTotalCount(totalCount); await _cacheProvider.KeyDeleteAsync <ProviderSummary>(CacheKeys.AllProviderSummaries); Dictionary <string, ProviderSummary> providersFromSearch = new Dictionary <string, ProviderSummary>(); for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) { IEnumerable <ProviderSummary> summaries = await FetchProviderSummariesFromSearch(pageNumber, MaxResultsCount); foreach (ProviderSummary providerSummary in summaries) { if (providersFromSearch.ContainsKey(providerSummary.Id)) { await _cacheProvider.KeyDeleteAsync <ProviderSummary>(CacheKeys.AllProviderSummaries); throw new NonRetriableException($"Duplicate provider from search with ID '{providerSummary.Id}'"); } else { providersFromSearch.Add(providerSummary.Id, providerSummary); } } await _cacheProvider.CreateListAsync(summaries.ToList(), CacheKeys.AllProviderSummaries); } await _cacheProvider.SetAsync(CacheKeys.AllProviderSummaryCount, totalCount.ToString(), TimeSpan.FromDays(365), true); return(providersFromSearch.Values); } else { return(await _cacheProvider.ListRangeAsync <ProviderSummary>(CacheKeys.AllProviderSummaries, 0, totalCount)); } }
async Task PopulateProviderSummariesForSpecification(string specificationId, IEnumerable <ProviderSummary> allCachedProviders) { string cacheKey = $"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}"; IEnumerable <string> providerIdsAll = await _providerResultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.GetAllProviderIdsForSpecificationId(specificationId)); IList <ProviderSummary> providerSummaries = new List <ProviderSummary>(); foreach (string providerId in providerIdsAll) { ProviderSummary cachedProvider = allCachedProviders.FirstOrDefault(m => m.Id == providerId); if (cachedProvider != null) { providerSummaries.Add(cachedProvider); } } await _cacheProvider.KeyDeleteAsync <ProviderSummary>(cacheKey); await _cacheProvider.CreateListAsync <ProviderSummary>(providerSummaries, cacheKey); }
public override async Task Process(Message message) { Guard.ArgumentNotNull(message, nameof(message)); string specificationId = message.GetUserProperty <string>(SpecificationId); string scopedProviderSummariesCountCacheKey = $"{CacheKeys.ScopedProviderSummariesCount}{specificationId}"; string cacheKeyScopedListCacheKey = $"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}"; SpecificationSummary specificationSummary = await GetSpecificationSummary(specificationId); string providerVersionId = specificationSummary?.ProviderVersionId; if (string.IsNullOrWhiteSpace(providerVersionId)) { return; } ProviderVersion providerVersion = await _providerVersionService.GetProvidersByVersion(providerVersionId); if (providerVersion == null) { return; } IEnumerable <Provider> sourceProviders = providerVersion.Providers; if (specificationSummary.ProviderSource == ProviderSource.CFS) { HashSet <string> scopedProviderIds = (await GetScopedProviderIdsBySpecification(specificationId)).ToHashSet(); sourceProviders = sourceProviders.Where(_ => scopedProviderIds.Contains(_.ProviderId)); } IEnumerable <ProviderSummary> providerSummaries = sourceProviders .Select(x => new ProviderSummary { Name = x.Name, Id = x.ProviderId, ProviderProfileIdType = x.ProviderProfileIdType, UKPRN = x.UKPRN, URN = x.URN, Authority = x.Authority, UPIN = x.UPIN, ProviderSubType = x.ProviderSubType, EstablishmentNumber = x.EstablishmentNumber, ProviderType = x.ProviderType, DateOpened = x.DateOpened, DateClosed = x.DateClosed, LACode = x.LACode, CrmAccountId = x.CrmAccountId, LegalName = x.LegalName, NavVendorNo = x.NavVendorNo, DfeEstablishmentNumber = x.DfeEstablishmentNumber, Status = x.Status, PhaseOfEducation = x.PhaseOfEducation, ReasonEstablishmentClosed = x.ReasonEstablishmentClosed, ReasonEstablishmentOpened = x.ReasonEstablishmentOpened, Successor = x.Successor, TrustStatus = x.TrustStatus, TrustName = x.TrustName, TrustCode = x.TrustCode, Town = x.Town, Postcode = x.Postcode, CompaniesHouseNumber = x.CompaniesHouseNumber, GroupIdNumber = x.GroupIdNumber, RscRegionName = x.RscRegionName, RscRegionCode = x.RscRegionCode, GovernmentOfficeRegionName = x.GovernmentOfficeRegionName, GovernmentOfficeRegionCode = x.GovernmentOfficeRegionCode, DistrictCode = x.DistrictCode, DistrictName = x.DistrictName, WardName = x.WardName, WardCode = x.WardCode, CensusWardCode = x.CensusWardCode, CensusWardName = x.CensusWardName, MiddleSuperOutputAreaCode = x.MiddleSuperOutputAreaCode, MiddleSuperOutputAreaName = x.MiddleSuperOutputAreaName, LowerSuperOutputAreaCode = x.LowerSuperOutputAreaCode, LowerSuperOutputAreaName = x.LowerSuperOutputAreaName, ParliamentaryConstituencyCode = x.ParliamentaryConstituencyCode, ParliamentaryConstituencyName = x.ParliamentaryConstituencyName, CountryCode = x.CountryCode, CountryName = x.CountryName, LocalGovernmentGroupTypeCode = x.LocalGovernmentGroupTypeCode, LocalGovernmentGroupTypeName = x.LocalGovernmentGroupTypeName, Street = x.Street, Locality = x.Locality, Address3 = x.Address3, PaymentOrganisationIdentifier = x.PaymentOrganisationIdentifier, PaymentOrganisationName = x.PaymentOrganisationName }); await _cachePolicy.ExecuteAsync(() => _cacheProvider.KeyDeleteAsync <ProviderSummary>(cacheKeyScopedListCacheKey)); // Batch to get around redis timeouts foreach (IEnumerable <ProviderSummary> batch in providerSummaries.ToBatches(1000)) { // Create list is an upsert into the redis list await _cachePolicy.ExecuteAsync(() => _cacheProvider.CreateListAsync(batch, cacheKeyScopedListCacheKey)); await _cachePolicy.ExecuteAsync(() => _cacheProvider.SetExpiry <ProviderSummary>(cacheKeyScopedListCacheKey, DateTime.UtcNow.AddDays(7))); } await _cachePolicy.ExecuteAsync(() => _cacheProvider.SetAsync(scopedProviderSummariesCountCacheKey, providerSummaries.Count().ToString(), TimeSpan.FromDays(7), true)); string filesystemCacheKey = $"{CacheKeys.ScopedProviderSummariesFilesystemKeyPrefix}{specificationId}"; await _cachePolicy.ExecuteAsync(() => _cacheProvider.KeyDeleteAsync <string>(filesystemCacheKey)); }