public void Id_PopulatedCorrectly(string specificationId, string providerId, string Id)
{
ProviderCalculationResultsIndex resultsIndex = new ProviderCalculationResultsIndex
{
SpecificationId = specificationId,
ProviderId = providerId
};
Assert.AreEqual(Id, resultsIndex.Id);
}
public async Task ShouldWriteSearchIndexForAllProviderResults_WhenProcessTheMessage_GivenProviderResultsForSepecificationAndProviderIds()
{
// Arrange
string specificationId = new RandomString();
string specificationName = new RandomString();
string providerId1 = new RandomString();
string providerId2 = new RandomString();
string providerResultId1 = CreateProviderResultId(specificationId, providerId1);
string providerResultId2 = CreateProviderResultId(specificationId, providerId2);
ProviderResultDataKey providerResultDataKey1 = new ProviderResultDataKey(providerResultId1, providerId1);
ProviderResultDataKey providerResultDataKey2 = new ProviderResultDataKey(providerResultId2, providerId2);
ProviderResult providerResult1 = new ProviderResult()
{
Id = providerResultId1
};
ProviderResult providerResult2 = new ProviderResult()
{
Id = providerResultId2
};
ProviderCalculationResultsIndex index1 = new ProviderCalculationResultsIndex()
{
SpecificationId = specificationId, ProviderId = providerId1
};
ProviderCalculationResultsIndex index2 = new ProviderCalculationResultsIndex()
{
SpecificationId = specificationId, ProviderId = providerId2
};
_reader.GetData(Arg.Is <ProviderResultDataKey>(_ => _.ProviderResultId == providerResultId1 && _.PartitionKey == providerId1))
.Returns(providerResult1);
_reader.GetData(Arg.Is <ProviderResultDataKey>(_ => _.ProviderResultId == providerResultId2 && _.PartitionKey == providerId2))
.Returns(providerResult2);
_transformer.Transform(Arg.Is <ProviderResult>(_ => _.Id == providerResultId1), Arg.Is <ISearchIndexProcessorContext>(_ => _.GetType() == typeof(ProviderCalculationResultsIndexProcessorContext)))
.Returns(index1);
_transformer.Transform(Arg.Is <ProviderResult>(_ => _.Id == providerResultId2), Arg.Is <ISearchIndexProcessorContext>(_ => _.GetType() == typeof(ProviderCalculationResultsIndexProcessorContext)))
.Returns(index2);
_searchRepository.Index(Arg.Is <IEnumerable <ProviderCalculationResultsIndex> >(_ => _.Any(x => x.ProviderId == providerId1 || x.ProviderId == providerId2)))
.Returns(Enumerable.Empty <IndexError>());
Message message = new Message();
message.UserProperties.Add("specification-id", specificationId);
message.UserProperties.Add("specification-name", specificationName);
message.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new[] { providerId1, providerId2 }));
// Act
await _processor.Process(message);
// Assert
await _searchRepository
.Received(2)
.Index(Arg.Is <IEnumerable <ProviderCalculationResultsIndex> >(_ => _.Any(x => x.ProviderId == providerId1 || x.ProviderId == providerId2)));
await _reader
.Received(2)
.GetData(Arg.Is <ProviderResultDataKey>(_ => (_.ProviderResultId == providerResultId1 && _.PartitionKey == providerId1) ||
(_.ProviderResultId == providerResultId2 && _.PartitionKey == providerId2)));
await _transformer
.Received(2)
.Transform(Arg.Is <ProviderResult>(_ => _.Id == providerResultId1 || _.Id == providerResultId2),
Arg.Is <ISearchIndexProcessorContext>(_ => _.GetType() == typeof(ProviderCalculationResultsIndexProcessorContext)));
}
public async Task ShouldPartiallySuccessful_WhenProcessTheMessage_GivenOneOfTheIndexProcessingRaiseAnException()
{
// Arrange
string specificationId = new RandomString();
string specificationName = new RandomString();
string providerId1 = new RandomString();
string providerId2 = new RandomString();
string providerResultId1 = CreateProviderResultId(specificationId, providerId1);
string providerResultId2 = CreateProviderResultId(specificationId, providerId2);
ProviderResultDataKey providerResultDataKey1 = new ProviderResultDataKey(providerResultId1, providerId1);
ProviderResultDataKey providerResultDataKey2 = new ProviderResultDataKey(providerResultId2, providerId2);
ProviderResult providerResult1 = new ProviderResult()
{
Id = providerResultId1
};
ProviderResult providerResult2 = new ProviderResult()
{
Id = providerResultId2
};
ProviderCalculationResultsIndex index1 = new ProviderCalculationResultsIndex()
{
SpecificationId = specificationId, ProviderId = providerId1
};
ProviderCalculationResultsIndex index2 = new ProviderCalculationResultsIndex()
{
SpecificationId = specificationId, ProviderId = providerId2
};
_reader.GetData(Arg.Is <ProviderResultDataKey>(_ => _.ProviderResultId == providerResultId1 && _.PartitionKey == providerId1))
.Returns(Task.FromException <ProviderResult>(new Exception("Reader Failed")));
_reader.GetData(Arg.Is <ProviderResultDataKey>(_ => _.ProviderResultId == providerResultId2 && _.PartitionKey == providerId2))
.Returns(providerResult2);
_transformer.Transform(Arg.Is <ProviderResult>(_ => _.Id == providerResultId2), Arg.Is <ISearchIndexProcessorContext>(_ => _.GetType() == typeof(ProviderCalculationResultsIndexProcessorContext)))
.Returns(index2);
_searchRepository.Index(Arg.Is <IEnumerable <ProviderCalculationResultsIndex> >(_ => _.Any(x => x.ProviderId == providerId1 || x.ProviderId == providerId2)))
.Returns(Enumerable.Empty <IndexError>());
Message message = new Message();
message.UserProperties.Add("specification-id", specificationId);
message.UserProperties.Add("specification-name", specificationName);
message.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new[] { providerId1, providerId2 }));
// Act
Func <Task> test = async() => await _processor.Process(message);
// Assert
test
.Should()
.ThrowExactly <Exception>()
.Which
.Message
.Should()
.Be("Error occurred while processing the ProviderCalculationResultsIndex. Reader Failed");
await _searchRepository
.Received(1)
.Index(Arg.Is <IEnumerable <ProviderCalculationResultsIndex> >(_ => _.Any(x => x.ProviderId == providerId1 || x.ProviderId == providerId2)));
await _reader
.Received(2)
.GetData(Arg.Is <ProviderResultDataKey>(_ => (_.ProviderResultId == providerResultId1 && _.PartitionKey == providerId1) ||
(_.ProviderResultId == providerResultId2 && _.PartitionKey == providerId2)));
await _transformer
.Received(1)
.Transform(Arg.Is <ProviderResult>(_ => _.Id == providerResultId1 || _.Id == providerResultId2),
Arg.Is <ISearchIndexProcessorContext>(_ => _.GetType() == typeof(ProviderCalculationResultsIndexProcessorContext)));
}
public override async Task Process(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
Reference user = message.GetUserDetails();
_logger.Information($"{nameof(ReIndexCalculationResults)} initiated by: '{user.Name}'");
ApiResponse <IEnumerable <SpecModel.SpecificationSummary> > specificationsApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaries());
if (specificationsApiResponse == null || !specificationsApiResponse.StatusCode.IsSuccess() || specificationsApiResponse.Content == null)
{
return;
}
IEnumerable <SpecModel.SpecificationSummary> specifications = specificationsApiResponse.Content;
foreach (SpecModel.SpecificationSummary specification in specifications)
{
await _resultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.ProviderResultsBatchProcessing(specification.Id, async(x) =>
{
IList <ProviderCalculationResultsIndex> results = new List <ProviderCalculationResultsIndex>();
foreach (ProviderResult providerResult in x)
{
if (!providerResult.CalculationResults.IsNullOrEmpty())
{
ProviderCalculationResultsIndex calculationResult = new ProviderCalculationResultsIndex
{
SpecificationId = providerResult.SpecificationId,
SpecificationName = specification?.Name,
ProviderId = providerResult.Provider?.Id,
ProviderName = providerResult.Provider?.Name,
ProviderType = providerResult.Provider?.ProviderType,
ProviderSubType = providerResult.Provider?.ProviderSubType,
LocalAuthority = providerResult.Provider?.Authority,
LastUpdatedDate = providerResult.CreatedAt,
UKPRN = providerResult.Provider?.UKPRN,
URN = providerResult.Provider?.URN,
UPIN = providerResult.Provider?.UPIN,
EstablishmentNumber = providerResult.Provider?.EstablishmentNumber,
OpenDate = providerResult.Provider?.DateOpened,
CalculationId = providerResult.CalculationResults.Select(m => m.Calculation.Id).ToArraySafe(),
CalculationName = providerResult.CalculationResults.Select(m => m.Calculation.Name).ToArraySafe(),
CalculationResult = providerResult.CalculationResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe()
};
if (providerResult.FundingLineResults != null)
{
calculationResult.FundingLineName = providerResult.FundingLineResults?.Select(m => m.FundingLine.Name).ToArraySafe();
calculationResult.FundingLineFundingStreamId = providerResult.FundingLineResults?.Select(m => m.FundingLineFundingStreamId).ToArraySafe();
calculationResult.FundingLineId = providerResult.FundingLineResults?.Select(m => m.FundingLine.Id).ToArraySafe();
calculationResult.FundingLineResult = providerResult.FundingLineResults?.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe();
}
if (_featureToggle.IsExceptionMessagesEnabled())
{
calculationResult.CalculationException = providerResult.CalculationResults
.Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType))
.Select(e => e.Calculation.Id)
.ToArraySafe();
calculationResult.CalculationExceptionType = providerResult.CalculationResults
.Select(m => m.ExceptionType ?? string.Empty)
.ToArraySafe();
calculationResult.CalculationExceptionMessage = providerResult.CalculationResults
.Select(m => m.ExceptionMessage ?? string.Empty)
.ToArraySafe();
if (providerResult.FundingLineResults != null)
{
calculationResult.FundingLineException = providerResult.FundingLineResults
.Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType))
.Select(e => e.FundingLine.Id)
.ToArraySafe();
calculationResult.FundingLineExceptionType = providerResult.FundingLineResults
.Select(m => m.ExceptionType ?? string.Empty)
.ToArraySafe();
calculationResult.FundingLineExceptionMessage = providerResult.FundingLineResults
.Select(m => m.ExceptionMessage ?? string.Empty)
.ToArraySafe();
}
}
results.Add(calculationResult);
}
}
IEnumerable <IndexError> errors = await _resultsSearchRepositoryPolicy.ExecuteAsync(() => _providerCalculationResultsSearchRepository.Index(results));
if (errors.Any())
{
string errorMessage = $"Failed to index calculation provider result documents with errors: { string.Join(";", errors.Select(m => m.ErrorMessage)) }";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
}));
}
}
private async Task <long> UpdateCalculationProviderResultsIndex(IEnumerable <ProviderResult> providerResults, IDictionary <string, SpecificationSummary> specifications)
{
Stopwatch assembleStopwatch = Stopwatch.StartNew();
List <ProviderCalculationResultsIndex> results = new List <ProviderCalculationResultsIndex>();
foreach (ProviderResult providerResult in providerResults)
{
if (!providerResult.CalculationResults.IsNullOrEmpty())
{
SpecificationSummary specification = specifications[providerResult.SpecificationId];
ProviderCalculationResultsIndex providerCalculationResultsIndex = new ProviderCalculationResultsIndex
{
SpecificationId = providerResult.SpecificationId,
SpecificationName = specification?.Name,
ProviderId = providerResult.Provider?.Id,
ProviderName = providerResult.Provider?.Name,
ProviderType = providerResult.Provider?.ProviderType,
ProviderSubType = providerResult.Provider?.ProviderSubType,
LocalAuthority = providerResult.Provider?.Authority,
LastUpdatedDate = DateTimeOffset.Now,
UKPRN = providerResult.Provider?.UKPRN,
URN = providerResult.Provider?.URN,
UPIN = providerResult.Provider?.UPIN,
EstablishmentNumber = providerResult.Provider?.EstablishmentNumber,
OpenDate = providerResult.Provider?.DateOpened,
CalculationId = providerResult.CalculationResults.Select(m => m.Calculation.Id).ToArraySafe(),
CalculationName = providerResult.CalculationResults.Select(m => m.Calculation.Name).ToArraySafe(),
CalculationResult = providerResult.CalculationResults.Select(m => m.Value.HasValue ? m.Value.ToString() : "null").ToArraySafe()
};
if (_featureToggle.IsExceptionMessagesEnabled())
{
providerCalculationResultsIndex.CalculationException = providerResult.CalculationResults
.Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType))
.Select(e => e.Calculation.Id)
.ToArraySafe();
providerCalculationResultsIndex.CalculationExceptionType = providerResult.CalculationResults
.Select(m => m.ExceptionType ?? string.Empty)
.ToArraySafe();
providerCalculationResultsIndex.CalculationExceptionMessage = providerResult.CalculationResults
.Select(m => m.ExceptionMessage ?? string.Empty)
.ToArraySafe();
}
results.Add(providerCalculationResultsIndex);
}
}
assembleStopwatch.Stop();
Stopwatch stopwatch = Stopwatch.StartNew();
foreach (IEnumerable <ProviderCalculationResultsIndex> resultsBatch in results.ToBatches(_engineSettings.CalculationResultSearchIndexBatchSize))
{
IEnumerable <IndexError> indexErrors = await _providerCalculationResultsSearchRepository.Index(resultsBatch);
if (!indexErrors.IsNullOrEmpty())
{
_logger.Error($"Failed to index provider results with the following errors: {string.Join(";", indexErrors.Select(m => m.ErrorMessage))}");
// Throw exception so Service Bus message can be requeued and calc results can have a chance to get saved again
throw new FailedToIndexSearchException(indexErrors);
}
}
stopwatch.Stop();
return(stopwatch.ElapsedMilliseconds);
}
public async Task <IActionResult> ReIndexCalculationProviderResults()
{
IEnumerable <DocumentEntity <ProviderResult> > providerResults = await _resultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.GetAllProviderResults());
IList <ProviderCalculationResultsIndex> searchItems = new List <ProviderCalculationResultsIndex>();
Dictionary <string, SpecModel.SpecificationSummary> specifications = new Dictionary <string, SpecModel.SpecificationSummary>();
foreach (DocumentEntity <ProviderResult> documentEntity in providerResults)
{
ProviderResult providerResult = documentEntity.Content;
foreach (CalculationResult calculationResult in providerResult.CalculationResults)
{
SpecModel.SpecificationSummary specificationSummary = null;
if (!specifications.ContainsKey(providerResult.SpecificationId))
{
Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(providerResult.SpecificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
throw new InvalidOperationException($"Specification Summary returned null for specification ID '{providerResult.SpecificationId}'");
}
specificationSummary = specificationApiResponse.Content;
specifications.Add(providerResult.SpecificationId, specificationSummary);
}
else
{
specificationSummary = specifications[providerResult.SpecificationId];
}
ProviderCalculationResultsIndex searchItem = new ProviderCalculationResultsIndex
{
SpecificationId = providerResult.SpecificationId,
SpecificationName = specificationSummary?.Name,
CalculationName = providerResult.CalculationResults.Select(x => x.Calculation.Name).ToArraySafe(),
CalculationId = providerResult.CalculationResults.Select(x => x.Calculation.Id).ToArraySafe(),
ProviderId = providerResult.Provider.Id,
ProviderName = providerResult.Provider.Name,
ProviderType = providerResult.Provider.ProviderType,
ProviderSubType = providerResult.Provider.ProviderSubType,
LocalAuthority = providerResult.Provider.Authority,
LastUpdatedDate = documentEntity.UpdatedAt,
UKPRN = providerResult.Provider.UKPRN,
URN = providerResult.Provider.URN,
UPIN = providerResult.Provider.UPIN,
EstablishmentNumber = providerResult.Provider.EstablishmentNumber,
OpenDate = providerResult.Provider.DateOpened,
CalculationResult = providerResult.CalculationResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe()
};
if (_featureToggle.IsExceptionMessagesEnabled())
{
searchItem.CalculationException = providerResult.CalculationResults
.Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType))
.Select(e => e.Calculation.Id)
.ToArraySafe();
searchItem.CalculationExceptionType = providerResult.CalculationResults
.Select(m => m.ExceptionType ?? string.Empty)
.ToArraySafe();
searchItem.CalculationExceptionMessage = providerResult.CalculationResults
.Select(m => m.ExceptionMessage ?? string.Empty)
.ToArraySafe();
}
searchItems.Add(searchItem);
}
}
const int partitionSize = 500;
for (int i = 0; i < searchItems.Count; i += partitionSize)
{
IEnumerable <ProviderCalculationResultsIndex> partitionedResults = searchItems
.Skip(i)
.Take(partitionSize);
IEnumerable <IndexError> errors = await _resultsSearchRepositoryPolicy.ExecuteAsync(() => _calculationProviderResultsSearchRepository.Index(partitionedResults));
if (errors.Any())
{
_logger.Error($"Failed to index calculation provider result documents with errors: { string.Join(";", errors.Select(m => m.ErrorMessage)) }");
return(new InternalServerErrorResult(null));
}
}
return(new NoContentResult());
}
