protected override IEnumerable <ProviderResultDataKey> IndexDataItemKeys(ISearchIndexProcessorContext context)
{
var processorContext = context as ProviderCalculationResultsIndexProcessorContext;
Guard.ArgumentNotNull(processorContext, nameof(ProviderCalculationResultsIndexProcessorContext));
foreach (string providerId in processorContext.ProviderIds)
{
byte[] providerResultIdBytes = Encoding.UTF8.GetBytes($"{providerId}-{processorContext.SpecificationId}");
yield return(new ProviderResultDataKey(Convert.ToBase64String(providerResultIdBytes), providerId));
}
}
public async Task Process(Message message)
{
List <string> exceptionMessages = new List <string>();
ISearchIndexProcessorContext context = CreateContext(message);
SemaphoreSlim throttler = new SemaphoreSlim(context.DegreeOfParallelism, context.DegreeOfParallelism);
List <Task> indexTasks = new List <Task>();
foreach (TKey key in IndexDataItemKeys(context))
{
await throttler.WaitAsync();
indexTasks.Add
(
Task.Run(async() =>
{
try
{
TInput indexData = await _reader.GetData(key);
if (indexData != null)
{
TOutput indexDocument = await _transformer.Transform(indexData, context);
IEnumerable <IndexError> results = await _searchRepository.Index(new[] { indexDocument });
if (!results.IsNullOrEmpty())
{
IndexError indexError = results.First(); // Only indexing one document
exceptionMessages.Add($"{indexError.Key}:{indexError.ErrorMessage}");
}
}
else
{
string message = $"No data found for given Key - {key.ToString()}";
_logger.Error(message);
exceptionMessages.Add(message);
}
}
catch (Exception ex)
{
if (ex is AggregateException)
{
ex = ((AggregateException)ex).Flatten().InnerExceptions.FirstOrDefault() ?? ex;
}
_logger.Error(ex, $"Error occurred while processing the {IndexName} - {ex.Message}");
exceptionMessages.Add(ex.Message);
}
finally
{
throttler.Release();
}
})
);
}
await TaskHelper.WhenAllAndThrow(indexTasks.ToArray());
if (exceptionMessages.Any())
{
throw new Exception($"Error occurred while processing the {IndexName}. {string.Join(";", exceptionMessages)}");
}
}
protected abstract IEnumerable <TKey> IndexDataItemKeys(ISearchIndexProcessorContext context);
public Task <ProviderCalculationResultsIndex> Transform(ProviderResult providerResult, ISearchIndexProcessorContext context)
{
Guard.ArgumentNotNull(providerResult, nameof(ProviderResult));
var processorContext = context as ProviderCalculationResultsIndexProcessorContext;
Guard.ArgumentNotNull(processorContext, nameof(ProviderCalculationResultsIndexProcessorContext));
ProviderCalculationResultsIndex providerCalculationResultsIndex = new ProviderCalculationResultsIndex
{
SpecificationId = providerResult.SpecificationId,
SpecificationName = processorContext.SpecificationName,
ProviderId = providerResult.Provider?.Id,
ProviderName = providerResult.Provider?.Name,
ProviderType = providerResult.Provider?.ProviderType,
ProviderSubType = providerResult.Provider?.ProviderSubType,
LocalAuthority = providerResult.Provider?.Authority,
LastUpdatedDate = DateTimeOffset.Now,
UKPRN = providerResult.Provider?.UKPRN,
URN = providerResult.Provider?.URN,
UPIN = providerResult.Provider?.UPIN,
EstablishmentNumber = providerResult.Provider?.EstablishmentNumber,
OpenDate = providerResult.Provider?.DateOpened,
CalculationId = providerResult.CalculationResults.Select(m => m.Calculation.Id).ToArraySafe(),
CalculationName = providerResult.CalculationResults.Select(m => m.Calculation.Name).ToArraySafe(),
CalculationResult = providerResult.CalculationResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe(),
};
if (providerResult.FundingLineResults != null)
{
providerCalculationResultsIndex.FundingLineName = providerResult.FundingLineResults.Select(m => m.FundingLine.Name).ToArraySafe();
providerCalculationResultsIndex.FundingLineFundingStreamId = providerResult.FundingLineResults.Select(m => m.FundingLineFundingStreamId).ToArraySafe();
providerCalculationResultsIndex.FundingLineId = providerResult.FundingLineResults.Select(m => m.FundingLine.Id).ToArraySafe();
providerCalculationResultsIndex.FundingLineResult = providerResult.FundingLineResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe();
}
if (_featureToggle.IsExceptionMessagesEnabled())
{
providerCalculationResultsIndex.CalculationException = providerResult.CalculationResults
.Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType))
.Select(e => e.Calculation.Id)
.ToArraySafe();
providerCalculationResultsIndex.CalculationExceptionType = providerResult.CalculationResults
.Select(m => m.ExceptionType ?? string.Empty)
.ToArraySafe();
providerCalculationResultsIndex.CalculationExceptionMessage = providerResult.CalculationResults
.Select(m => m.ExceptionMessage ?? string.Empty)
.ToArraySafe();
if (providerResult.FundingLineResults != null)
{
providerCalculationResultsIndex.FundingLineException = providerResult.FundingLineResults
.Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType))
.Select(e => e.FundingLine.Id)
.ToArraySafe();
providerCalculationResultsIndex.FundingLineExceptionType = providerResult.FundingLineResults
.Select(m => m.ExceptionType ?? string.Empty)
.ToArraySafe();
providerCalculationResultsIndex.FundingLineExceptionMessage = providerResult.FundingLineResults
.Select(m => m.ExceptionMessage ?? string.Empty)
.ToArraySafe();
}
}
return(Task.FromResult(providerCalculationResultsIndex));
}