protected override IEnumerable <ProviderResultDataKey> IndexDataItemKeys(ISearchIndexProcessorContext context) { var processorContext = context as ProviderCalculationResultsIndexProcessorContext; Guard.ArgumentNotNull(processorContext, nameof(ProviderCalculationResultsIndexProcessorContext)); foreach (string providerId in processorContext.ProviderIds) { byte[] providerResultIdBytes = Encoding.UTF8.GetBytes($"{providerId}-{processorContext.SpecificationId}"); yield return(new ProviderResultDataKey(Convert.ToBase64String(providerResultIdBytes), providerId)); } }
public async Task Process(Message message) { List <string> exceptionMessages = new List <string>(); ISearchIndexProcessorContext context = CreateContext(message); SemaphoreSlim throttler = new SemaphoreSlim(context.DegreeOfParallelism, context.DegreeOfParallelism); List <Task> indexTasks = new List <Task>(); foreach (TKey key in IndexDataItemKeys(context)) { await throttler.WaitAsync(); indexTasks.Add ( Task.Run(async() => { try { TInput indexData = await _reader.GetData(key); if (indexData != null) { TOutput indexDocument = await _transformer.Transform(indexData, context); IEnumerable <IndexError> results = await _searchRepository.Index(new[] { indexDocument }); if (!results.IsNullOrEmpty()) { IndexError indexError = results.First(); // Only indexing one document exceptionMessages.Add($"{indexError.Key}:{indexError.ErrorMessage}"); } } else { string message = $"No data found for given Key - {key.ToString()}"; _logger.Error(message); exceptionMessages.Add(message); } } catch (Exception ex) { if (ex is AggregateException) { ex = ((AggregateException)ex).Flatten().InnerExceptions.FirstOrDefault() ?? ex; } _logger.Error(ex, $"Error occurred while processing the {IndexName} - {ex.Message}"); exceptionMessages.Add(ex.Message); } finally { throttler.Release(); } }) ); } await TaskHelper.WhenAllAndThrow(indexTasks.ToArray()); if (exceptionMessages.Any()) { throw new Exception($"Error occurred while processing the {IndexName}. {string.Join(";", exceptionMessages)}"); } }
protected abstract IEnumerable <TKey> IndexDataItemKeys(ISearchIndexProcessorContext context);
public Task <ProviderCalculationResultsIndex> Transform(ProviderResult providerResult, ISearchIndexProcessorContext context) { Guard.ArgumentNotNull(providerResult, nameof(ProviderResult)); var processorContext = context as ProviderCalculationResultsIndexProcessorContext; Guard.ArgumentNotNull(processorContext, nameof(ProviderCalculationResultsIndexProcessorContext)); ProviderCalculationResultsIndex providerCalculationResultsIndex = new ProviderCalculationResultsIndex { SpecificationId = providerResult.SpecificationId, SpecificationName = processorContext.SpecificationName, ProviderId = providerResult.Provider?.Id, ProviderName = providerResult.Provider?.Name, ProviderType = providerResult.Provider?.ProviderType, ProviderSubType = providerResult.Provider?.ProviderSubType, LocalAuthority = providerResult.Provider?.Authority, LastUpdatedDate = DateTimeOffset.Now, UKPRN = providerResult.Provider?.UKPRN, URN = providerResult.Provider?.URN, UPIN = providerResult.Provider?.UPIN, EstablishmentNumber = providerResult.Provider?.EstablishmentNumber, OpenDate = providerResult.Provider?.DateOpened, CalculationId = providerResult.CalculationResults.Select(m => m.Calculation.Id).ToArraySafe(), CalculationName = providerResult.CalculationResults.Select(m => m.Calculation.Name).ToArraySafe(), CalculationResult = providerResult.CalculationResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe(), }; if (providerResult.FundingLineResults != null) { providerCalculationResultsIndex.FundingLineName = providerResult.FundingLineResults.Select(m => m.FundingLine.Name).ToArraySafe(); providerCalculationResultsIndex.FundingLineFundingStreamId = providerResult.FundingLineResults.Select(m => m.FundingLineFundingStreamId).ToArraySafe(); providerCalculationResultsIndex.FundingLineId = providerResult.FundingLineResults.Select(m => m.FundingLine.Id).ToArraySafe(); providerCalculationResultsIndex.FundingLineResult = providerResult.FundingLineResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe(); } if (_featureToggle.IsExceptionMessagesEnabled()) { providerCalculationResultsIndex.CalculationException = providerResult.CalculationResults .Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType)) .Select(e => e.Calculation.Id) .ToArraySafe(); providerCalculationResultsIndex.CalculationExceptionType = providerResult.CalculationResults .Select(m => m.ExceptionType ?? string.Empty) .ToArraySafe(); providerCalculationResultsIndex.CalculationExceptionMessage = providerResult.CalculationResults .Select(m => m.ExceptionMessage ?? string.Empty) .ToArraySafe(); if (providerResult.FundingLineResults != null) { providerCalculationResultsIndex.FundingLineException = providerResult.FundingLineResults .Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType)) .Select(e => e.FundingLine.Id) .ToArraySafe(); providerCalculationResultsIndex.FundingLineExceptionType = providerResult.FundingLineResults .Select(m => m.ExceptionType ?? string.Empty) .ToArraySafe(); providerCalculationResultsIndex.FundingLineExceptionMessage = providerResult.FundingLineResults .Select(m => m.ExceptionMessage ?? string.Empty) .ToArraySafe(); } } return(Task.FromResult(providerCalculationResultsIndex)); }