public async Task ShouldWriteSearchIndexForAllProviderResults_WhenProcessTheMessage_GivenProviderResultsForSepecificationAndProviderIds() { // Arrange string specificationId = new RandomString(); string specificationName = new RandomString(); string providerId1 = new RandomString(); string providerId2 = new RandomString(); string providerResultId1 = CreateProviderResultId(specificationId, providerId1); string providerResultId2 = CreateProviderResultId(specificationId, providerId2); ProviderResultDataKey providerResultDataKey1 = new ProviderResultDataKey(providerResultId1, providerId1); ProviderResultDataKey providerResultDataKey2 = new ProviderResultDataKey(providerResultId2, providerId2); ProviderResult providerResult1 = new ProviderResult() { Id = providerResultId1 }; ProviderResult providerResult2 = new ProviderResult() { Id = providerResultId2 }; ProviderCalculationResultsIndex index1 = new ProviderCalculationResultsIndex() { SpecificationId = specificationId, ProviderId = providerId1 }; ProviderCalculationResultsIndex index2 = new ProviderCalculationResultsIndex() { SpecificationId = specificationId, ProviderId = providerId2 }; _reader.GetData(Arg.Is <ProviderResultDataKey>(_ => _.ProviderResultId == providerResultId1 && _.PartitionKey == providerId1)) .Returns(providerResult1); _reader.GetData(Arg.Is <ProviderResultDataKey>(_ => _.ProviderResultId == providerResultId2 && _.PartitionKey == providerId2)) .Returns(providerResult2); _transformer.Transform(Arg.Is <ProviderResult>(_ => _.Id == providerResultId1), Arg.Is <ISearchIndexProcessorContext>(_ => _.GetType() == typeof(ProviderCalculationResultsIndexProcessorContext))) .Returns(index1); _transformer.Transform(Arg.Is <ProviderResult>(_ => _.Id == providerResultId2), Arg.Is <ISearchIndexProcessorContext>(_ => _.GetType() == typeof(ProviderCalculationResultsIndexProcessorContext))) .Returns(index2); _searchRepository.Index(Arg.Is <IEnumerable <ProviderCalculationResultsIndex> >(_ => _.Any(x => x.ProviderId == providerId1 || x.ProviderId == providerId2))) .Returns(Enumerable.Empty <IndexError>()); Message message = new Message(); message.UserProperties.Add("specification-id", specificationId); message.UserProperties.Add("specification-name", specificationName); message.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new[] { providerId1, providerId2 })); // Act await _processor.Process(message); // Assert await _searchRepository .Received(2) .Index(Arg.Is <IEnumerable <ProviderCalculationResultsIndex> >(_ => _.Any(x => x.ProviderId == providerId1 || x.ProviderId == providerId2))); await _reader .Received(2) .GetData(Arg.Is <ProviderResultDataKey>(_ => (_.ProviderResultId == providerResultId1 && _.PartitionKey == providerId1) || (_.ProviderResultId == providerResultId2 && _.PartitionKey == providerId2))); await _transformer .Received(2) .Transform(Arg.Is <ProviderResult>(_ => _.Id == providerResultId1 || _.Id == providerResultId2), Arg.Is <ISearchIndexProcessorContext>(_ => _.GetType() == typeof(ProviderCalculationResultsIndexProcessorContext))); }
public async Task Process(Message message) { List <string> exceptionMessages = new List <string>(); ISearchIndexProcessorContext context = CreateContext(message); SemaphoreSlim throttler = new SemaphoreSlim(context.DegreeOfParallelism, context.DegreeOfParallelism); List <Task> indexTasks = new List <Task>(); foreach (TKey key in IndexDataItemKeys(context)) { await throttler.WaitAsync(); indexTasks.Add ( Task.Run(async() => { try { TInput indexData = await _reader.GetData(key); if (indexData != null) { TOutput indexDocument = await _transformer.Transform(indexData, context); IEnumerable <IndexError> results = await _searchRepository.Index(new[] { indexDocument }); if (!results.IsNullOrEmpty()) { IndexError indexError = results.First(); // Only indexing one document exceptionMessages.Add($"{indexError.Key}:{indexError.ErrorMessage}"); } } else { string message = $"No data found for given Key - {key.ToString()}"; _logger.Error(message); exceptionMessages.Add(message); } } catch (Exception ex) { if (ex is AggregateException) { ex = ((AggregateException)ex).Flatten().InnerExceptions.FirstOrDefault() ?? ex; } _logger.Error(ex, $"Error occurred while processing the {IndexName} - {ex.Message}"); exceptionMessages.Add(ex.Message); } finally { throttler.Release(); } }) ); } await TaskHelper.WhenAllAndThrow(indexTasks.ToArray()); if (exceptionMessages.Any()) { throw new Exception($"Error occurred while processing the {IndexName}. {string.Join(";", exceptionMessages)}"); } }