public async Task <IEnumerable <CalculationAggregation> > BuildAggregations(BuildAggregationRequest aggreagationRequest) { IEnumerable <CalculationAggregation> aggregations = Enumerable.Empty <CalculationAggregation>(); aggregations = await _cacheProvider.GetAsync <List <CalculationAggregation> >( $"{ CacheKeys.DatasetAggregationsForSpecification}{aggreagationRequest.SpecificationId}"); if (DoesNotExistInCache(aggregations)) { aggregations = (await _datasetAggregationsRepository.GetDatasetAggregationsForSpecificationId(aggreagationRequest.SpecificationId)).Select(m => new CalculationAggregation { SpecificationId = m.SpecificationId, Values = m.Fields.IsNullOrEmpty() ? Enumerable.Empty <AggregateValue>() : m.Fields.Select(f => new AggregateValue { AggregatedType = f.FieldType, FieldDefinitionName = f.FieldDefinitionName, Value = f.Value }) }); await _cacheProvider.SetAsync($"{CacheKeys.DatasetAggregationsForSpecification}{aggreagationRequest.SpecificationId}", aggregations.ToList()); } if (!aggreagationRequest.GenerateCalculationAggregationsOnly) { ConcurrentDictionary <string, List <decimal> > cachedCalculationAggregations = new ConcurrentDictionary <string, List <decimal> >( StringComparer.InvariantCultureIgnoreCase); List <Task> allTasks = new List <Task>(); SemaphoreSlim throttler = new SemaphoreSlim(_engineSettings.CalculationAggregationRetreivalParallelism); for (int i = 1; i <= aggreagationRequest.BatchCount; i++) { await throttler.WaitAsync(); int currentBatchNumber = i; allTasks.Add( Task.Run(async() => { try { string batchedCacheKey = $"{CacheKeys.CalculationAggregations}{aggreagationRequest.SpecificationId}_{currentBatchNumber}"; Dictionary <string, List <decimal> > cachedCalculationAggregationsPart = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <Dictionary <string, List <decimal> > >(batchedCacheKey)); if (!cachedCalculationAggregationsPart.IsNullOrEmpty()) { foreach (KeyValuePair <string, List <decimal> > cachedAggregations in cachedCalculationAggregationsPart) { List <decimal> values = cachedCalculationAggregations.GetOrAdd(cachedAggregations.Key, new List <decimal>()); values.AddRange(cachedAggregations.Value); } } } finally { throttler.Release(); } })); } await TaskHelper.WhenAllAndThrow(allTasks.ToArray()); if (!cachedCalculationAggregations.IsNullOrEmpty()) { foreach (KeyValuePair <string, List <decimal> > cachedCalculationAggregation in cachedCalculationAggregations.OrderBy(o => o.Key)) { aggregations = aggregations.Concat(new[] { new CalculationAggregation { SpecificationId = aggreagationRequest.SpecificationId, Values = new [] { new AggregateValue { FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Sum, Value = cachedCalculationAggregation.Value.Sum() }, new AggregateValue { FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Min, Value = cachedCalculationAggregation.Value.Min() }, new AggregateValue { FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Max, Value = cachedCalculationAggregation.Value.Max() }, new AggregateValue { FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Average, Value = cachedCalculationAggregation.Value.Average() }, } } }); } } } return(aggregations); }
private async Task <IEnumerable <CalculationAggregation> > BuildAggregations(GenerateAllocationMessageProperties messageProperties) { IEnumerable <CalculationAggregation> aggregations = Enumerable.Empty <CalculationAggregation>(); aggregations = await _cacheProvider.GetAsync <List <CalculationAggregation> >($"{ CacheKeys.DatasetAggregationsForSpecification}{messageProperties.SpecificationId}"); if (aggregations.IsNullOrEmpty()) { aggregations = (await _datasetAggregationsRepository.GetDatasetAggregationsForSpecificationId(messageProperties.SpecificationId)).Select(m => new CalculationAggregation { SpecificationId = m.SpecificationId, Values = m.Fields.IsNullOrEmpty() ? Enumerable.Empty <AggregateValue>() : m.Fields.Select(f => new AggregateValue { AggregatedType = f.FieldType, FieldDefinitionName = f.FieldDefinitionName, Value = f.Value }) }); await _cacheProvider.SetAsync <List <CalculationAggregation> >($"{CacheKeys.DatasetAggregationsForSpecification}{messageProperties.SpecificationId}", aggregations.ToList()); } if (!messageProperties.GenerateCalculationAggregationsOnly) { Dictionary <string, List <decimal> > cachedCalculationAggregations = new Dictionary <string, List <decimal> >(StringComparer.InvariantCultureIgnoreCase); for (int i = 1; i <= messageProperties.BatchCount; i++) { string batchedCacheKey = $"{CacheKeys.CalculationAggregations}{messageProperties.SpecificationId}_{i}"; Dictionary <string, List <decimal> > cachedCalculationAggregationsPart = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <Dictionary <string, List <decimal> > >(batchedCacheKey)); if (!cachedCalculationAggregationsPart.IsNullOrEmpty()) { foreach (KeyValuePair <string, List <decimal> > cachedAggregations in cachedCalculationAggregationsPart) { if (!cachedCalculationAggregations.ContainsKey(cachedAggregations.Key)) { cachedCalculationAggregations.Add(cachedAggregations.Key, new List <decimal>()); } cachedCalculationAggregations[cachedAggregations.Key].AddRange(cachedAggregations.Value); } } } if (!cachedCalculationAggregations.IsNullOrEmpty()) { foreach (KeyValuePair <string, List <decimal> > cachedCalculationAggregation in cachedCalculationAggregations) { aggregations = aggregations.Concat(new[] { new CalculationAggregation { SpecificationId = messageProperties.SpecificationId, Values = new [] { new AggregateValue { FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Sum, Value = cachedCalculationAggregation.Value.Sum() }, new AggregateValue { FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Min, Value = cachedCalculationAggregation.Value.Min() }, new AggregateValue { FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Max, Value = cachedCalculationAggregation.Value.Max() }, new AggregateValue { FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Average, Value = cachedCalculationAggregation.Value.Average() }, } } }); } } } return(aggregations); }