private GenerateAllocationMessageProperties GetMessageProperties(Message message)
{
GenerateAllocationMessageProperties properties = new GenerateAllocationMessageProperties();
if (message.UserProperties.ContainsKey("jobId"))
{
properties.JobId = message.UserProperties["jobId"].ToString();
}
else
{
_logger.Error("Missing job id for generating allocations");
}
string specificationId = message.UserProperties["specification-id"].ToString();
properties.SpecificationId = specificationId;
int batchNumber = 0;
if (message.UserProperties.ContainsKey("batch-number"))
{
batchNumber = int.Parse(message.UserProperties["batch-number"].ToString());
}
int batchCount = 0;
if (message.UserProperties.ContainsKey("batch-count"))
{
batchCount = int.Parse(message.UserProperties["batch-count"].ToString());
}
properties.BatchNumber = batchNumber;
properties.BatchCount = batchCount;
properties.ProviderCacheKey = message.UserProperties["provider-cache-key"].ToString();
properties.SpecificationSummaryCacheKey = message.UserProperties["specification-summary-cache-key"].ToString();
properties.PartitionIndex = int.Parse(message.UserProperties["provider-summaries-partition-index"].ToString());
properties.PartitionSize = int.Parse(message.UserProperties["provider-summaries-partition-size"].ToString());
properties.CalculationsAggregationsBatchCacheKey = $"{CacheKeys.CalculationAggregations}{specificationId}_{batchNumber}";
properties.CalculationsToAggregate = message.UserProperties.ContainsKey("calculations-to-aggregate") && !string.IsNullOrWhiteSpace(message.UserProperties["calculations-to-aggregate"].ToString()) ? message.UserProperties["calculations-to-aggregate"].ToString().Split(',') : null;
properties.User = message.GetUserDetails();
properties.CorrelationId = message.GetCorrelationId();
properties.AssemblyETag = message.GetUserProperty <string>("assembly-etag");
return(properties);
}
private async Task CompleteBatch(SpecificationSummary specificationSummary,
GenerateAllocationMessageProperties messageProperties,
Dictionary <string, List <object> > cachedCalculationAggregationsBatch)
{
Outcome = $"{ItemsSucceeded} provider results were generated successfully from {ItemsProcessed} providers";
if (messageProperties.GenerateCalculationAggregationsOnly)
{
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync(messageProperties.CalculationsAggregationsBatchCacheKey, cachedCalculationAggregationsBatch));
Outcome = $"{ItemsSucceeded} provider result calculation aggregations were generated successfully from {ItemsProcessed} providers";
}
}
private async Task CompleteBatch(GenerateAllocationMessageProperties messageProperties, Dictionary <string, List <decimal> > cachedCalculationAggregationsBatch, int itemsProcessed, int totalProviderResults)
{
int itemsSucceeded = totalProviderResults;
int itemsFailed = itemsProcessed - itemsSucceeded;
string outcome = $"{itemsSucceeded} provider results were generated successfully from {itemsProcessed} providers";
if (messageProperties.GenerateCalculationAggregationsOnly)
{
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync <Dictionary <string, List <decimal> > >(messageProperties.CalculationsAggregationsBatchCacheKey, cachedCalculationAggregationsBatch));
outcome = $"{itemsSucceeded} provider result calculation aggregations were generated successfully from {itemsProcessed} providers";
}
await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.AddJobLog(messageProperties.JobId, new JobLogUpdateModel
{
CompletedSuccessfully = true,
ItemsSucceeded = itemsSucceeded,
ItemsFailed = itemsFailed,
ItemsProcessed = itemsProcessed,
Outcome = outcome
}));
}
private async Task <(long saveCosmosElapsedMs, long queueSerachWriterElapsedMs, long saveRedisElapsedMs, long?saveQueueElapsedMs, int savedProviders)> ProcessProviderResults(
IEnumerable <ProviderResult> providerResults,
SpecificationSummary specificationSummary,
GenerateAllocationMessageProperties messageProperties,
Message message)
{
(long saveToCosmosElapsedMs, long saveToSearchElapsedMs, int savedProviders)saveProviderResultsTimings = (-1, -1, -1);
if (!message.UserProperties.ContainsKey("ignore-save-provider-results"))
{
_logger.Information($"Saving results for specification id {messageProperties.SpecificationId}");
saveProviderResultsTimings = await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providerResultsRepository.SaveProviderResults(providerResults,
specificationSummary,
messageProperties.PartitionIndex,
messageProperties.PartitionSize,
messageProperties.User,
messageProperties.CorrelationId,
messageProperties.JobId));
_logger.Information($"Saving results completeed for specification id {messageProperties.SpecificationId}");
}
Stopwatch saveQueueStopwatch = null;
Stopwatch saveRedisStopwatch = null;
if (_engineSettings.IsTestEngineEnabled)
{
// Should just be the GUID as the content, as the prefix is read by the receiver, rather than the sender
string providerResultsCacheKey = Guid.NewGuid().ToString();
_logger.Information($"Saving results to cache for specification id {messageProperties.SpecificationId} with key {providerResultsCacheKey}");
saveRedisStopwatch = Stopwatch.StartNew();
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync($"{CacheKeys.ProviderResultBatch}{providerResultsCacheKey}", providerResults.ToList(), TimeSpan.FromHours(12), false));
saveRedisStopwatch.Stop();
_logger.Information($"Saved results to cache for specification id {messageProperties.SpecificationId} with key {providerResultsCacheKey}");
IDictionary <string, string> properties = message.BuildMessageProperties();
properties.Add("specificationId", messageProperties.SpecificationId);
properties.Add("providerResultsCacheKey", providerResultsCacheKey);
_logger.Information($"Sending message for test exceution for specification id {messageProperties.SpecificationId}");
saveQueueStopwatch = Stopwatch.StartNew();
await _messengerServicePolicy.ExecuteAsync(() => _messengerService.SendToQueue <string>(ServiceBusConstants.QueueNames.TestEngineExecuteTests, null, properties));
saveQueueStopwatch.Stop();
_logger.Information($"Message sent for test exceution for specification id {messageProperties.SpecificationId}");
}
return(saveProviderResultsTimings.saveToCosmosElapsedMs,
saveProviderResultsTimings.saveToSearchElapsedMs,
saveRedisStopwatch != null ? saveRedisStopwatch.ElapsedMilliseconds : 0,
saveQueueStopwatch?.ElapsedMilliseconds,
saveProviderResultsTimings.savedProviders);
}
private void PopulateCachedCalculationAggregationsBatch(IEnumerable <ProviderResult> providerResults, Dictionary <string, List <object> > cachedCalculationAggregationsBatch, GenerateAllocationMessageProperties messageProperties)
{
if (cachedCalculationAggregationsBatch == null)
{
_logger.Error($"Cached calculation aggregations not found for key: {messageProperties.CalculationsAggregationsBatchCacheKey}");
throw new Exception($"Cached calculation aggregations not found for key: {messageProperties.CalculationsAggregationsBatchCacheKey}");
}
IEnumerable <string> calculationsToAggregate = messageProperties.CalculationsToAggregate;
foreach (ProviderResult providerResult in providerResults)
{
IEnumerable <CalculationResult> calculationResultsForAggregation = providerResult.CalculationResults.Where(m =>
calculationsToAggregate.Contains(_typeIdentifierGenerator.GenerateIdentifier(m.Calculation.Name), StringComparer.InvariantCultureIgnoreCase));
foreach (CalculationResult calculationResult in calculationResultsForAggregation)
{
string calculationReferenceName = _typeIdentifierGenerator.GenerateIdentifier(calculationResult.Calculation.Name.Trim());
string calcNameFromCalcsToAggregate = messageProperties.CalculationsToAggregate.FirstOrDefault(m => string.Equals(m, calculationReferenceName, StringComparison.InvariantCultureIgnoreCase));
if (!string.IsNullOrWhiteSpace(calcNameFromCalcsToAggregate) && cachedCalculationAggregationsBatch.ContainsKey(calculationReferenceName))
{
cachedCalculationAggregationsBatch[calcNameFromCalcsToAggregate].Add(calculationResult.Value ?? 0);
}
}
}
}
private async Task <(IEnumerable <CalculationAggregation>, long)> BuildAggregations(GenerateAllocationMessageProperties messageProperties)
{
Stopwatch sw = Stopwatch.StartNew();
BuildAggregationRequest aggreagationRequest = new BuildAggregationRequest
{
BatchCount = messageProperties.BatchCount,
GenerateCalculationAggregationsOnly = messageProperties.GenerateCalculationAggregationsOnly,
SpecificationId = messageProperties.SpecificationId
};
IEnumerable <CalculationAggregation> aggregations = await _calculationAggregationService.BuildAggregations(aggreagationRequest);
return(aggregations, sw.ElapsedMilliseconds);
}
private Dictionary <string, List <object> > CreateCalculationAggregateBatchDictionary(GenerateAllocationMessageProperties messageProperties)
{
if (!messageProperties.GenerateCalculationAggregationsOnly)
{
return(null);
}
Dictionary <string, List <object> > cachedCalculationAggregationsBatch = new Dictionary <string, List <object> >(StringComparer.InvariantCultureIgnoreCase);
if (!messageProperties.CalculationsToAggregate.IsNullOrEmpty())
{
foreach (string calcToAggregate in messageProperties.CalculationsToAggregate)
{
if (!cachedCalculationAggregationsBatch.ContainsKey(calcToAggregate))
{
cachedCalculationAggregationsBatch.Add(calcToAggregate, new List <object>());
}
}
}
return(cachedCalculationAggregationsBatch);
}
private async Task <CalculationResultsModel> CalculateResults(string specificationId, IEnumerable <ProviderSummary> summaries,
IEnumerable <CalculationSummaryModel> calculations,
IEnumerable <CalculationAggregation> aggregations,
IEnumerable <string> dataRelationshipIds,
byte[] assemblyForSpecification,
GenerateAllocationMessageProperties messageProperties,
int providerBatchSize,
int index)
{
ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>();
Guard.ArgumentNotNull(summaries, nameof(summaries));
IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize);
IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList();
Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew();
_logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}");
Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasetResult = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(
() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(specificationId, providerIdList, dataRelationshipIds));
providerSourceDatasetsStopwatch.Stop();
_logger.Information($"Fetched provider sources found for specification id {messageProperties.SpecificationId}");
_logger.Information($"Calculating results for specification id {messageProperties.SpecificationId}");
Stopwatch assemblyLoadStopwatch = Stopwatch.StartNew();
Assembly assembly = Assembly.Load(assemblyForSpecification);
assemblyLoadStopwatch.Stop();
Stopwatch calculationStopwatch = Stopwatch.StartNew();
List <Task> allTasks = new List <Task>();
SemaphoreSlim throttler = new SemaphoreSlim(_engineSettings.CalculateProviderResultsDegreeOfParallelism);
IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly);
foreach (ProviderSummary provider in partitionedSummaries)
{
await throttler.WaitAsync();
allTasks.Add(
Task.Run(() =>
{
try
{
if (provider == null)
{
throw new Exception("Provider summary is null");
}
if (providerSourceDatasetResult.AnyWithNullCheck())
{
if (!providerSourceDatasetResult.TryGetValue(provider.Id, out Dictionary <string, ProviderSourceDataset> providerDatasets))
{
throw new Exception($"Provider source dataset not found for {provider.Id}.");
}
ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, specificationId, calculations, provider, providerDatasets, aggregations);
if (result == null)
{
throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults");
}
providerResults.Add(result);
}
}
finally
{
throttler.Release();
}
return(Task.CompletedTask);
}));
}
await TaskHelper.WhenAllAndThrow(allTasks.ToArray());
calculationStopwatch.Stop();
_logger.Information($"Calculating results complete for specification id {messageProperties.SpecificationId} in {calculationStopwatch.ElapsedMilliseconds}ms");
return(new CalculationResultsModel
{
ProviderResults = providerResults,
PartitionedSummaries = partitionedSummaries,
CalculationRunMs = calculationStopwatch.ElapsedMilliseconds,
AssemblyLoadMs = assemblyLoadStopwatch.ElapsedMilliseconds,
ProviderSourceDatasetsLookupMs = providerSourceDatasetsStopwatch.ElapsedMilliseconds,
});
}
private async Task <(IEnumerable <ProviderSummary>, long)> GetProviderSummaries(GenerateAllocationMessageProperties messageProperties)
{
_logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}");
int start = messageProperties.PartitionIndex;
int stop = start + messageProperties.PartitionSize - 1;
Stopwatch providerSummaryLookup = Stopwatch.StartNew();
IEnumerable <ProviderSummary> summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop));
if (summaries == null)
{
throw new InvalidOperationException("Null provider summaries returned");
}
if (!summaries.Any())
{
throw new InvalidOperationException("No provider summaries returned to process");
}
providerSummaryLookup.Stop();
return(summaries, providerSummaryLookup.ElapsedMilliseconds);
}
private async Task GenerateAllocations(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
_logger.Debug("Validating new allocations message");
_calculationEngineServiceValidator.ValidateMessage(_logger, message);
GenerateAllocationMessageProperties messageProperties = GetMessageProperties(message);
string specificationId = messageProperties.SpecificationId;
messageProperties.GenerateCalculationAggregationsOnly = Job.JobDefinitionId == JobConstants.DefinitionNames.GenerateCalculationAggregationsJob;
_logger.Information($"Generating allocations for specification id {specificationId} on server '{Environment.MachineName}'");
Stopwatch prerequisiteStopwatch = Stopwatch.StartNew();
Task <(byte[], long)> getAssemblyTask = GetAssemblyForSpecification(specificationId, messageProperties.AssemblyETag);
Task <(IEnumerable <ProviderSummary>, long)> providerSummaryTask = GetProviderSummaries(messageProperties);
Task <(IEnumerable <CalculationSummaryModel>, long)> calculationSummaryTask = GetCalculationSummaries(specificationId);
Task <(SpecificationSummary, long)> specificationSummaryTask = GetSpecificationSummary(specificationId);
Task <(IEnumerable <CalculationAggregation>, long)> aggregationsTask = BuildAggregations(messageProperties);
await TaskHelper.WhenAllAndThrow(getAssemblyTask, providerSummaryTask, calculationSummaryTask, specificationSummaryTask, aggregationsTask);
(byte[] assembly, long assemblyLookupElapsedMilliseconds) = getAssemblyTask.Result;
(IEnumerable <ProviderSummary> summaries, long providerSummaryLookupElapsedMilliseconds) = providerSummaryTask.Result;
(IEnumerable <CalculationSummaryModel> calculations, long calculationsLookupStopwatchElapsedMilliseconds) = calculationSummaryTask.Result;
(SpecificationSummary specificationSummary, long specificationSummaryElapsedMilliseconds) = specificationSummaryTask.Result;
(IEnumerable <CalculationAggregation> aggregations, long aggregationsElapsedMilliseconds) = aggregationsTask.Result;
prerequisiteStopwatch.Stop();
int providerBatchSize = _engineSettings.ProviderBatchSize;
IEnumerable <string> dataRelationshipIds = specificationSummary.DataDefinitionRelationshipIds;
if (dataRelationshipIds == null)
{
throw new InvalidOperationException("Data relationship ids returned null");
}
int totalProviderResults = 0;
bool calculationResultsHaveExceptions = false;
Dictionary <string, List <object> > cachedCalculationAggregationsBatch = CreateCalculationAggregateBatchDictionary(messageProperties);
for (int i = 0; i < summaries.Count(); i += providerBatchSize)
{
Stopwatch calcTiming = Stopwatch.StartNew();
CalculationResultsModel calculationResults = await CalculateResults(specificationId,
summaries,
calculations,
aggregations,
dataRelationshipIds,
assembly,
messageProperties,
providerBatchSize,
i);
_logger.Information($"Calculating results complete for specification id {specificationId}");
long saveCosmosElapsedMs = -1;
long queueSearchWriterElapsedMs = -1;
long saveRedisElapsedMs = 0;
long?saveQueueElapsedMs = null;
int savedProviders = 0;
int percentageProvidersSaved = 0;
if (calculationResults.ProviderResults.Any())
{
if (messageProperties.GenerateCalculationAggregationsOnly)
{
PopulateCachedCalculationAggregationsBatch(calculationResults.ProviderResults, cachedCalculationAggregationsBatch, messageProperties);
totalProviderResults += calculationResults.ProviderResults.Count();
}
else
{
(long saveCosmosElapsedMs, long queueSerachWriterElapsedMs, long saveRedisElapsedMs, long?saveQueueElapsedMs, int savedProviders)processResultsMetrics =
await ProcessProviderResults(calculationResults.ProviderResults, specificationSummary, messageProperties, message);
saveCosmosElapsedMs = processResultsMetrics.saveCosmosElapsedMs;
queueSearchWriterElapsedMs = processResultsMetrics.queueSerachWriterElapsedMs;
saveRedisElapsedMs = processResultsMetrics.saveRedisElapsedMs;
saveQueueElapsedMs = processResultsMetrics.saveQueueElapsedMs;
savedProviders = processResultsMetrics.savedProviders;
totalProviderResults += calculationResults.ProviderResults.Count();
percentageProvidersSaved = savedProviders / totalProviderResults * 100;
if (calculationResults.ResultsContainExceptions)
{
_logger.Warning($"Exception(s) executing specification id '{specificationId}: {calculationResults.ExceptionMessages}");
calculationResultsHaveExceptions = true;
}
}
}
calcTiming.Stop();
IDictionary <string, double> metrics = new Dictionary <string, double>()
{
{ "calculation-run-providersProcessed", calculationResults.PartitionedSummaries.Count() },
{ "calculation-run-lookupCalculationDefinitionsMs", calculationsLookupStopwatchElapsedMilliseconds },
{ "calculation-run-providersResultsFromCache", summaries.Count() },
{ "calculation-run-partitionSize", messageProperties.PartitionSize },
{ "calculation-run-saveProviderResultsRedisMs", saveRedisElapsedMs },
{ "calculation-run-runningCalculationMs", calculationResults.CalculationRunMs },
{ "calculation-run-savedProviders", savedProviders },
{ "calculation-run-savePercentage ", percentageProvidersSaved },
{ "calculation-run-specLookup ", specificationSummaryElapsedMilliseconds },
{ "calculation-run-providerSummaryLookup ", providerSummaryLookupElapsedMilliseconds },
{ "calculation-run-providerSourceDatasetsLookupMs ", calculationResults.ProviderSourceDatasetsLookupMs },
{ "calculation-run-assemblyLookup ", assemblyLookupElapsedMilliseconds },
{ "calculation-run-aggregationsLookup ", aggregationsElapsedMilliseconds },
{ "calculation-run-prerequisiteMs ", prerequisiteStopwatch.ElapsedMilliseconds },
};
if (saveQueueElapsedMs.HasValue)
{
metrics.Add("calculation-run-saveProviderResultsServiceBusMs", saveQueueElapsedMs.Value);
}
if (saveCosmosElapsedMs > -1)
{
metrics.Add("calculation-run-batchElapsedMilliseconds", calcTiming.ElapsedMilliseconds);
metrics.Add("calculation-run-saveProviderResultsCosmosMs", saveCosmosElapsedMs);
}
else
{
metrics.Add("calculation-run-for-tests-ms", calcTiming.ElapsedMilliseconds);
}
if (queueSearchWriterElapsedMs > 0)
{
metrics.Add("calculation-run-queueSearchWriterMs", queueSearchWriterElapsedMs);
}
_telemetry.TrackEvent("CalculationRunProvidersProcessed",
new Dictionary <string, string>()
{
{ "specificationId", specificationId },
},
metrics
);
}
ItemsProcessed = summaries.Count();
ItemsFailed = summaries.Count() - totalProviderResults;
if (calculationResultsHaveExceptions)
{
throw new NonRetriableException($"Exceptions were thrown during generation of calculation results for specification '{specificationId}'");
}
else
{
await CompleteBatch(specificationSummary, messageProperties, cachedCalculationAggregationsBatch);
}
}
private async Task <IEnumerable <CalculationAggregation> > BuildAggregations(GenerateAllocationMessageProperties messageProperties)
{
IEnumerable <CalculationAggregation> aggregations = Enumerable.Empty <CalculationAggregation>();
aggregations = await _cacheProvider.GetAsync <List <CalculationAggregation> >($"{ CacheKeys.DatasetAggregationsForSpecification}{messageProperties.SpecificationId}");
if (aggregations.IsNullOrEmpty())
{
aggregations = (await _datasetAggregationsRepository.GetDatasetAggregationsForSpecificationId(messageProperties.SpecificationId)).Select(m => new CalculationAggregation
{
SpecificationId = m.SpecificationId,
Values = m.Fields.IsNullOrEmpty() ? Enumerable.Empty <AggregateValue>() : m.Fields.Select(f => new AggregateValue
{
AggregatedType = f.FieldType,
FieldDefinitionName = f.FieldDefinitionName,
Value = f.Value
})
});
await _cacheProvider.SetAsync <List <CalculationAggregation> >($"{CacheKeys.DatasetAggregationsForSpecification}{messageProperties.SpecificationId}", aggregations.ToList());
}
if (!messageProperties.GenerateCalculationAggregationsOnly)
{
Dictionary <string, List <decimal> > cachedCalculationAggregations = new Dictionary <string, List <decimal> >(StringComparer.InvariantCultureIgnoreCase);
for (int i = 1; i <= messageProperties.BatchCount; i++)
{
string batchedCacheKey = $"{CacheKeys.CalculationAggregations}{messageProperties.SpecificationId}_{i}";
Dictionary <string, List <decimal> > cachedCalculationAggregationsPart = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <Dictionary <string, List <decimal> > >(batchedCacheKey));
if (!cachedCalculationAggregationsPart.IsNullOrEmpty())
{
foreach (KeyValuePair <string, List <decimal> > cachedAggregations in cachedCalculationAggregationsPart)
{
if (!cachedCalculationAggregations.ContainsKey(cachedAggregations.Key))
{
cachedCalculationAggregations.Add(cachedAggregations.Key, new List <decimal>());
}
cachedCalculationAggregations[cachedAggregations.Key].AddRange(cachedAggregations.Value);
}
}
}
if (!cachedCalculationAggregations.IsNullOrEmpty())
{
foreach (KeyValuePair <string, List <decimal> > cachedCalculationAggregation in cachedCalculationAggregations)
{
aggregations = aggregations.Concat(new[]
{
new CalculationAggregation
{
SpecificationId = messageProperties.SpecificationId,
Values = new []
{
new AggregateValue {
FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Sum, Value = cachedCalculationAggregation.Value.Sum()
},
new AggregateValue {
FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Min, Value = cachedCalculationAggregation.Value.Min()
},
new AggregateValue {
FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Max, Value = cachedCalculationAggregation.Value.Max()
},
new AggregateValue {
FieldDefinitionName = cachedCalculationAggregation.Key, AggregatedType = AggregatedType.Average, Value = cachedCalculationAggregation.Value.Average()
},
}
}
});
}
}
}
return(aggregations);
}
private async Task <CalculationResultsModel> CalculateResults(IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, BuildProject buildProject,
GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index, Stopwatch providerSourceDatasetsStopwatch, Stopwatch calculationStopwatch)
{
ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>();
IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize);
IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList();
providerSourceDatasetsStopwatch.Start();
_logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}");
List <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>(await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIdList, messageProperties.SpecificationId)));
providerSourceDatasetsStopwatch.Stop();
if (providerSourceDatasets == null)
{
_logger.Information($"No provider sources found for specification id {messageProperties.SpecificationId}");
providerSourceDatasets = new List <ProviderSourceDataset>();
}
_logger.Information($"fetched provider sources found for specification id {messageProperties.SpecificationId}");
calculationStopwatch.Start();
_logger.Information($"calculating results for specification id {messageProperties.SpecificationId}");
Assembly assembly = Assembly.Load(buildProject.Build.Assembly);
Parallel.ForEach(partitionedSummaries, new ParallelOptions {
MaxDegreeOfParallelism = _engineSettings.CalculateProviderResultsDegreeOfParallelism
}, provider =>
{
IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly);
IEnumerable <ProviderSourceDataset> providerDatasets = providerSourceDatasets.Where(m => m.ProviderId == provider.Id);
ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, buildProject, calculations, provider, providerDatasets, aggregations);
if (result != null)
{
providerResults.Add(result);
}
else
{
throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults");
}
});
_logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}");
calculationStopwatch.Stop();
return(new CalculationResultsModel
{
ProviderResults = providerResults,
PartitionedSummaries = partitionedSummaries
});
}
public async Task GenerateAllocations(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
_logger.Information($"Validating new allocations message");
CalculationEngineServiceValidator.ValidateMessage(_logger, message);
GenerateAllocationMessageProperties messageProperties = GetMessageProperties(message);
JobViewModel job = await AddStartingProcessJobLog(messageProperties.JobId);
if (job == null)
{
return;
}
messageProperties.GenerateCalculationAggregationsOnly = (job.JobDefinitionId == JobConstants.DefinitionNames.GenerateCalculationAggregationsJob);
IEnumerable <ProviderSummary> summaries = null;
_logger.Information($"Generating allocations for specification id {messageProperties.SpecificationId}");
BuildProject buildProject = await GetBuildProject(messageProperties.SpecificationId);
byte[] assembly = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetAssemblyBySpecificationId(messageProperties.SpecificationId));
if (assembly == null)
{
string error = $"Failed to get assembly for specification Id '{messageProperties.SpecificationId}'";
_logger.Error(error);
throw new RetriableException(error);
}
buildProject.Build.Assembly = assembly;
Dictionary <string, List <decimal> > cachedCalculationAggregationsBatch = CreateCalculationAggregateBatchDictionary(messageProperties);
_logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}");
int start = messageProperties.PartitionIndex;
int stop = start + messageProperties.PartitionSize - 1;
summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop));
int providerBatchSize = _engineSettings.ProviderBatchSize;
Stopwatch calculationsLookupStopwatch = Stopwatch.StartNew();
IEnumerable <CalculationSummaryModel> calculations = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetCalculationSummariesForSpecification(messageProperties.SpecificationId));
if (calculations == null)
{
_logger.Error($"Calculations lookup API returned null for specification id {messageProperties.SpecificationId}");
throw new InvalidOperationException("Calculations lookup API returned null");
}
calculationsLookupStopwatch.Stop();
IEnumerable <CalculationAggregation> aggregations = await BuildAggregations(messageProperties);
int totalProviderResults = 0;
bool calculationResultsHaveExceptions = false;
for (int i = 0; i < summaries.Count(); i += providerBatchSize)
{
Stopwatch calculationStopwatch = new Stopwatch();
Stopwatch providerSourceDatasetsStopwatch = new Stopwatch();
Stopwatch calcTiming = Stopwatch.StartNew();
CalculationResultsModel calculationResults = await CalculateResults(summaries, calculations, aggregations, buildProject, messageProperties, providerBatchSize, i, providerSourceDatasetsStopwatch, calculationStopwatch);
_logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}");
long saveCosmosElapsedMs = -1;
long saveSearchElapsedMs = -1;
long saveRedisElapsedMs = 0;
long saveQueueElapsedMs = 0;
if (calculationResults.ProviderResults.Any())
{
if (messageProperties.GenerateCalculationAggregationsOnly)
{
PopulateCachedCalculationAggregationsBatch(calculationResults.ProviderResults, cachedCalculationAggregationsBatch, messageProperties);
}
else
{
(long saveCosmosElapsedMs, long saveSearchElapsedMs, long saveRedisElapsedMs, long saveQueueElapsedMs)timingMetrics = await ProcessProviderResults(calculationResults.ProviderResults, messageProperties, message);
saveCosmosElapsedMs = timingMetrics.saveCosmosElapsedMs;
saveSearchElapsedMs = timingMetrics.saveSearchElapsedMs;
saveRedisElapsedMs = timingMetrics.saveRedisElapsedMs;
saveQueueElapsedMs = timingMetrics.saveQueueElapsedMs;
totalProviderResults += calculationResults.ProviderResults.Count();
if (calculationResults.ResultsContainExceptions)
{
if (!calculationResultsHaveExceptions)
{
calculationResultsHaveExceptions = true;
}
}
}
}
calcTiming.Stop();
IDictionary <string, double> metrics = new Dictionary <string, double>()
{
{ "calculation-run-providersProcessed", calculationResults.PartitionedSummaries.Count() },
{ "calculation-run-lookupCalculationDefinitionsMs", calculationsLookupStopwatch.ElapsedMilliseconds },
{ "calculation-run-providersResultsFromCache", summaries.Count() },
{ "calculation-run-partitionSize", messageProperties.PartitionSize },
{ "calculation-run-providerSourceDatasetQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds },
{ "calculation-run-saveProviderResultsRedisMs", saveRedisElapsedMs },
{ "calculation-run-saveProviderResultsServiceBusMs", saveQueueElapsedMs },
{ "calculation-run-runningCalculationMs", calculationStopwatch.ElapsedMilliseconds },
};
if (saveCosmosElapsedMs > -1)
{
metrics.Add("calculation-run-elapsedMilliseconds", calcTiming.ElapsedMilliseconds);
metrics.Add("calculation-run-saveProviderResultsCosmosMs", saveCosmosElapsedMs);
metrics.Add("calculation-run-saveProviderResultsSearchMs", saveSearchElapsedMs);
}
else
{
metrics.Add("calculation-run-for-tests-ms", calcTiming.ElapsedMilliseconds);
}
_telemetry.TrackEvent("CalculationRunProvidersProcessed",
new Dictionary <string, string>()
{
{ "specificationId", messageProperties.SpecificationId },
{ "buildProjectId", buildProject.Id },
},
metrics
);
}
if (calculationResultsHaveExceptions)
{
await FailJob(messageProperties.JobId, totalProviderResults, "Exceptions were thrown during generation of calculation results");
}
else
{
await CompleteBatch(messageProperties, cachedCalculationAggregationsBatch, summaries.Count(), totalProviderResults);
}
}
