private async Task GenerateAllocations(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
_logger.Debug("Validating new allocations message");
_calculationEngineServiceValidator.ValidateMessage(_logger, message);
GenerateAllocationMessageProperties messageProperties = GetMessageProperties(message);
string specificationId = messageProperties.SpecificationId;
messageProperties.GenerateCalculationAggregationsOnly = Job.JobDefinitionId == JobConstants.DefinitionNames.GenerateCalculationAggregationsJob;
_logger.Information($"Generating allocations for specification id {specificationId} on server '{Environment.MachineName}'");
Stopwatch prerequisiteStopwatch = Stopwatch.StartNew();
Task <(byte[], long)> getAssemblyTask = GetAssemblyForSpecification(specificationId, messageProperties.AssemblyETag);
Task <(IEnumerable <ProviderSummary>, long)> providerSummaryTask = GetProviderSummaries(messageProperties);
Task <(IEnumerable <CalculationSummaryModel>, long)> calculationSummaryTask = GetCalculationSummaries(specificationId);
Task <(SpecificationSummary, long)> specificationSummaryTask = GetSpecificationSummary(specificationId);
Task <(IEnumerable <CalculationAggregation>, long)> aggregationsTask = BuildAggregations(messageProperties);
await TaskHelper.WhenAllAndThrow(getAssemblyTask, providerSummaryTask, calculationSummaryTask, specificationSummaryTask, aggregationsTask);
(byte[] assembly, long assemblyLookupElapsedMilliseconds) = getAssemblyTask.Result;
(IEnumerable <ProviderSummary> summaries, long providerSummaryLookupElapsedMilliseconds) = providerSummaryTask.Result;
(IEnumerable <CalculationSummaryModel> calculations, long calculationsLookupStopwatchElapsedMilliseconds) = calculationSummaryTask.Result;
(SpecificationSummary specificationSummary, long specificationSummaryElapsedMilliseconds) = specificationSummaryTask.Result;
(IEnumerable <CalculationAggregation> aggregations, long aggregationsElapsedMilliseconds) = aggregationsTask.Result;
prerequisiteStopwatch.Stop();
int providerBatchSize = _engineSettings.ProviderBatchSize;
IEnumerable <string> dataRelationshipIds = specificationSummary.DataDefinitionRelationshipIds;
if (dataRelationshipIds == null)
{
throw new InvalidOperationException("Data relationship ids returned null");
}
int totalProviderResults = 0;
bool calculationResultsHaveExceptions = false;
Dictionary <string, List <object> > cachedCalculationAggregationsBatch = CreateCalculationAggregateBatchDictionary(messageProperties);
for (int i = 0; i < summaries.Count(); i += providerBatchSize)
{
Stopwatch calcTiming = Stopwatch.StartNew();
CalculationResultsModel calculationResults = await CalculateResults(specificationId,
summaries,
calculations,
aggregations,
dataRelationshipIds,
assembly,
messageProperties,
providerBatchSize,
i);
_logger.Information($"Calculating results complete for specification id {specificationId}");
long saveCosmosElapsedMs = -1;
long queueSearchWriterElapsedMs = -1;
long saveRedisElapsedMs = 0;
long?saveQueueElapsedMs = null;
int savedProviders = 0;
int percentageProvidersSaved = 0;
if (calculationResults.ProviderResults.Any())
{
if (messageProperties.GenerateCalculationAggregationsOnly)
{
PopulateCachedCalculationAggregationsBatch(calculationResults.ProviderResults, cachedCalculationAggregationsBatch, messageProperties);
totalProviderResults += calculationResults.ProviderResults.Count();
}
else
{
(long saveCosmosElapsedMs, long queueSerachWriterElapsedMs, long saveRedisElapsedMs, long?saveQueueElapsedMs, int savedProviders)processResultsMetrics =
await ProcessProviderResults(calculationResults.ProviderResults, specificationSummary, messageProperties, message);
saveCosmosElapsedMs = processResultsMetrics.saveCosmosElapsedMs;
queueSearchWriterElapsedMs = processResultsMetrics.queueSerachWriterElapsedMs;
saveRedisElapsedMs = processResultsMetrics.saveRedisElapsedMs;
saveQueueElapsedMs = processResultsMetrics.saveQueueElapsedMs;
savedProviders = processResultsMetrics.savedProviders;
totalProviderResults += calculationResults.ProviderResults.Count();
percentageProvidersSaved = savedProviders / totalProviderResults * 100;
if (calculationResults.ResultsContainExceptions)
{
_logger.Warning($"Exception(s) executing specification id '{specificationId}: {calculationResults.ExceptionMessages}");
calculationResultsHaveExceptions = true;
}
}
}
calcTiming.Stop();
IDictionary <string, double> metrics = new Dictionary <string, double>()
{
{ "calculation-run-providersProcessed", calculationResults.PartitionedSummaries.Count() },
{ "calculation-run-lookupCalculationDefinitionsMs", calculationsLookupStopwatchElapsedMilliseconds },
{ "calculation-run-providersResultsFromCache", summaries.Count() },
{ "calculation-run-partitionSize", messageProperties.PartitionSize },
{ "calculation-run-saveProviderResultsRedisMs", saveRedisElapsedMs },
{ "calculation-run-runningCalculationMs", calculationResults.CalculationRunMs },
{ "calculation-run-savedProviders", savedProviders },
{ "calculation-run-savePercentage ", percentageProvidersSaved },
{ "calculation-run-specLookup ", specificationSummaryElapsedMilliseconds },
{ "calculation-run-providerSummaryLookup ", providerSummaryLookupElapsedMilliseconds },
{ "calculation-run-providerSourceDatasetsLookupMs ", calculationResults.ProviderSourceDatasetsLookupMs },
{ "calculation-run-assemblyLookup ", assemblyLookupElapsedMilliseconds },
{ "calculation-run-aggregationsLookup ", aggregationsElapsedMilliseconds },
{ "calculation-run-prerequisiteMs ", prerequisiteStopwatch.ElapsedMilliseconds },
};
if (saveQueueElapsedMs.HasValue)
{
metrics.Add("calculation-run-saveProviderResultsServiceBusMs", saveQueueElapsedMs.Value);
}
if (saveCosmosElapsedMs > -1)
{
metrics.Add("calculation-run-batchElapsedMilliseconds", calcTiming.ElapsedMilliseconds);
metrics.Add("calculation-run-saveProviderResultsCosmosMs", saveCosmosElapsedMs);
}
else
{
metrics.Add("calculation-run-for-tests-ms", calcTiming.ElapsedMilliseconds);
}
if (queueSearchWriterElapsedMs > 0)
{
metrics.Add("calculation-run-queueSearchWriterMs", queueSearchWriterElapsedMs);
}
_telemetry.TrackEvent("CalculationRunProvidersProcessed",
new Dictionary <string, string>()
{
{ "specificationId", specificationId },
},
metrics
);
}
ItemsProcessed = summaries.Count();
ItemsFailed = summaries.Count() - totalProviderResults;
if (calculationResultsHaveExceptions)
{
throw new NonRetriableException($"Exceptions were thrown during generation of calculation results for specification '{specificationId}'");
}
else
{
await CompleteBatch(specificationSummary, messageProperties, cachedCalculationAggregationsBatch);
}
}
public async Task GenerateAllocations(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
_logger.Information($"Validating new allocations message");
CalculationEngineServiceValidator.ValidateMessage(_logger, message);
GenerateAllocationMessageProperties messageProperties = GetMessageProperties(message);
JobViewModel job = await AddStartingProcessJobLog(messageProperties.JobId);
if (job == null)
{
return;
}
messageProperties.GenerateCalculationAggregationsOnly = (job.JobDefinitionId == JobConstants.DefinitionNames.GenerateCalculationAggregationsJob);
IEnumerable <ProviderSummary> summaries = null;
_logger.Information($"Generating allocations for specification id {messageProperties.SpecificationId}");
BuildProject buildProject = await GetBuildProject(messageProperties.SpecificationId);
byte[] assembly = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetAssemblyBySpecificationId(messageProperties.SpecificationId));
if (assembly == null)
{
string error = $"Failed to get assembly for specification Id '{messageProperties.SpecificationId}'";
_logger.Error(error);
throw new RetriableException(error);
}
buildProject.Build.Assembly = assembly;
Dictionary <string, List <decimal> > cachedCalculationAggregationsBatch = CreateCalculationAggregateBatchDictionary(messageProperties);
_logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}");
int start = messageProperties.PartitionIndex;
int stop = start + messageProperties.PartitionSize - 1;
summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop));
int providerBatchSize = _engineSettings.ProviderBatchSize;
Stopwatch calculationsLookupStopwatch = Stopwatch.StartNew();
IEnumerable <CalculationSummaryModel> calculations = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetCalculationSummariesForSpecification(messageProperties.SpecificationId));
if (calculations == null)
{
_logger.Error($"Calculations lookup API returned null for specification id {messageProperties.SpecificationId}");
throw new InvalidOperationException("Calculations lookup API returned null");
}
calculationsLookupStopwatch.Stop();
IEnumerable <CalculationAggregation> aggregations = await BuildAggregations(messageProperties);
int totalProviderResults = 0;
bool calculationResultsHaveExceptions = false;
for (int i = 0; i < summaries.Count(); i += providerBatchSize)
{
Stopwatch calculationStopwatch = new Stopwatch();
Stopwatch providerSourceDatasetsStopwatch = new Stopwatch();
Stopwatch calcTiming = Stopwatch.StartNew();
CalculationResultsModel calculationResults = await CalculateResults(summaries, calculations, aggregations, buildProject, messageProperties, providerBatchSize, i, providerSourceDatasetsStopwatch, calculationStopwatch);
_logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}");
long saveCosmosElapsedMs = -1;
long saveSearchElapsedMs = -1;
long saveRedisElapsedMs = 0;
long saveQueueElapsedMs = 0;
if (calculationResults.ProviderResults.Any())
{
if (messageProperties.GenerateCalculationAggregationsOnly)
{
PopulateCachedCalculationAggregationsBatch(calculationResults.ProviderResults, cachedCalculationAggregationsBatch, messageProperties);
}
else
{
(long saveCosmosElapsedMs, long saveSearchElapsedMs, long saveRedisElapsedMs, long saveQueueElapsedMs)timingMetrics = await ProcessProviderResults(calculationResults.ProviderResults, messageProperties, message);
saveCosmosElapsedMs = timingMetrics.saveCosmosElapsedMs;
saveSearchElapsedMs = timingMetrics.saveSearchElapsedMs;
saveRedisElapsedMs = timingMetrics.saveRedisElapsedMs;
saveQueueElapsedMs = timingMetrics.saveQueueElapsedMs;
totalProviderResults += calculationResults.ProviderResults.Count();
if (calculationResults.ResultsContainExceptions)
{
if (!calculationResultsHaveExceptions)
{
calculationResultsHaveExceptions = true;
}
}
}
}
calcTiming.Stop();
IDictionary <string, double> metrics = new Dictionary <string, double>()
{
{ "calculation-run-providersProcessed", calculationResults.PartitionedSummaries.Count() },
{ "calculation-run-lookupCalculationDefinitionsMs", calculationsLookupStopwatch.ElapsedMilliseconds },
{ "calculation-run-providersResultsFromCache", summaries.Count() },
{ "calculation-run-partitionSize", messageProperties.PartitionSize },
{ "calculation-run-providerSourceDatasetQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds },
{ "calculation-run-saveProviderResultsRedisMs", saveRedisElapsedMs },
{ "calculation-run-saveProviderResultsServiceBusMs", saveQueueElapsedMs },
{ "calculation-run-runningCalculationMs", calculationStopwatch.ElapsedMilliseconds },
};
if (saveCosmosElapsedMs > -1)
{
metrics.Add("calculation-run-elapsedMilliseconds", calcTiming.ElapsedMilliseconds);
metrics.Add("calculation-run-saveProviderResultsCosmosMs", saveCosmosElapsedMs);
metrics.Add("calculation-run-saveProviderResultsSearchMs", saveSearchElapsedMs);
}
else
{
metrics.Add("calculation-run-for-tests-ms", calcTiming.ElapsedMilliseconds);
}
_telemetry.TrackEvent("CalculationRunProvidersProcessed",
new Dictionary <string, string>()
{
{ "specificationId", messageProperties.SpecificationId },
{ "buildProjectId", buildProject.Id },
},
metrics
);
}
if (calculationResultsHaveExceptions)
{
await FailJob(messageProperties.JobId, totalProviderResults, "Exceptions were thrown during generation of calculation results");
}
else
{
await CompleteBatch(messageProperties, cachedCalculationAggregationsBatch, summaries.Count(), totalProviderResults);
}
}
