public void ValidateMessage_WhenMessageContainsAllComponents_ShouldNotThrowException() { // Arrange ILogger mockLogger = Substitute.For <ILogger>(); const string cacheKey = "Cache-key"; const int partitionIndex = 0; const int partitionSize = 100; Message message = new Message(); IDictionary <string, object> messageUserProperties = message.UserProperties; messageUserProperties.Add(PartitionIndexKey, partitionIndex); messageUserProperties.Add(PartitionSizeKey, partitionSize); messageUserProperties.Add(PartitionCacheKeyKey, cacheKey); EngineSettings mockEngineSettings = Substitute.For <EngineSettings>(); ICalculationsRepository mockCalculationRepository = Substitute.For <ICalculationsRepository>(); ICalculatorResiliencePolicies mockCalculatorResiliencePolicies = Substitute.For <ICalculatorResiliencePolicies>(); IValidator <ICalculatorResiliencePolicies> validator = Substitute.For <IValidator <ICalculatorResiliencePolicies> >(); ValidationResult result = new ValidationResult(); validator.Validate(mockCalculatorResiliencePolicies) .Returns(result); ICalculationEngineServiceValidator calculationEngineServiceValidator = new CalculationEngineServiceValidator(validator, mockEngineSettings, mockCalculatorResiliencePolicies, mockCalculationRepository); // Act, Assert calculationEngineServiceValidator.ValidateMessage(mockLogger, message); }
public void ValidateMessage_WhenPartitionSizeIsLessThanZero_ShouldThrowException() { // Arrange ILogger mockLogger = Substitute.For <ILogger>(); const int partitionSize = -1; const int partitionIndex = 0; const string cacheKey = "Cache-key"; Message message = new Message(); IDictionary <string, object> messageUserProperties = message.UserProperties; messageUserProperties.Add(PartitionIndexKey, partitionIndex); messageUserProperties.Add(PartitionSizeKey, partitionSize); messageUserProperties.Add(PartitionCacheKeyKey, cacheKey); // Act Action validateMethod = () => { CalculationEngineServiceValidator.ValidateMessage(mockLogger, message); }; // Assert validateMethod .Should() .ThrowExactly <KeyNotFoundException>() .And.Message .Should() .BeEquivalentTo(GeneratePartitionSizeErrorMessage(partitionSize)); }
public void ValidateMessage_WhenMessageDoesNotContainProviderCacheKey_ShouldThrowException() { // Arrange ILogger mockLogger = Substitute.For <ILogger>(); const int partitionSize = 100; const int partitionIndex = 0; Message message = new Message(); IDictionary <string, object> messageUserProperties = message.UserProperties; messageUserProperties.Add(PartitionIndexKey, partitionIndex); messageUserProperties.Add(PartitionSizeKey, partitionSize); // Act Action validateMethod = () => { CalculationEngineServiceValidator.ValidateMessage(mockLogger, message); }; // Assert validateMethod .Should() .ThrowExactly <KeyNotFoundException>() .And.Message .Should() .BeEquivalentTo("Provider cache key not found"); }
public void ValidateMessage_WhenMessageContainsAllComponents_ShouldNotThrowException() { // Arrange ILogger mockLogger = Substitute.For <ILogger>(); const string cacheKey = "Cache-key"; const int partitionIndex = 0; const int partitionSize = 100; Message message = new Message(); IDictionary <string, object> messageUserProperties = message.UserProperties; messageUserProperties.Add(PartitionIndexKey, partitionIndex); messageUserProperties.Add(PartitionSizeKey, partitionSize); messageUserProperties.Add(PartitionCacheKeyKey, cacheKey); // Act, Assert CalculationEngineServiceValidator.ValidateMessage(mockLogger, message); }
public void ValidateMessage_WhenPartitionSizeIsLessThanZero_ShouldThrowException() { // Arrange ILogger mockLogger = Substitute.For <ILogger>(); const int partitionSize = -1; const int partitionIndex = 0; const string cacheKey = "Cache-key"; Message message = new Message(); IDictionary <string, object> messageUserProperties = message.UserProperties; messageUserProperties.Add(PartitionIndexKey, partitionIndex); messageUserProperties.Add(PartitionSizeKey, partitionSize); messageUserProperties.Add(PartitionCacheKeyKey, cacheKey); EngineSettings mockEngineSettings = Substitute.For <EngineSettings>(); ICalculationsRepository mockCalculationRepository = Substitute.For <ICalculationsRepository>(); ICalculatorResiliencePolicies mockCalculatorResiliencePolicies = Substitute.For <ICalculatorResiliencePolicies>(); IValidator <ICalculatorResiliencePolicies> validator = Substitute.For <IValidator <ICalculatorResiliencePolicies> >(); ValidationResult result = new ValidationResult(); validator.Validate(mockCalculatorResiliencePolicies) .Returns(result); // Act Action validateMethod = () => { ICalculationEngineServiceValidator calculationEngineServiceValidator = new CalculationEngineServiceValidator(validator, mockEngineSettings, mockCalculatorResiliencePolicies, mockCalculationRepository); calculationEngineServiceValidator.ValidateMessage(mockLogger, message); }; // Assert validateMethod .Should() .ThrowExactly <KeyNotFoundException>() .And.Message .Should() .BeEquivalentTo(GeneratePartitionSizeErrorMessage(partitionSize)); }
public void ValidateMessage_WhenMessageDoesNotContainProviderCacheKey_ShouldThrowException() { // Arrange ILogger mockLogger = Substitute.For <ILogger>(); const int partitionSize = 100; const int partitionIndex = 0; Message message = new Message(); IDictionary <string, object> messageUserProperties = message.UserProperties; messageUserProperties.Add(PartitionIndexKey, partitionIndex); messageUserProperties.Add(PartitionSizeKey, partitionSize); EngineSettings mockEngineSettings = Substitute.For <EngineSettings>(); ICalculationsRepository mockCalculationRepository = Substitute.For <ICalculationsRepository>(); ICalculatorResiliencePolicies mockCalculatorResiliencePolicies = Substitute.For <ICalculatorResiliencePolicies>(); IValidator <ICalculatorResiliencePolicies> validator = Substitute.For <IValidator <ICalculatorResiliencePolicies> >(); ValidationResult result = new ValidationResult(); validator.Validate(mockCalculatorResiliencePolicies) .Returns(result); // Act Action validateMethod = () => { ICalculationEngineServiceValidator calculationEngineServiceValidator = new CalculationEngineServiceValidator(validator, mockEngineSettings, mockCalculatorResiliencePolicies, mockCalculationRepository); calculationEngineServiceValidator.ValidateMessage(mockLogger, message); }; // Assert validateMethod .Should() .ThrowExactly <KeyNotFoundException>() .And.Message .Should() .BeEquivalentTo("Provider cache key not found"); }
public async Task GenerateAllocations(Message message) { Guard.ArgumentNotNull(message, nameof(message)); _logger.Information($"Validating new allocations message"); CalculationEngineServiceValidator.ValidateMessage(_logger, message); GenerateAllocationMessageProperties messageProperties = GetMessageProperties(message); JobViewModel job = await AddStartingProcessJobLog(messageProperties.JobId); if (job == null) { return; } messageProperties.GenerateCalculationAggregationsOnly = (job.JobDefinitionId == JobConstants.DefinitionNames.GenerateCalculationAggregationsJob); IEnumerable <ProviderSummary> summaries = null; _logger.Information($"Generating allocations for specification id {messageProperties.SpecificationId}"); BuildProject buildProject = await GetBuildProject(messageProperties.SpecificationId); byte[] assembly = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetAssemblyBySpecificationId(messageProperties.SpecificationId)); if (assembly == null) { string error = $"Failed to get assembly for specification Id '{messageProperties.SpecificationId}'"; _logger.Error(error); throw new RetriableException(error); } buildProject.Build.Assembly = assembly; Dictionary <string, List <decimal> > cachedCalculationAggregationsBatch = CreateCalculationAggregateBatchDictionary(messageProperties); _logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}"); int start = messageProperties.PartitionIndex; int stop = start + messageProperties.PartitionSize - 1; summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop)); int providerBatchSize = _engineSettings.ProviderBatchSize; Stopwatch calculationsLookupStopwatch = Stopwatch.StartNew(); IEnumerable <CalculationSummaryModel> calculations = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetCalculationSummariesForSpecification(messageProperties.SpecificationId)); if (calculations == null) { _logger.Error($"Calculations lookup API returned null for specification id {messageProperties.SpecificationId}"); throw new InvalidOperationException("Calculations lookup API returned null"); } calculationsLookupStopwatch.Stop(); IEnumerable <CalculationAggregation> aggregations = await BuildAggregations(messageProperties); int totalProviderResults = 0; bool calculationResultsHaveExceptions = false; for (int i = 0; i < summaries.Count(); i += providerBatchSize) { Stopwatch calculationStopwatch = new Stopwatch(); Stopwatch providerSourceDatasetsStopwatch = new Stopwatch(); Stopwatch calcTiming = Stopwatch.StartNew(); CalculationResultsModel calculationResults = await CalculateResults(summaries, calculations, aggregations, buildProject, messageProperties, providerBatchSize, i, providerSourceDatasetsStopwatch, calculationStopwatch); _logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}"); long saveCosmosElapsedMs = -1; long saveSearchElapsedMs = -1; long saveRedisElapsedMs = 0; long saveQueueElapsedMs = 0; if (calculationResults.ProviderResults.Any()) { if (messageProperties.GenerateCalculationAggregationsOnly) { PopulateCachedCalculationAggregationsBatch(calculationResults.ProviderResults, cachedCalculationAggregationsBatch, messageProperties); } else { (long saveCosmosElapsedMs, long saveSearchElapsedMs, long saveRedisElapsedMs, long saveQueueElapsedMs)timingMetrics = await ProcessProviderResults(calculationResults.ProviderResults, messageProperties, message); saveCosmosElapsedMs = timingMetrics.saveCosmosElapsedMs; saveSearchElapsedMs = timingMetrics.saveSearchElapsedMs; saveRedisElapsedMs = timingMetrics.saveRedisElapsedMs; saveQueueElapsedMs = timingMetrics.saveQueueElapsedMs; totalProviderResults += calculationResults.ProviderResults.Count(); if (calculationResults.ResultsContainExceptions) { if (!calculationResultsHaveExceptions) { calculationResultsHaveExceptions = true; } } } } calcTiming.Stop(); IDictionary <string, double> metrics = new Dictionary <string, double>() { { "calculation-run-providersProcessed", calculationResults.PartitionedSummaries.Count() }, { "calculation-run-lookupCalculationDefinitionsMs", calculationsLookupStopwatch.ElapsedMilliseconds }, { "calculation-run-providersResultsFromCache", summaries.Count() }, { "calculation-run-partitionSize", messageProperties.PartitionSize }, { "calculation-run-providerSourceDatasetQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds }, { "calculation-run-saveProviderResultsRedisMs", saveRedisElapsedMs }, { "calculation-run-saveProviderResultsServiceBusMs", saveQueueElapsedMs }, { "calculation-run-runningCalculationMs", calculationStopwatch.ElapsedMilliseconds }, }; if (saveCosmosElapsedMs > -1) { metrics.Add("calculation-run-elapsedMilliseconds", calcTiming.ElapsedMilliseconds); metrics.Add("calculation-run-saveProviderResultsCosmosMs", saveCosmosElapsedMs); metrics.Add("calculation-run-saveProviderResultsSearchMs", saveSearchElapsedMs); } else { metrics.Add("calculation-run-for-tests-ms", calcTiming.ElapsedMilliseconds); } _telemetry.TrackEvent("CalculationRunProvidersProcessed", new Dictionary <string, string>() { { "specificationId", messageProperties.SpecificationId }, { "buildProjectId", buildProject.Id }, }, metrics ); } if (calculationResultsHaveExceptions) { await FailJob(messageProperties.JobId, totalProviderResults, "Exceptions were thrown during generation of calculation results"); } else { await CompleteBatch(messageProperties, cachedCalculationAggregationsBatch, summaries.Count(), totalProviderResults); } }