public async Task GetAssemblyForSpecification_GivenAssemblyFoundonSpecificationAssemblies_AssemblyContentReturnedFromRepository() { _mockCalculationsRepository .GetAssemblyBySpecificationId(Arg.Is(specificationId)) .Returns(MockData.GetMockAssembly()); byte[] assemblyContent = await _assemblyService.GetAssemblyForSpecification(specificationId, eTag); assemblyContent.Should().BeEquivalentTo(MockData.GetMockAssembly()); }
public async Task <byte[]> GetAssemblyForSpecification(string specificationId, string etag) { if (etag.IsNotNullOrWhitespace()) { Stream cachedAssembly = await _specificationAssemblies.GetAssembly(specificationId, etag); if (cachedAssembly != null) { return(cachedAssembly.ReadAllBytes()); } } byte[] assembly = await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsRepository.GetAssemblyBySpecificationId(specificationId)); if (assembly == null) { string error = $"Failed to get assembly for specification Id '{specificationId}'"; _logger.Error(error); throw new RetriableException(error); } await _specificationAssemblies.SetAssembly(specificationId, new MemoryStream(assembly)); return(assembly); }
public async Task <GherkinParseResult> Parse(string specificationId, string gherkin, BuildProject buildProject) { Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId)); Guard.IsNullOrWhiteSpace(gherkin, nameof(gherkin)); Guard.ArgumentNotNull(buildProject, nameof(buildProject)); buildProject.Build.Assembly = await _calculationsRepository.GetAssemblyBySpecificationId(specificationId); GherkinParseResult result = new GherkinParseResult(); Parser parser = new Parser(); try { StringBuilder builder = new StringBuilder(); builder.AppendLine("Feature: Feature Wrapper"); builder.AppendLine(" Scenario: Scenario Wrapper"); builder.Append(gherkin); using (StringReader reader = new StringReader(builder.ToString())) { GherkinDocument document = null; try { document = parser.Parse(reader); } catch (InvalidOperationException ex) { string buildProjectId = buildProject.Id; _logger.Error(ex, $"Gherkin parser error for build project {{buildProjectId}}: {builder.ToString()}", buildProjectId); throw; } if (document.Feature?.Children != null) { foreach (ScenarioDefinition scenario in document.Feature?.Children) { if (!scenario.Steps.IsNullOrEmpty()) { foreach (Step step in scenario.Steps) { IEnumerable <KeyValuePair <StepType, string> > expression = stepExpressions.Where(m => Regex.IsMatch(step.Text, m.Value, RegexOptions.IgnoreCase)); if (expression.Any()) { IStepParser stepParser = _stepParserFactory.GetStepParser(expression.First().Key); if (stepParser == null) { result.AddError("The supplied gherkin could not be parsed", step.Location.Line, step.Location.Column); } else { await stepParser.Parse(step, expression.First().Value, result, buildProject); } } else { result.AddError("The supplied gherkin could not be parsed", step.Location.Line, step.Location.Column); } string keyword = step.Keyword?.ToLowerInvariant().Trim(); } } else { result.AddError("The supplied gherkin could not be parsed", 0, 0); } } } } } catch (CompositeParserException exception) { foreach (ParserException error in exception.Errors) { result.AddError(error.Message, error.Location.Line, error.Location.Column); } } return(result); }
public async Task Execute_WhenFieldNameCaseIsDifferent_ThenTestIsSuccessfullyExecuted() { // Arrange string dataSetName = "Test Dataset"; string fieldName = "URN"; string calcName = "Test Calc"; string gherkin = $"Given the dataset '{dataSetName}' field '{fieldName.ToLower()}' is equal to '100050'\n\nThen the result for '{calcName}' is greater than '12' "; ICodeMetadataGeneratorService codeMetadataGeneratorService = CreateCodeMetadataGeneratorService(); codeMetadataGeneratorService .GetTypeInformation(Arg.Any <byte[]>()) .Returns(new List <TypeInformation> { new TypeInformation { Type = "Calculations", Methods = new List <MethodInformation> { new MethodInformation { FriendlyName = calcName } } }, new TypeInformation { Type = "Datasets", Properties = new List <PropertyInformation> { new PropertyInformation { FriendlyName = dataSetName, Type = "DSType" } } }, new TypeInformation { Type = "DSType", Properties = new List <PropertyInformation> { new PropertyInformation { FriendlyName = fieldName, Type = "String" } } } }); IProviderResultsRepository providerResultsRepository = CreateProviderResultsRepository(); ITestRunnerResiliencePolicies resiliencePolicies = CreateResiliencePolicies(); IStepParserFactory stepParserFactory = new StepParserFactory(codeMetadataGeneratorService, providerResultsRepository, resiliencePolicies); ICalculationsRepository calculationsRepository = CreateCalculationsRepository(); calculationsRepository .GetAssemblyBySpecificationId(Arg.Is("spec1")) .Returns(new byte[1]); ILogger logger = CreateLogger(); GherkinParser gherkinParser = new GherkinParser(stepParserFactory, calculationsRepository, logger); ICacheProvider cacheProvider = CreateCacheProvider(); GherkinExecutor gherkinExecutor = CreateGherkinExecutor(gherkinParser, cacheProvider); ProviderResult providerResult = new ProviderResult { Provider = new ProviderSummary { Id = "prov1" }, CalculationResults = new List <CalculationResult> { new CalculationResult { Calculation = new Common.Models.Reference { Name = calcName }, Value = 14 } } }; IEnumerable <ProviderSourceDataset> datasets = new List <ProviderSourceDataset> { new ProviderSourceDataset { DataRelationship = new Common.Models.Reference { Name = dataSetName }, Current = new ProviderSourceDatasetVersion { Rows = new List <Dictionary <string, object> > { new Dictionary <string, object> { { fieldName, 100050 } } } } } }; IEnumerable <TestScenario> testScenarios = new List <TestScenario> { new TestScenario { Id = "ts1", Name = "Test Scenario 1", SpecificationId = "spec1", Current = new TestScenarioVersion { Gherkin = gherkin } } }; BuildProject buildProject = new BuildProject { Build = new Build() }; // Act IEnumerable <ScenarioResult> scenarioResults = await gherkinExecutor.Execute(providerResult, datasets, testScenarios, buildProject); // Assert scenarioResults .Should() .HaveCount(1); scenarioResults .First().HasErrors .Should() .BeFalse("there should be no errors"); scenarioResults .First().StepsExecuted .Should() .Be(scenarioResults.First().TotalSteps, "all steps should be executed"); }
public async Task RunTests(Message message) { Stopwatch runTestsStopWatch = Stopwatch.StartNew(); string specificationId = message.UserProperties["specificationId"].ToString(); if (string.IsNullOrWhiteSpace(specificationId)) { _logger.Error("Null or empty specification id provided"); return; } BuildProject buildProject = await _builProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectRepository.GetBuildProjectBySpecificationId(specificationId)); if (buildProject == null) { _logger.Error("A null build project was provided to UpdateAllocations"); throw new ArgumentNullException(nameof(buildProject)); } string cacheKey = message.UserProperties["providerResultsCacheKey"].ToString(); if (string.IsNullOrWhiteSpace(cacheKey)) { _logger.Error("Null or empty cache key provided"); return; } Stopwatch providerResultsQueryStopwatch = Stopwatch.StartNew(); IEnumerable <ProviderResult> providerResults = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <List <ProviderResult> >($"{CacheKeys.ProviderResultBatch}{cacheKey}")); providerResultsQueryStopwatch.Stop(); if (providerResults.IsNullOrEmpty()) { _logger.Error($"No provider results found in cache for key: {cacheKey}"); return; } await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.RemoveAsync <List <ProviderResult> >($"{CacheKeys.ProviderResultBatch}{cacheKey}")); Stopwatch testScenariosStopwatch = Stopwatch.StartNew(); IEnumerable <TestScenario> testScenarios = await _scenariosRepositoryPolicy.ExecuteAsync(() => _scenariosRepository.GetTestScenariosBySpecificationId(specificationId)); testScenariosStopwatch.Stop(); if (testScenarios.IsNullOrEmpty()) { _logger.Warning($"No test scenarios found for specification id: {specificationId}"); return; } Stopwatch specificationLookupStopwatch = Stopwatch.StartNew(); SpecificationSummary specification = await _specificationRepositoryPolicy.ExecuteAsync(() => _specificationRepository.GetSpecificationSummaryById(specificationId)); specificationLookupStopwatch.Stop(); if (specification == null) { _logger.Error($"No specification found for specification id: {specificationId}"); return; } IEnumerable <string> providerIds = providerResults.Select(m => m.Provider.Id); Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew(); IEnumerable <ProviderSourceDataset> sourceDatasets = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIds, specificationId)); providerSourceDatasetsStopwatch.Stop(); if (sourceDatasets.IsNullOrEmpty()) { _logger.Error($"No source datasets found for specification id: {specificationId}"); return; } byte[] assembly = await _calculationsRepository.GetAssemblyBySpecificationId(specificationId); if (assembly.IsNullOrEmpty()) { _logger.Error($"No assemblyfor specification id: {specificationId}"); return; } buildProject.Build.Assembly = assembly; Stopwatch existingTestResultsStopwatch = Stopwatch.StartNew(); IEnumerable <TestScenarioResult> testScenarioResults = await _testResultsRepositoryPolicy.ExecuteAsync(() => _testResultsRepository.GetCurrentTestResults(providerIds, specificationId)); existingTestResultsStopwatch.Stop(); Stopwatch runTestsStopwatch = Stopwatch.StartNew(); IEnumerable <TestScenarioResult> results = await _testEngine.RunTests(testScenarios, providerResults, sourceDatasets, testScenarioResults.ToList(), specification, buildProject); runTestsStopwatch.Stop(); Stopwatch saveResultsStopwatch = new Stopwatch(); if (results.Any()) { saveResultsStopwatch.Start(); HttpStatusCode status = await _testResultsService.SaveTestProviderResults(results, providerResults); saveResultsStopwatch.Stop(); if (!status.IsSuccess()) { _logger.Error($"Failed to save test results with status code: {status.ToString()}"); } } runTestsStopWatch.Stop(); IDictionary <string, double> metrics = new Dictionary <string, double>() { { "tests-run-totalMs", runTestsStopWatch.ElapsedMilliseconds }, { "tests-run-testScenarioQueryMs", testScenariosStopwatch.ElapsedMilliseconds }, { "tests-run-numberOfTestScenarios", testScenarios.Count() }, { "tests-run-providersResultsQueryMs", providerResultsQueryStopwatch.ElapsedMilliseconds }, { "tests-run-totalProvidersProcessed", providerIds.Count() }, { "tests-run-specificationQueryMs", specificationLookupStopwatch.ElapsedMilliseconds }, { "tests-run-providerSourceDatasetsQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds }, { "tests-run-existingTestsQueryMs", existingTestResultsStopwatch.ElapsedMilliseconds }, { "tests-run-existingTestScenarioResultsTotal", testScenarioResults.Count() }, { "tests-run-runTestsMs", runTestsStopwatch.ElapsedMilliseconds }, }; if (results.Any()) { metrics.Add("tests-run-saveTestResultsMs", saveResultsStopwatch.ElapsedMilliseconds); metrics.Add("tests-run-numberOfSavedResults", results.Count()); } _telemetry.TrackEvent("RunTests", new Dictionary <string, string>() { { "specificationId", specificationId }, { "buildProjectId", buildProject.Id }, { "cacheKey", cacheKey }, }, metrics ); }
public async Task GenerateAllocations(Message message) { Guard.ArgumentNotNull(message, nameof(message)); _logger.Information($"Validating new allocations message"); CalculationEngineServiceValidator.ValidateMessage(_logger, message); GenerateAllocationMessageProperties messageProperties = GetMessageProperties(message); JobViewModel job = await AddStartingProcessJobLog(messageProperties.JobId); if (job == null) { return; } messageProperties.GenerateCalculationAggregationsOnly = (job.JobDefinitionId == JobConstants.DefinitionNames.GenerateCalculationAggregationsJob); IEnumerable <ProviderSummary> summaries = null; _logger.Information($"Generating allocations for specification id {messageProperties.SpecificationId}"); BuildProject buildProject = await GetBuildProject(messageProperties.SpecificationId); byte[] assembly = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetAssemblyBySpecificationId(messageProperties.SpecificationId)); if (assembly == null) { string error = $"Failed to get assembly for specification Id '{messageProperties.SpecificationId}'"; _logger.Error(error); throw new RetriableException(error); } buildProject.Build.Assembly = assembly; Dictionary <string, List <decimal> > cachedCalculationAggregationsBatch = CreateCalculationAggregateBatchDictionary(messageProperties); _logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}"); int start = messageProperties.PartitionIndex; int stop = start + messageProperties.PartitionSize - 1; summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop)); int providerBatchSize = _engineSettings.ProviderBatchSize; Stopwatch calculationsLookupStopwatch = Stopwatch.StartNew(); IEnumerable <CalculationSummaryModel> calculations = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetCalculationSummariesForSpecification(messageProperties.SpecificationId)); if (calculations == null) { _logger.Error($"Calculations lookup API returned null for specification id {messageProperties.SpecificationId}"); throw new InvalidOperationException("Calculations lookup API returned null"); } calculationsLookupStopwatch.Stop(); IEnumerable <CalculationAggregation> aggregations = await BuildAggregations(messageProperties); int totalProviderResults = 0; bool calculationResultsHaveExceptions = false; for (int i = 0; i < summaries.Count(); i += providerBatchSize) { Stopwatch calculationStopwatch = new Stopwatch(); Stopwatch providerSourceDatasetsStopwatch = new Stopwatch(); Stopwatch calcTiming = Stopwatch.StartNew(); CalculationResultsModel calculationResults = await CalculateResults(summaries, calculations, aggregations, buildProject, messageProperties, providerBatchSize, i, providerSourceDatasetsStopwatch, calculationStopwatch); _logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}"); long saveCosmosElapsedMs = -1; long saveSearchElapsedMs = -1; long saveRedisElapsedMs = 0; long saveQueueElapsedMs = 0; if (calculationResults.ProviderResults.Any()) { if (messageProperties.GenerateCalculationAggregationsOnly) { PopulateCachedCalculationAggregationsBatch(calculationResults.ProviderResults, cachedCalculationAggregationsBatch, messageProperties); } else { (long saveCosmosElapsedMs, long saveSearchElapsedMs, long saveRedisElapsedMs, long saveQueueElapsedMs)timingMetrics = await ProcessProviderResults(calculationResults.ProviderResults, messageProperties, message); saveCosmosElapsedMs = timingMetrics.saveCosmosElapsedMs; saveSearchElapsedMs = timingMetrics.saveSearchElapsedMs; saveRedisElapsedMs = timingMetrics.saveRedisElapsedMs; saveQueueElapsedMs = timingMetrics.saveQueueElapsedMs; totalProviderResults += calculationResults.ProviderResults.Count(); if (calculationResults.ResultsContainExceptions) { if (!calculationResultsHaveExceptions) { calculationResultsHaveExceptions = true; } } } } calcTiming.Stop(); IDictionary <string, double> metrics = new Dictionary <string, double>() { { "calculation-run-providersProcessed", calculationResults.PartitionedSummaries.Count() }, { "calculation-run-lookupCalculationDefinitionsMs", calculationsLookupStopwatch.ElapsedMilliseconds }, { "calculation-run-providersResultsFromCache", summaries.Count() }, { "calculation-run-partitionSize", messageProperties.PartitionSize }, { "calculation-run-providerSourceDatasetQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds }, { "calculation-run-saveProviderResultsRedisMs", saveRedisElapsedMs }, { "calculation-run-saveProviderResultsServiceBusMs", saveQueueElapsedMs }, { "calculation-run-runningCalculationMs", calculationStopwatch.ElapsedMilliseconds }, }; if (saveCosmosElapsedMs > -1) { metrics.Add("calculation-run-elapsedMilliseconds", calcTiming.ElapsedMilliseconds); metrics.Add("calculation-run-saveProviderResultsCosmosMs", saveCosmosElapsedMs); metrics.Add("calculation-run-saveProviderResultsSearchMs", saveSearchElapsedMs); } else { metrics.Add("calculation-run-for-tests-ms", calcTiming.ElapsedMilliseconds); } _telemetry.TrackEvent("CalculationRunProvidersProcessed", new Dictionary <string, string>() { { "specificationId", messageProperties.SpecificationId }, { "buildProjectId", buildProject.Id }, }, metrics ); } if (calculationResultsHaveExceptions) { await FailJob(messageProperties.JobId, totalProviderResults, "Exceptions were thrown during generation of calculation results"); } else { await CompleteBatch(messageProperties, cachedCalculationAggregationsBatch, summaries.Count(), totalProviderResults); } }