public async Task GetAssemblyForSpecification_GivenAssemblyFoundonSpecificationAssemblies_AssemblyContentReturnedFromRepository()
{
_mockCalculationsRepository
.GetAssemblyBySpecificationId(Arg.Is(specificationId))
.Returns(MockData.GetMockAssembly());
byte[] assemblyContent = await _assemblyService.GetAssemblyForSpecification(specificationId, eTag);
assemblyContent.Should().BeEquivalentTo(MockData.GetMockAssembly());
}
public async Task <byte[]> GetAssemblyForSpecification(string specificationId, string etag)
{
if (etag.IsNotNullOrWhitespace())
{
Stream cachedAssembly = await _specificationAssemblies.GetAssembly(specificationId, etag);
if (cachedAssembly != null)
{
return(cachedAssembly.ReadAllBytes());
}
}
byte[] assembly =
await _calculationsApiClientPolicy.ExecuteAsync(() =>
_calculationsRepository.GetAssemblyBySpecificationId(specificationId));
if (assembly == null)
{
string error = $"Failed to get assembly for specification Id '{specificationId}'";
_logger.Error(error);
throw new RetriableException(error);
}
await _specificationAssemblies.SetAssembly(specificationId, new MemoryStream(assembly));
return(assembly);
}
public async Task <GherkinParseResult> Parse(string specificationId, string gherkin, BuildProject buildProject)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.IsNullOrWhiteSpace(gherkin, nameof(gherkin));
Guard.ArgumentNotNull(buildProject, nameof(buildProject));
buildProject.Build.Assembly = await _calculationsRepository.GetAssemblyBySpecificationId(specificationId);
GherkinParseResult result = new GherkinParseResult();
Parser parser = new Parser();
try
{
StringBuilder builder = new StringBuilder();
builder.AppendLine("Feature: Feature Wrapper");
builder.AppendLine(" Scenario: Scenario Wrapper");
builder.Append(gherkin);
using (StringReader reader = new StringReader(builder.ToString()))
{
GherkinDocument document = null;
try
{
document = parser.Parse(reader);
}
catch (InvalidOperationException ex)
{
string buildProjectId = buildProject.Id;
_logger.Error(ex, $"Gherkin parser error for build project {{buildProjectId}}: {builder.ToString()}", buildProjectId);
throw;
}
if (document.Feature?.Children != null)
{
foreach (ScenarioDefinition scenario in document.Feature?.Children)
{
if (!scenario.Steps.IsNullOrEmpty())
{
foreach (Step step in scenario.Steps)
{
IEnumerable <KeyValuePair <StepType, string> > expression = stepExpressions.Where(m => Regex.IsMatch(step.Text, m.Value, RegexOptions.IgnoreCase));
if (expression.Any())
{
IStepParser stepParser = _stepParserFactory.GetStepParser(expression.First().Key);
if (stepParser == null)
{
result.AddError("The supplied gherkin could not be parsed", step.Location.Line, step.Location.Column);
}
else
{
await stepParser.Parse(step, expression.First().Value, result, buildProject);
}
}
else
{
result.AddError("The supplied gherkin could not be parsed", step.Location.Line, step.Location.Column);
}
string keyword = step.Keyword?.ToLowerInvariant().Trim();
}
}
else
{
result.AddError("The supplied gherkin could not be parsed", 0, 0);
}
}
}
}
}
catch (CompositeParserException exception)
{
foreach (ParserException error in exception.Errors)
{
result.AddError(error.Message, error.Location.Line, error.Location.Column);
}
}
return(result);
}
public async Task Execute_WhenFieldNameCaseIsDifferent_ThenTestIsSuccessfullyExecuted()
{
// Arrange
string dataSetName = "Test Dataset";
string fieldName = "URN";
string calcName = "Test Calc";
string gherkin = $"Given the dataset '{dataSetName}' field '{fieldName.ToLower()}' is equal to '100050'\n\nThen the result for '{calcName}' is greater than '12' ";
ICodeMetadataGeneratorService codeMetadataGeneratorService = CreateCodeMetadataGeneratorService();
codeMetadataGeneratorService
.GetTypeInformation(Arg.Any <byte[]>())
.Returns(new List <TypeInformation>
{
new TypeInformation {
Type = "Calculations", Methods = new List <MethodInformation> {
new MethodInformation {
FriendlyName = calcName
}
}
},
new TypeInformation {
Type = "Datasets", Properties = new List <PropertyInformation> {
new PropertyInformation {
FriendlyName = dataSetName, Type = "DSType"
}
}
},
new TypeInformation {
Type = "DSType", Properties = new List <PropertyInformation> {
new PropertyInformation {
FriendlyName = fieldName, Type = "String"
}
}
}
});
IProviderResultsRepository providerResultsRepository = CreateProviderResultsRepository();
ITestRunnerResiliencePolicies resiliencePolicies = CreateResiliencePolicies();
IStepParserFactory stepParserFactory = new StepParserFactory(codeMetadataGeneratorService, providerResultsRepository, resiliencePolicies);
ICalculationsRepository calculationsRepository = CreateCalculationsRepository();
calculationsRepository
.GetAssemblyBySpecificationId(Arg.Is("spec1"))
.Returns(new byte[1]);
ILogger logger = CreateLogger();
GherkinParser gherkinParser = new GherkinParser(stepParserFactory, calculationsRepository, logger);
ICacheProvider cacheProvider = CreateCacheProvider();
GherkinExecutor gherkinExecutor = CreateGherkinExecutor(gherkinParser, cacheProvider);
ProviderResult providerResult = new ProviderResult
{
Provider = new ProviderSummary {
Id = "prov1"
},
CalculationResults = new List <CalculationResult>
{
new CalculationResult {
Calculation = new Common.Models.Reference {
Name = calcName
}, Value = 14
}
}
};
IEnumerable <ProviderSourceDataset> datasets = new List <ProviderSourceDataset>
{
new ProviderSourceDataset
{
DataRelationship = new Common.Models.Reference {
Name = dataSetName
},
Current = new ProviderSourceDatasetVersion
{
Rows = new List <Dictionary <string, object> >
{
new Dictionary <string, object> {
{ fieldName, 100050 }
}
}
}
}
};
IEnumerable <TestScenario> testScenarios = new List <TestScenario>
{
new TestScenario {
Id = "ts1", Name = "Test Scenario 1", SpecificationId = "spec1", Current = new TestScenarioVersion {
Gherkin = gherkin
}
}
};
BuildProject buildProject = new BuildProject {
Build = new Build()
};
// Act
IEnumerable <ScenarioResult> scenarioResults = await gherkinExecutor.Execute(providerResult, datasets, testScenarios, buildProject);
// Assert
scenarioResults
.Should()
.HaveCount(1);
scenarioResults
.First().HasErrors
.Should()
.BeFalse("there should be no errors");
scenarioResults
.First().StepsExecuted
.Should()
.Be(scenarioResults.First().TotalSteps, "all steps should be executed");
}
public async Task RunTests(Message message)
{
Stopwatch runTestsStopWatch = Stopwatch.StartNew();
string specificationId = message.UserProperties["specificationId"].ToString();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("Null or empty specification id provided");
return;
}
BuildProject buildProject = await _builProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectRepository.GetBuildProjectBySpecificationId(specificationId));
if (buildProject == null)
{
_logger.Error("A null build project was provided to UpdateAllocations");
throw new ArgumentNullException(nameof(buildProject));
}
string cacheKey = message.UserProperties["providerResultsCacheKey"].ToString();
if (string.IsNullOrWhiteSpace(cacheKey))
{
_logger.Error("Null or empty cache key provided");
return;
}
Stopwatch providerResultsQueryStopwatch = Stopwatch.StartNew();
IEnumerable <ProviderResult> providerResults = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <List <ProviderResult> >($"{CacheKeys.ProviderResultBatch}{cacheKey}"));
providerResultsQueryStopwatch.Stop();
if (providerResults.IsNullOrEmpty())
{
_logger.Error($"No provider results found in cache for key: {cacheKey}");
return;
}
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.RemoveAsync <List <ProviderResult> >($"{CacheKeys.ProviderResultBatch}{cacheKey}"));
Stopwatch testScenariosStopwatch = Stopwatch.StartNew();
IEnumerable <TestScenario> testScenarios = await _scenariosRepositoryPolicy.ExecuteAsync(() => _scenariosRepository.GetTestScenariosBySpecificationId(specificationId));
testScenariosStopwatch.Stop();
if (testScenarios.IsNullOrEmpty())
{
_logger.Warning($"No test scenarios found for specification id: {specificationId}");
return;
}
Stopwatch specificationLookupStopwatch = Stopwatch.StartNew();
SpecificationSummary specification = await _specificationRepositoryPolicy.ExecuteAsync(() => _specificationRepository.GetSpecificationSummaryById(specificationId));
specificationLookupStopwatch.Stop();
if (specification == null)
{
_logger.Error($"No specification found for specification id: {specificationId}");
return;
}
IEnumerable <string> providerIds = providerResults.Select(m => m.Provider.Id);
Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew();
IEnumerable <ProviderSourceDataset> sourceDatasets = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() =>
_providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIds, specificationId));
providerSourceDatasetsStopwatch.Stop();
if (sourceDatasets.IsNullOrEmpty())
{
_logger.Error($"No source datasets found for specification id: {specificationId}");
return;
}
byte[] assembly = await _calculationsRepository.GetAssemblyBySpecificationId(specificationId);
if (assembly.IsNullOrEmpty())
{
_logger.Error($"No assemblyfor specification id: {specificationId}");
return;
}
buildProject.Build.Assembly = assembly;
Stopwatch existingTestResultsStopwatch = Stopwatch.StartNew();
IEnumerable <TestScenarioResult> testScenarioResults = await _testResultsRepositoryPolicy.ExecuteAsync(() => _testResultsRepository.GetCurrentTestResults(providerIds, specificationId));
existingTestResultsStopwatch.Stop();
Stopwatch runTestsStopwatch = Stopwatch.StartNew();
IEnumerable <TestScenarioResult> results = await _testEngine.RunTests(testScenarios, providerResults, sourceDatasets, testScenarioResults.ToList(), specification, buildProject);
runTestsStopwatch.Stop();
Stopwatch saveResultsStopwatch = new Stopwatch();
if (results.Any())
{
saveResultsStopwatch.Start();
HttpStatusCode status = await _testResultsService.SaveTestProviderResults(results, providerResults);
saveResultsStopwatch.Stop();
if (!status.IsSuccess())
{
_logger.Error($"Failed to save test results with status code: {status.ToString()}");
}
}
runTestsStopWatch.Stop();
IDictionary <string, double> metrics = new Dictionary <string, double>()
{
{ "tests-run-totalMs", runTestsStopWatch.ElapsedMilliseconds },
{ "tests-run-testScenarioQueryMs", testScenariosStopwatch.ElapsedMilliseconds },
{ "tests-run-numberOfTestScenarios", testScenarios.Count() },
{ "tests-run-providersResultsQueryMs", providerResultsQueryStopwatch.ElapsedMilliseconds },
{ "tests-run-totalProvidersProcessed", providerIds.Count() },
{ "tests-run-specificationQueryMs", specificationLookupStopwatch.ElapsedMilliseconds },
{ "tests-run-providerSourceDatasetsQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds },
{ "tests-run-existingTestsQueryMs", existingTestResultsStopwatch.ElapsedMilliseconds },
{ "tests-run-existingTestScenarioResultsTotal", testScenarioResults.Count() },
{ "tests-run-runTestsMs", runTestsStopwatch.ElapsedMilliseconds },
};
if (results.Any())
{
metrics.Add("tests-run-saveTestResultsMs", saveResultsStopwatch.ElapsedMilliseconds);
metrics.Add("tests-run-numberOfSavedResults", results.Count());
}
_telemetry.TrackEvent("RunTests",
new Dictionary <string, string>()
{
{ "specificationId", specificationId },
{ "buildProjectId", buildProject.Id },
{ "cacheKey", cacheKey },
},
metrics
);
}
public async Task GenerateAllocations(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
_logger.Information($"Validating new allocations message");
CalculationEngineServiceValidator.ValidateMessage(_logger, message);
GenerateAllocationMessageProperties messageProperties = GetMessageProperties(message);
JobViewModel job = await AddStartingProcessJobLog(messageProperties.JobId);
if (job == null)
{
return;
}
messageProperties.GenerateCalculationAggregationsOnly = (job.JobDefinitionId == JobConstants.DefinitionNames.GenerateCalculationAggregationsJob);
IEnumerable <ProviderSummary> summaries = null;
_logger.Information($"Generating allocations for specification id {messageProperties.SpecificationId}");
BuildProject buildProject = await GetBuildProject(messageProperties.SpecificationId);
byte[] assembly = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetAssemblyBySpecificationId(messageProperties.SpecificationId));
if (assembly == null)
{
string error = $"Failed to get assembly for specification Id '{messageProperties.SpecificationId}'";
_logger.Error(error);
throw new RetriableException(error);
}
buildProject.Build.Assembly = assembly;
Dictionary <string, List <decimal> > cachedCalculationAggregationsBatch = CreateCalculationAggregateBatchDictionary(messageProperties);
_logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}");
int start = messageProperties.PartitionIndex;
int stop = start + messageProperties.PartitionSize - 1;
summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop));
int providerBatchSize = _engineSettings.ProviderBatchSize;
Stopwatch calculationsLookupStopwatch = Stopwatch.StartNew();
IEnumerable <CalculationSummaryModel> calculations = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetCalculationSummariesForSpecification(messageProperties.SpecificationId));
if (calculations == null)
{
_logger.Error($"Calculations lookup API returned null for specification id {messageProperties.SpecificationId}");
throw new InvalidOperationException("Calculations lookup API returned null");
}
calculationsLookupStopwatch.Stop();
IEnumerable <CalculationAggregation> aggregations = await BuildAggregations(messageProperties);
int totalProviderResults = 0;
bool calculationResultsHaveExceptions = false;
for (int i = 0; i < summaries.Count(); i += providerBatchSize)
{
Stopwatch calculationStopwatch = new Stopwatch();
Stopwatch providerSourceDatasetsStopwatch = new Stopwatch();
Stopwatch calcTiming = Stopwatch.StartNew();
CalculationResultsModel calculationResults = await CalculateResults(summaries, calculations, aggregations, buildProject, messageProperties, providerBatchSize, i, providerSourceDatasetsStopwatch, calculationStopwatch);
_logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}");
long saveCosmosElapsedMs = -1;
long saveSearchElapsedMs = -1;
long saveRedisElapsedMs = 0;
long saveQueueElapsedMs = 0;
if (calculationResults.ProviderResults.Any())
{
if (messageProperties.GenerateCalculationAggregationsOnly)
{
PopulateCachedCalculationAggregationsBatch(calculationResults.ProviderResults, cachedCalculationAggregationsBatch, messageProperties);
}
else
{
(long saveCosmosElapsedMs, long saveSearchElapsedMs, long saveRedisElapsedMs, long saveQueueElapsedMs)timingMetrics = await ProcessProviderResults(calculationResults.ProviderResults, messageProperties, message);
saveCosmosElapsedMs = timingMetrics.saveCosmosElapsedMs;
saveSearchElapsedMs = timingMetrics.saveSearchElapsedMs;
saveRedisElapsedMs = timingMetrics.saveRedisElapsedMs;
saveQueueElapsedMs = timingMetrics.saveQueueElapsedMs;
totalProviderResults += calculationResults.ProviderResults.Count();
if (calculationResults.ResultsContainExceptions)
{
if (!calculationResultsHaveExceptions)
{
calculationResultsHaveExceptions = true;
}
}
}
}
calcTiming.Stop();
IDictionary <string, double> metrics = new Dictionary <string, double>()
{
{ "calculation-run-providersProcessed", calculationResults.PartitionedSummaries.Count() },
{ "calculation-run-lookupCalculationDefinitionsMs", calculationsLookupStopwatch.ElapsedMilliseconds },
{ "calculation-run-providersResultsFromCache", summaries.Count() },
{ "calculation-run-partitionSize", messageProperties.PartitionSize },
{ "calculation-run-providerSourceDatasetQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds },
{ "calculation-run-saveProviderResultsRedisMs", saveRedisElapsedMs },
{ "calculation-run-saveProviderResultsServiceBusMs", saveQueueElapsedMs },
{ "calculation-run-runningCalculationMs", calculationStopwatch.ElapsedMilliseconds },
};
if (saveCosmosElapsedMs > -1)
{
metrics.Add("calculation-run-elapsedMilliseconds", calcTiming.ElapsedMilliseconds);
metrics.Add("calculation-run-saveProviderResultsCosmosMs", saveCosmosElapsedMs);
metrics.Add("calculation-run-saveProviderResultsSearchMs", saveSearchElapsedMs);
}
else
{
metrics.Add("calculation-run-for-tests-ms", calcTiming.ElapsedMilliseconds);
}
_telemetry.TrackEvent("CalculationRunProvidersProcessed",
new Dictionary <string, string>()
{
{ "specificationId", messageProperties.SpecificationId },
{ "buildProjectId", buildProject.Id },
},
metrics
);
}
if (calculationResultsHaveExceptions)
{
await FailJob(messageProperties.JobId, totalProviderResults, "Exceptions were thrown during generation of calculation results");
}
else
{
await CompleteBatch(messageProperties, cachedCalculationAggregationsBatch, summaries.Count(), totalProviderResults);
}
}