public async Task <(long saveToCosmosElapsedMs, long saveToSearchElapsedMs)> SaveProviderResults(IEnumerable <ProviderResult> providerResults, int degreeOfParallelism = 5)
{
if (providerResults == null || providerResults.Count() == 0)
{
return(0, 0);
}
IEnumerable <KeyValuePair <string, ProviderResult> > results = providerResults.Select(m => new KeyValuePair <string, ProviderResult>(m.Provider.Id, m));
IEnumerable <string> specificationIds = providerResults.Select(s => s.SpecificationId).Distinct();
Dictionary <string, SpecificationSummary> specifications = new Dictionary <string, SpecificationSummary>();
foreach (string specificationId in specificationIds)
{
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(specificationId);
if (specification == null)
{
throw new InvalidOperationException($"Result for Specification Summary lookup was null with ID '{specificationId}'");
}
specifications.Add(specificationId, specification);
}
Task <long> cosmosSaveTask = BulkSaveProviderResults(results, degreeOfParallelism);
Task <long> searchSaveTask = UpdateSearch(providerResults, specifications);
await TaskHelper.WhenAllAndThrow(cosmosSaveTask, searchSaveTask);
return(cosmosSaveTask.Result, searchSaveTask.Result);
}
public async Task <IActionResult> ReIndex(HttpRequest request)
{
IEnumerable <DocumentEntity <TestScenario> > testScenarios = await _scenariosRepository.GetAllTestScenarios();
List <ScenarioIndex> testScenarioIndexes = new List <ScenarioIndex>();
Dictionary <string, Models.Specs.SpecificationSummary> specifications = new Dictionary <string, Models.Specs.SpecificationSummary>();
foreach (DocumentEntity <TestScenario> entity in testScenarios)
{
TestScenario testScenario = entity.Content;
Models.Specs.SpecificationSummary specificationSummary = null;
if (!specifications.ContainsKey(testScenario.SpecificationId))
{
specificationSummary = await _specificationsRepository.GetSpecificationSummaryById(testScenario.SpecificationId);
specifications.Add(testScenario.SpecificationId, specificationSummary);
}
else
{
specificationSummary = specifications[testScenario.SpecificationId];
}
testScenarioIndexes.Add(new ScenarioIndex()
{
Id = testScenario.Id,
Name = testScenario.Name,
Description = testScenario.Current.Description,
LastUpdatedDate = entity.UpdatedAt,
FundingStreamIds = testScenario.Current?.FundingStreamIds.ToArray(),
FundingStreamNames = specificationSummary.FundingStreams.Select(s => s.Name).ToArray(),
FundingPeriodId = testScenario.Current?.FundingPeriodId,
FundingPeriodName = specificationSummary.FundingPeriod.Name,
SpecificationId = testScenario.SpecificationId,
SpecificationName = specificationSummary.Name,
Status = Enum.GetName(typeof(PublishStatus), testScenario.Current.PublishStatus),
});
}
await _searchRepository.Index(testScenarioIndexes);
return(new OkObjectResult($"Updated {testScenarioIndexes.Count} records"));
}
public async Task <IActionResult> SaveVersion(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
CreateNewTestScenarioVersion scenarioVersion = JsonConvert.DeserializeObject <CreateNewTestScenarioVersion>(json);
if (scenarioVersion == null)
{
_logger.Error("A null scenario version was provided");
return(new BadRequestObjectResult("Null or empty calculation Id provided"));
}
BadRequestObjectResult validationResult = (await _createNewTestScenarioVersionValidator.ValidateAsync(scenarioVersion)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
TestScenario testScenario = null;
if (!string.IsNullOrEmpty(scenarioVersion.Id))
{
testScenario = await _scenariosRepository.GetTestScenarioById(scenarioVersion.Id);
}
bool saveAsVersion = true;
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(scenarioVersion.SpecificationId);
if (specification == null)
{
_logger.Error($"Unable to find a specification for specification id : {scenarioVersion.SpecificationId}");
return(new StatusCodeResult(412));
}
Reference user = request.GetUserOrDefault();
if (testScenario == null)
{
string Id = Guid.NewGuid().ToString();
testScenario = new TestScenario
{
Id = Id,
SpecificationId = specification.Id,
Name = scenarioVersion.Name,
Current = new TestScenarioVersion
{
Date = DateTimeOffset.Now.ToLocalTime(),
TestScenarioId = Id,
PublishStatus = PublishStatus.Draft,
Version = 1,
Author = user,
Gherkin = scenarioVersion.Scenario,
Description = scenarioVersion.Description,
FundingPeriodId = specification.FundingPeriod.Id,
FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(),
}
};
}
else
{
testScenario.Name = scenarioVersion.Name;
saveAsVersion = !string.Equals(scenarioVersion.Scenario, testScenario.Current.Gherkin) ||
scenarioVersion.Description != testScenario.Current.Description;
TestScenarioVersion newVersion = testScenario.Current.Clone() as TestScenarioVersion;
if (saveAsVersion == true)
{
newVersion.Author = user;
newVersion.Gherkin = scenarioVersion.Scenario;
newVersion.Description = scenarioVersion.Description;
newVersion.FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe();
newVersion.FundingPeriodId = specification.FundingPeriod.Id;
newVersion = await _versionRepository.CreateVersion(newVersion, testScenario.Current);
testScenario.Current = newVersion;
}
}
HttpStatusCode statusCode = await _scenariosRepository.SaveTestScenario(testScenario);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save test scenario with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
await _versionRepository.SaveVersion(testScenario.Current);
ScenarioIndex scenarioIndex = CreateScenarioIndexFromScenario(testScenario, specification);
await _searchRepository.Index(new List <ScenarioIndex> {
scenarioIndex
});
await _cacheProvider.RemoveAsync <List <TestScenario> >($"{CacheKeys.TestScenarios}{testScenario.SpecificationId}");
await _cacheProvider.RemoveAsync <GherkinParseResult>($"{CacheKeys.GherkinParseResult}{testScenario.Id}");
IEnumerable <Models.Calcs.CalculationCurrentVersion> calculations = await _calcsRepositoryPolicy.ExecuteAsync(() => _calcsRepository.GetCurrentCalculationsBySpecificationId(specification.Id));
if (calculations.IsNullOrEmpty())
{
_logger.Information($"No calculations found to test for specification id: '{specification.Id}'");
}
else
{
string correlationId = request.GetCorrelationId();
try
{
Trigger trigger = new Trigger
{
EntityId = testScenario.Id,
EntityType = nameof(TestScenario),
Message = $"Saving test scenario: '{testScenario.Id}'"
};
bool generateCalculationAggregations = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(calculations.Select(m => m.SourceCode));
Job job = await SendInstructAllocationsToJobService(specification.Id, user, trigger, correlationId, generateCalculationAggregations);
_logger.Information($"New job of type '{job.JobDefinitionId}' created with id: '{job.Id}'");
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specification.Id}'");
return(new InternalServerErrorResult($"An error occurred attempting to execute calculations prior to running tests on specification '{specification.Id}'"));
}
}
CurrentTestScenario testScenarioResult = await _scenariosRepository.GetCurrentTestScenarioById(testScenario.Id);
return(new OkObjectResult(testScenarioResult));
}
public async Task SaveProviderResults_WhenExcludedResultsAndIsNewProviderCalculationResultsIndexEnabled_ThenResultsSavedToCosmosSavesNull()
{
// Arrange
ICosmosRepository cosmosRepository = CreateCosmosRepository();
ISearchRepository <ProviderCalculationResultsIndex> searchRepository = CreateProviderCalculationResultsSearchRepository();
ISpecificationsRepository specificationsRepository = CreateSpecificationsRepository();
IFeatureToggle featureToggle = CreateFeatureToggle();
featureToggle
.IsNewProviderCalculationResultsIndexEnabled()
.Returns(true);
ProviderResultsRepository repo = CreateProviderResultsRepository(cosmosRepository, specificationsRepository: specificationsRepository, providerCalculationResultsSearchRepository: searchRepository, featureToggle: featureToggle);
specificationsRepository.GetSpecificationSummaryById(Arg.Any <string>()).Returns(new SpecificationSummary {
Name = "Specification 1"
});
IEnumerable <ProviderResult> results = new List <ProviderResult>
{
new ProviderResult
{
AllocationLineResults = new List <AllocationLineResult>
{
new AllocationLineResult
{
AllocationLine = new Reference {
Id = "alloc 1", Name = "Allocation one"
},
Value = 1112.3M
}
},
CalculationResults = new List <CalculationResult>
{
new CalculationResult
{
AllocationLine = new Reference {
Id = "alloc1", Name = "Allocation one"
},
Calculation = new Reference {
Id = "calc1", Name = "calculation one"
},
CalculationSpecification = new Reference {
Id = "calc1", Name = "calculation one"
},
CalculationType = Models.Calcs.CalculationType.Funding,
Value = null
}
},
Id = Guid.NewGuid().ToString(),
Provider = new ProviderSummary
{
Id = "prov1",
Name = "Provider 1",
ProviderType = "TYpe 1",
ProviderSubType = "Sub type 1",
Authority = "Authority",
UKPRN = "ukprn123",
URN = "urn123",
EstablishmentNumber = "en123",
UPIN = "upin123",
DateOpened = DateTime.Now
},
SpecificationId = "spec1"
}
};
// Act
await repo.SaveProviderResults(results);
// Assert
await cosmosRepository.Received().BulkUpsertAsync(Arg.Is <IEnumerable <KeyValuePair <string, ProviderResult> > >(r => r.Count() == 1),
Arg.Any <int>(),
Arg.Any <bool>(),
Arg.Is <bool>(false));
await searchRepository.Received(1).Index(Arg.Is <IEnumerable <ProviderCalculationResultsIndex> >(r =>
r.First().SpecificationId == results.First().SpecificationId&&
r.First().SpecificationName == "Specification 1" &&
r.First().CalculationId.Any() &&
r.First().CalculationId.First() == results.First().CalculationResults.First().Calculation.Id&&
r.First().CalculationName.Any() &&
r.First().CalculationName.First() == results.First().CalculationResults.First().Calculation.Name&&
r.First().ProviderId == results.First().Provider.Id&&
r.First().ProviderName == results.First().Provider.Name&&
r.First().ProviderType == results.First().Provider.ProviderType&&
r.First().ProviderSubType == results.First().Provider.ProviderSubType&&
r.First().LocalAuthority == results.First().Provider.Authority&&
r.First().UKPRN == results.First().Provider.UKPRN&&
r.First().URN == results.First().Provider.URN&&
r.First().UPIN == results.First().Provider.UPIN&&
r.First().EstablishmentNumber == results.First().Provider.EstablishmentNumber&&
r.First().OpenDate == results.First().Provider.DateOpened&&
r.First().CalculationResult.Any() &&
r.First().CalculationResult.First() == "null"));
}
public async Task SaveProviderResults_WhenResults_ThenResultsSavedToCosmos()
{
// Arrange
ICosmosRepository cosmosRepository = CreateCosmosRepository();
ISearchRepository <CalculationProviderResultsIndex> searchRepository = CreateCalculationProviderResultsSearchRepository();
ISpecificationsRepository specificationsRepository = CreateSpecificationsRepository();
ProviderResultsRepository repo = CreateProviderResultsRepository(cosmosRepository, searchRepository, specificationsRepository);
specificationsRepository.GetSpecificationSummaryById(Arg.Any <string>()).Returns(new SpecificationSummary {
Name = "Specification 1"
});
IEnumerable <ProviderResult> results = new List <ProviderResult>
{
new ProviderResult
{
AllocationLineResults = new List <AllocationLineResult>
{
new AllocationLineResult
{
AllocationLine = new Reference {
Id = "alloc 1", Name = "Allocation one"
},
Value = 1112.3M
}
},
CalculationResults = new List <CalculationResult>
{
new CalculationResult
{
AllocationLine = new Reference {
Id = "alloc1", Name = "Allocation one"
},
Calculation = new Reference {
Id = "calc1", Name = "calculation one"
},
CalculationSpecification = new Reference {
Id = "calc1", Name = "calculation one"
},
CalculationType = Models.Calcs.CalculationType.Funding,
Value = 1112.3M
}
},
Id = Guid.NewGuid().ToString(),
Provider = new ProviderSummary
{
Id = "prov1",
Name = "Provider 1",
ProviderType = "TYpe 1",
ProviderSubType = "Sub type 1",
Authority = "Authority",
UKPRN = "ukprn123",
URN = "urn123",
EstablishmentNumber = "en123",
UPIN = "upin123",
DateOpened = DateTime.Now
},
SpecificationId = "spec1"
}
};
// Act
await repo.SaveProviderResults(results);
// Assert
await cosmosRepository.Received().BulkUpsertAsync(Arg.Is <IEnumerable <KeyValuePair <string, ProviderResult> > >(r => r.Count() == 1),
Arg.Any <int>(),
Arg.Any <bool>(),
Arg.Is <bool>(false));
}
public async Task SaveProviderResults_WhenResults_ThenResultsSavedToSearch()
{
// Arrange
ICosmosRepository cosmosRepository = CreateCosmosRepository();
ISearchRepository <CalculationProviderResultsIndex> searchRepository = CreateCalculationProviderResultsSearchRepository();
ISpecificationsRepository specificationsRepository = CreateSpecificationsRepository();
ProviderResultsRepository repo = CreateProviderResultsRepository(cosmosRepository, searchRepository, specificationsRepository);
specificationsRepository.GetSpecificationSummaryById(Arg.Any <string>()).Returns(new SpecificationSummary {
Name = "Specification 1"
});
IEnumerable <ProviderResult> results = new List <ProviderResult>
{
new ProviderResult
{
AllocationLineResults = new List <AllocationLineResult>
{
new AllocationLineResult
{
AllocationLine = new Reference {
Id = "alloc 1", Name = "Allocation one"
},
Value = 1112.3M
}
},
CalculationResults = new List <CalculationResult>
{
new CalculationResult
{
AllocationLine = new Reference {
Id = "alloc1", Name = "Allocation one"
},
Calculation = new Reference {
Id = "calc1", Name = "calculation one"
},
CalculationSpecification = new Reference {
Id = "calc1", Name = "calculation one"
},
CalculationType = Models.Calcs.CalculationType.Funding,
Value = 1112.3M
}
},
Id = Guid.NewGuid().ToString(),
Provider = new ProviderSummary
{
Id = "prov1",
Name = "Provider 1",
ProviderType = "TYpe 1",
ProviderSubType = "Sub type 1",
Authority = "Authority",
UKPRN = "ukprn123",
URN = "urn123",
EstablishmentNumber = "en123",
UPIN = "upin123",
DateOpened = DateTime.Now
},
SpecificationId = "spec1"
}
};
// Act
await repo.SaveProviderResults(results);
// Assert
await searchRepository.Received(1).Index(Arg.Is <IEnumerable <CalculationProviderResultsIndex> >(r => r.Count() == 1));
await searchRepository.Received(1).Index(Arg.Is <IEnumerable <CalculationProviderResultsIndex> >(r =>
r.First().SpecificationId == results.First().SpecificationId&&
r.First().SpecificationName == "Specification 1" &&
r.First().CalculationSpecificationId == results.First().CalculationResults.First().CalculationSpecification.Id&&
r.First().CalculationSpecificationName == results.First().CalculationResults.First().CalculationSpecification.Name&&
r.First().CalculationName == results.First().CalculationResults.First().Calculation.Name&&
r.First().CalculationId == results.First().CalculationResults.First().Calculation.Id&&
r.First().CalculationType == results.First().CalculationResults.First().CalculationType.ToString() &&
r.First().ProviderId == results.First().Provider.Id&&
r.First().ProviderName == results.First().Provider.Name&&
r.First().ProviderType == results.First().Provider.ProviderType&&
r.First().ProviderSubType == results.First().Provider.ProviderSubType&&
r.First().LocalAuthority == results.First().Provider.Authority&&
r.First().UKPRN == results.First().Provider.UKPRN&&
r.First().URN == results.First().Provider.URN&&
r.First().UPIN == results.First().Provider.UPIN&&
r.First().EstablishmentNumber == results.First().Provider.EstablishmentNumber&&
r.First().OpenDate == results.First().Provider.DateOpened&&
r.First().CalculationResult == Convert.ToDouble(results.First().CalculationResults.First().Value) &&
r.First().IsExcluded == false));
}
public async Task <IActionResult> CreateRelationship(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json);
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId);
if (specification == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = request.BuildMessageProperties();
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}
public async Task <IActionResult> ReIndexCalculationProviderResults()
{
IEnumerable <DocumentEntity <ProviderResult> > providerResults = await _resultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.GetAllProviderResults());
IList <ProviderCalculationResultsIndex> searchItems = new List <ProviderCalculationResultsIndex>();
Dictionary <string, SpecificationSummary> specifications = new Dictionary <string, SpecificationSummary>();
foreach (DocumentEntity <ProviderResult> documentEntity in providerResults)
{
ProviderResult providerResult = documentEntity.Content;
foreach (CalculationResult calculationResult in providerResult.CalculationResults)
{
SpecificationSummary specificationSummary = null;
if (!specifications.ContainsKey(providerResult.SpecificationId))
{
specificationSummary = await _specificationsRepositoryPolicy.ExecuteAsync(() => _specificationsRepository.GetSpecificationSummaryById(providerResult.SpecificationId));
if (specificationSummary == null)
{
throw new InvalidOperationException($"Specification Summary returned null for specification ID '{providerResult.SpecificationId}'");
}
specifications.Add(providerResult.SpecificationId, specificationSummary);
}
else
{
specificationSummary = specifications[providerResult.SpecificationId];
}
ProviderCalculationResultsIndex searchItem = new ProviderCalculationResultsIndex
{
SpecificationId = providerResult.SpecificationId,
SpecificationName = specificationSummary?.Name,
CalculationName = providerResult.CalculationResults.Select(x => x.Calculation.Name).ToArraySafe(),
CalculationId = providerResult.CalculationResults.Select(x => x.Calculation.Id).ToArraySafe(),
ProviderId = providerResult.Provider.Id,
ProviderName = providerResult.Provider.Name,
ProviderType = providerResult.Provider.ProviderType,
ProviderSubType = providerResult.Provider.ProviderSubType,
LocalAuthority = providerResult.Provider.Authority,
LastUpdatedDate = documentEntity.UpdatedAt,
UKPRN = providerResult.Provider.UKPRN,
URN = providerResult.Provider.URN,
UPIN = providerResult.Provider.UPIN,
EstablishmentNumber = providerResult.Provider.EstablishmentNumber,
OpenDate = providerResult.Provider.DateOpened,
CalculationResult = providerResult.CalculationResults.Select(m => m.Value.HasValue ? m.Value.ToString() : "null").ToArraySafe()
};
if (_featureToggle.IsExceptionMessagesEnabled())
{
searchItem.CalculationException = providerResult.CalculationResults
.Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType))
.Select(e => e.Calculation.Id)
.ToArraySafe();
searchItem.CalculationExceptionType = providerResult.CalculationResults
.Select(m => m.ExceptionType ?? string.Empty)
.ToArraySafe();
searchItem.CalculationExceptionMessage = providerResult.CalculationResults
.Select(m => m.ExceptionMessage ?? string.Empty)
.ToArraySafe();
}
searchItems.Add(searchItem);
}
}
const int partitionSize = 500;
for (int i = 0; i < searchItems.Count; i += partitionSize)
{
IEnumerable <ProviderCalculationResultsIndex> partitionedResults = searchItems.Skip(i).Take(partitionSize);
IEnumerable <IndexError> errors = await _resultsSearchRepositoryPolicy.ExecuteAsync(() => _calculationProviderResultsSearchRepository.Index(partitionedResults));
if (errors.Any())
{
_logger.Error($"Failed to index calculation provider result documents with errors: { string.Join(";", errors.Select(m => m.ErrorMessage)) }");
return(new InternalServerErrorResult(null));
}
}
return(new NoContentResult());
}
