public async Task UpdateBuildProjectRelationships(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
DatasetRelationshipSummary relationship = message.GetPayloadAsInstanceOf <DatasetRelationshipSummary>();
if (relationship == null)
{
_logger.Error("A null relationship message was provided to UpdateBuildProjectRelationships");
throw new ArgumentNullException(nameof(relationship));
}
if (!message.UserProperties.ContainsKey("specification-id"))
{
_logger.Error("Message properties does not contain a specification id");
throw new KeyNotFoundException("specification-id");
}
string specificationId = message.UserProperties["specification-id"].ToString();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error($"Message does not contain a specification id");
throw new ArgumentNullException(nameof(specificationId));
}
BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId);
if (buildProject.DatasetRelationships == null)
{
buildProject.DatasetRelationships = new List <DatasetRelationshipSummary>();
}
if (!buildProject.DatasetRelationships.Any(m => m.Name == relationship.Name))
{
buildProject.DatasetRelationships.Add(relationship);
IEnumerable <Models.Calcs.Calculation> calculations = await _calculationsRepository.GetCalculationsBySpecificationId(specificationId);
CompilerOptions compilerOptions = await _calculationsRepository.GetCompilerOptions(specificationId);
buildProject.Build = _sourceCodeService.Compile(buildProject, calculations ?? Enumerable.Empty <Models.Calcs.Calculation>(), compilerOptions);
if (!_featureToggle.IsDynamicBuildProjectEnabled())
{
await _buildProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectsRepository.UpdateBuildProject(buildProject));
}
await _sourceCodeService.SaveAssembly(buildProject);
}
}
private static StatementSyntax GetDatasetProperties(DatasetRelationshipSummary datasetRelationship)
{
StringBuilder builder = new StringBuilder();
builder.AppendLine($"<DatasetRelationship(Id := \"{datasetRelationship.Id}\", Name := \"{datasetRelationship.Name}\")>");
builder.AppendLine($"<Field(Id := \"{datasetRelationship.DatasetDefinition.Id}\", Name := \"{datasetRelationship.DatasetDefinition.Name}\")>");
if (!string.IsNullOrWhiteSpace(datasetRelationship?.DatasetDefinition?.Description))
{
builder.AppendLine($"<Description(Description := \"{datasetRelationship.DatasetDefinition.Description?.Replace("\"", "\"\"")}\")>");
}
builder.AppendLine(datasetRelationship.DataGranularity == DataGranularity.SingleRowPerProvider
? $"Public Property {GenerateIdentifier(datasetRelationship.Name)}() As {GenerateIdentifier($"{datasetRelationship.DatasetDefinition.Name}Dataset")}"
: $"Public Property {GenerateIdentifier(datasetRelationship.Name)}() As System.Collections.Generic.List(Of {GenerateIdentifier($"{datasetRelationship.DatasetDefinition.Name}Dataset")})");
SyntaxTree tree = SyntaxFactory.ParseSyntaxTree(builder.ToString());
return(tree.GetRoot().DescendantNodes().OfType <StatementSyntax>()
.FirstOrDefault());
}
public async Task UpdateBuildProjectRelationships()
{
string id = NewRandomString();
string expectedUri = $"update-buildproject-relationships?specificationId={id}";
BuildProject expectedResponse = new BuildProject();
DatasetRelationshipSummary expectedRequest = new DatasetRelationshipSummary();
GivenTheResponse(expectedUri, expectedResponse, HttpMethod.Post);
ApiResponse <BuildProject> apiResponse = await _client.UpdateBuildProjectRelationships(id, expectedRequest);
apiResponse?.StatusCode
.Should()
.Be(HttpStatusCode.OK);
apiResponse
.Content
.Should()
.BeEquivalentTo(expectedResponse);
}
public async Task <IActionResult> UpdateBuildProjectRelationships(HttpRequest request)
{
request.Query.TryGetValue("specificationId", out Microsoft.Extensions.Primitives.StringValues specId);
string specificationId = specId.FirstOrDefault();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("No specification Id was provided to UpdateBuildProjectRelationships");
return(new BadRequestObjectResult("Null or empty specification Id provided"));
}
string json = await request.GetRawBodyStringAsync();
DatasetRelationshipSummary relationship = JsonConvert.DeserializeObject <DatasetRelationshipSummary>(json);
if (relationship == null)
{
_logger.Error("A null relationship message was provided to UpdateBuildProjectRelationships");
return(new BadRequestObjectResult("Null relationship provided"));
}
BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId);
IEnumerable <Models.Calcs.Calculation> calculations = await _calculationsRepository.GetCalculationsBySpecificationId(specificationId);
buildProject.Build = _sourceCodeService.Compile(buildProject, calculations ?? Enumerable.Empty <Models.Calcs.Calculation>());
if (!_featureToggle.IsDynamicBuildProjectEnabled())
{
await _buildProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectsRepository.UpdateBuildProject(buildProject));
}
await _sourceCodeService.SaveAssembly(buildProject);
return(new OkObjectResult(buildProject));
}
private async Task PersistDataset(TableLoadResult loadResult, Dataset dataset, DatasetDefinition datasetDefinition, BuildProject buildProject, string specificationId, string relationshipId, int version, Reference user)
{
IEnumerable <ProviderSummary> providerSummaries = await _providerService.FetchCoreProviderData();
Guard.IsNullOrWhiteSpace(relationshipId, nameof(relationshipId));
IList <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>();
if (buildProject.DatasetRelationships == null)
{
_logger.Error($"No dataset relationships found for build project with id : '{buildProject.Id}' for specification '{specificationId}'");
return;
}
DatasetRelationshipSummary relationshipSummary = buildProject.DatasetRelationships.FirstOrDefault(m => m.Relationship.Id == relationshipId);
if (relationshipSummary == null)
{
_logger.Error($"No dataset relationship found for build project with id : {buildProject.Id} with data definition id {datasetDefinition.Id} and relationshipId '{relationshipId}'");
return;
}
ConcurrentDictionary <string, ProviderSourceDataset> existingCurrent = new ConcurrentDictionary <string, ProviderSourceDataset>();
IEnumerable <ProviderSourceDataset> existingCurrentDatasets = await _providerResultsRepositoryPolicy.ExecuteAsync(() =>
_providersResultsRepository.GetCurrentProviderSourceDatasets(specificationId, relationshipId));
if (existingCurrentDatasets.AnyWithNullCheck())
{
foreach (ProviderSourceDataset currentDataset in existingCurrentDatasets)
{
existingCurrent.TryAdd(currentDataset.ProviderId, currentDataset);
}
}
ConcurrentDictionary <string, ProviderSourceDataset> resultsByProviderId = new ConcurrentDictionary <string, ProviderSourceDataset>();
ConcurrentDictionary <string, ProviderSourceDataset> updateCurrentDatasets = new ConcurrentDictionary <string, ProviderSourceDataset>();
Parallel.ForEach(loadResult.Rows, (RowLoadResult row) =>
{
IEnumerable <string> allProviderIds = GetProviderIdsForIdentifier(datasetDefinition, row, providerSummaries);
foreach (string providerId in allProviderIds)
{
if (!resultsByProviderId.TryGetValue(providerId, out ProviderSourceDataset sourceDataset))
{
sourceDataset = new ProviderSourceDataset
{
DataGranularity = relationshipSummary.DataGranularity,
SpecificationId = specificationId,
DefinesScope = relationshipSummary.DefinesScope,
DataRelationship = new Reference(relationshipSummary.Relationship.Id, relationshipSummary.Relationship.Name),
DatasetRelationshipSummary = new Reference(relationshipSummary.Id, relationshipSummary.Name),
ProviderId = providerId
};
sourceDataset.Current = new ProviderSourceDatasetVersion
{
Rows = new List <Dictionary <string, object> >(),
Dataset = new VersionReference(dataset.Id, dataset.Name, version),
Date = DateTimeOffset.Now.ToLocalTime(),
ProviderId = providerId,
Version = 1,
PublishStatus = Models.Versioning.PublishStatus.Draft,
ProviderSourceDatasetId = sourceDataset.Id,
Author = user
};
if (!resultsByProviderId.TryAdd(providerId, sourceDataset))
{
resultsByProviderId.TryGetValue(providerId, out sourceDataset);
}
}
if (_featureToggle.IsUseFieldDefinitionIdsInSourceDatasetsEnabled())
{
sourceDataset.DataDefinitionId = relationshipSummary.DatasetDefinition.Id;
Dictionary <string, object> rows = new Dictionary <string, object>();
foreach (KeyValuePair <string, object> rowField in row.Fields)
{
foreach (TableDefinition tableDefinition in datasetDefinition.TableDefinitions)
{
FieldDefinition fieldDefinition = tableDefinition.FieldDefinitions.FirstOrDefault(m => m.Name == rowField.Key);
if (fieldDefinition != null)
{
rows.Add(fieldDefinition.Id, rowField.Value);
}
}
}
sourceDataset.Current.Rows.Add(rows);
}
else
{
sourceDataset.DataDefinition = new Reference(relationshipSummary.DatasetDefinition.Id, relationshipSummary.DatasetDefinition.Name);
sourceDataset.Current.Rows.Add(row.Fields);
}
}
});
ConcurrentBag <ProviderSourceDatasetVersion> historyToSave = new ConcurrentBag <ProviderSourceDatasetVersion>();
List <Task> historySaveTasks = new List <Task>(resultsByProviderId.Count);
SemaphoreSlim throttler = new SemaphoreSlim(initialCount: 15);
foreach (KeyValuePair <string, ProviderSourceDataset> providerSourceDataset in resultsByProviderId)
{
await throttler.WaitAsync();
historySaveTasks.Add(
Task.Run(async() =>
{
try
{
string providerId = providerSourceDataset.Key;
ProviderSourceDataset sourceDataset = providerSourceDataset.Value;
ProviderSourceDatasetVersion newVersion = null;
if (existingCurrent.ContainsKey(providerId))
{
newVersion = existingCurrent[providerId].Current.Clone() as ProviderSourceDatasetVersion;
string existingDatasetJson = JsonConvert.SerializeObject(existingCurrent[providerId].Current.Rows);
string latestDatasetJson = JsonConvert.SerializeObject(sourceDataset.Current.Rows);
if (existingDatasetJson != latestDatasetJson)
{
newVersion = await _sourceDatasetsVersionRepository.CreateVersion(newVersion, existingCurrent[providerId].Current, providerId);
newVersion.Author = user;
newVersion.Rows = sourceDataset.Current.Rows;
sourceDataset.Current = newVersion;
updateCurrentDatasets.TryAdd(providerId, sourceDataset);
historyToSave.Add(newVersion);
}
existingCurrent.TryRemove(providerId, out ProviderSourceDataset existingProviderSourceDataset);
}
else
{
newVersion = sourceDataset.Current;
updateCurrentDatasets.TryAdd(providerId, sourceDataset);
historyToSave.Add(newVersion);
}
}
finally
{
throttler.Release();
}
}));
}
await TaskHelper.WhenAllAndThrow(historySaveTasks.ToArray());
if (updateCurrentDatasets.Count > 0)
{
_logger.Information($"Saving {updateCurrentDatasets.Count()} updated source datasets");
await _providerResultsRepositoryPolicy.ExecuteAsync(() =>
_providersResultsRepository.UpdateCurrentProviderSourceDatasets(updateCurrentDatasets.Values));
}
if (_featureToggle.IsProviderResultsSpecificationCleanupEnabled() && existingCurrent.Any())
{
_logger.Information($"Removing {existingCurrent.Count()} missing source datasets");
await _providerResultsRepositoryPolicy.ExecuteAsync(() =>
_providersResultsRepository.DeleteCurrentProviderSourceDatasets(existingCurrent.Values));
foreach (IEnumerable <ProviderSourceDataset> providerSourceDataSets in existingCurrent.Values.Partition <ProviderSourceDataset>(1000))
{
await SendProviderSourceDatasetCleanupMessageToTopic(specificationId, ServiceBusConstants.TopicNames.ProviderSourceDatasetCleanup, providerSourceDataSets);
}
}
if (historyToSave.Any())
{
_logger.Information($"Saving {historyToSave.Count()} items to history");
await _sourceDatasetsVersionRepository.SaveVersions(historyToSave);
}
Reference relationshipReference = new Reference(relationshipSummary.Relationship.Id, relationshipSummary.Relationship.Name);
DatasetAggregations datasetAggregations = GenerateAggregations(datasetDefinition, loadResult, specificationId, relationshipReference);
if (!datasetAggregations.Fields.IsNullOrEmpty())
{
await _datasetsAggregationsRepository.CreateDatasetAggregations(datasetAggregations);
}
await _cacheProvider.RemoveAsync <List <CalculationAggregation> >($"{CacheKeys.DatasetAggregationsForSpecification}{specificationId}");
await PopulateProviderSummariesForSpecification(specificationId, providerSummaries);
}
public Task <IActionResult> UpdateBuildProjectRelationships([FromQuery] string specificationId,
[FromBody] DatasetRelationshipSummary relationship) => _buildProjectsService.UpdateBuildProjectRelationships(specificationId, relationship);
public async Task <ApiResponse <BuildProject> > UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary datasetRelationshipSummary)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.ArgumentNotNull(datasetRelationshipSummary, nameof(datasetRelationshipSummary));
string url = $"{UrlRoot}/update-buildproject-relationships?specificationId={specificationId}";
return(await PostAsync <BuildProject, DatasetRelationshipSummary>(url, datasetRelationshipSummary));
}
public async Task <IActionResult> CreateRelationship(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json);
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId);
if (specification == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = request.BuildMessageProperties();
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}
public async Task <BuildProject> UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary datasetRelationshipSummary)
{
if (string.IsNullOrWhiteSpace(specificationId))
{
throw new ArgumentNullException(nameof(specificationId));
}
Guard.ArgumentNotNull(datasetRelationshipSummary, nameof(datasetRelationshipSummary));
string url = $"{updateRelationshipsUrl}{specificationId}";
return(await _apiClient.PostAsync <BuildProject, DatasetRelationshipSummary>(url, datasetRelationshipSummary));
}
public async Task <BuildProject> UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary datasetRelationshipSummary)
{
if (string.IsNullOrWhiteSpace(specificationId))
{
throw new ArgumentNullException(nameof(specificationId));
}
Guard.ArgumentNotNull(datasetRelationshipSummary, nameof(datasetRelationshipSummary));
ApiResponse <Common.ApiClient.Calcs.Models.BuildProject> apiResponse = await _apiClient.UpdateBuildProjectRelationships(specificationId, _mapper.Map <Common.ApiClient.Calcs.Models.DatasetRelationshipSummary>(datasetRelationshipSummary));
return(_mapper.Map <BuildProject>(apiResponse?.Content));
}
public async Task <IActionResult> UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary relationship)
{
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("No specification Id was provided to UpdateBuildProjectRelationships");
return(new BadRequestObjectResult("Null or empty specification Id provided"));
}
if (relationship == null)
{
_logger.Error("A null relationship message was provided to UpdateBuildProjectRelationships");
return(new BadRequestObjectResult("Null relationship provided"));
}
BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId);
IEnumerable <Models.Calcs.Calculation> calculations = await _calculationsRepository.GetCalculationsBySpecificationId(specificationId);
buildProject.Build = _sourceCodeService.Compile(buildProject, calculations ?? Enumerable.Empty <Models.Calcs.Calculation>());
if (!_featureToggle.IsDynamicBuildProjectEnabled())
{
await _buildProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectsRepository.UpdateBuildProject(buildProject));
}
await _sourceCodeService.SaveAssembly(buildProject);
return(new OkObjectResult(buildProject));
}
public async Task <IActionResult> CreateRelationship(CreateDefinitionSpecificationRelationshipModel model, Reference author, string correlationId)
{
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(model.SpecificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
SpecModel.SpecificationSummary specification = specificationApiResponse.Content;
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations,
Author = author,
LastUpdated = _dateTimeProvider.UtcNow,
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, author);
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}
