public async Task UpdateBuildProjectRelationships(Message message) { Guard.ArgumentNotNull(message, nameof(message)); DatasetRelationshipSummary relationship = message.GetPayloadAsInstanceOf <DatasetRelationshipSummary>(); if (relationship == null) { _logger.Error("A null relationship message was provided to UpdateBuildProjectRelationships"); throw new ArgumentNullException(nameof(relationship)); } if (!message.UserProperties.ContainsKey("specification-id")) { _logger.Error("Message properties does not contain a specification id"); throw new KeyNotFoundException("specification-id"); } string specificationId = message.UserProperties["specification-id"].ToString(); if (string.IsNullOrWhiteSpace(specificationId)) { _logger.Error($"Message does not contain a specification id"); throw new ArgumentNullException(nameof(specificationId)); } BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId); if (buildProject.DatasetRelationships == null) { buildProject.DatasetRelationships = new List <DatasetRelationshipSummary>(); } if (!buildProject.DatasetRelationships.Any(m => m.Name == relationship.Name)) { buildProject.DatasetRelationships.Add(relationship); IEnumerable <Models.Calcs.Calculation> calculations = await _calculationsRepository.GetCalculationsBySpecificationId(specificationId); CompilerOptions compilerOptions = await _calculationsRepository.GetCompilerOptions(specificationId); buildProject.Build = _sourceCodeService.Compile(buildProject, calculations ?? Enumerable.Empty <Models.Calcs.Calculation>(), compilerOptions); if (!_featureToggle.IsDynamicBuildProjectEnabled()) { await _buildProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectsRepository.UpdateBuildProject(buildProject)); } await _sourceCodeService.SaveAssembly(buildProject); } }
private static StatementSyntax GetDatasetProperties(DatasetRelationshipSummary datasetRelationship) { StringBuilder builder = new StringBuilder(); builder.AppendLine($"<DatasetRelationship(Id := \"{datasetRelationship.Id}\", Name := \"{datasetRelationship.Name}\")>"); builder.AppendLine($"<Field(Id := \"{datasetRelationship.DatasetDefinition.Id}\", Name := \"{datasetRelationship.DatasetDefinition.Name}\")>"); if (!string.IsNullOrWhiteSpace(datasetRelationship?.DatasetDefinition?.Description)) { builder.AppendLine($"<Description(Description := \"{datasetRelationship.DatasetDefinition.Description?.Replace("\"", "\"\"")}\")>"); } builder.AppendLine(datasetRelationship.DataGranularity == DataGranularity.SingleRowPerProvider ? $"Public Property {GenerateIdentifier(datasetRelationship.Name)}() As {GenerateIdentifier($"{datasetRelationship.DatasetDefinition.Name}Dataset")}" : $"Public Property {GenerateIdentifier(datasetRelationship.Name)}() As System.Collections.Generic.List(Of {GenerateIdentifier($"{datasetRelationship.DatasetDefinition.Name}Dataset")})"); SyntaxTree tree = SyntaxFactory.ParseSyntaxTree(builder.ToString()); return(tree.GetRoot().DescendantNodes().OfType <StatementSyntax>() .FirstOrDefault()); }
public async Task UpdateBuildProjectRelationships() { string id = NewRandomString(); string expectedUri = $"update-buildproject-relationships?specificationId={id}"; BuildProject expectedResponse = new BuildProject(); DatasetRelationshipSummary expectedRequest = new DatasetRelationshipSummary(); GivenTheResponse(expectedUri, expectedResponse, HttpMethod.Post); ApiResponse <BuildProject> apiResponse = await _client.UpdateBuildProjectRelationships(id, expectedRequest); apiResponse?.StatusCode .Should() .Be(HttpStatusCode.OK); apiResponse .Content .Should() .BeEquivalentTo(expectedResponse); }
public async Task <IActionResult> UpdateBuildProjectRelationships(HttpRequest request) { request.Query.TryGetValue("specificationId", out Microsoft.Extensions.Primitives.StringValues specId); string specificationId = specId.FirstOrDefault(); if (string.IsNullOrWhiteSpace(specificationId)) { _logger.Error("No specification Id was provided to UpdateBuildProjectRelationships"); return(new BadRequestObjectResult("Null or empty specification Id provided")); } string json = await request.GetRawBodyStringAsync(); DatasetRelationshipSummary relationship = JsonConvert.DeserializeObject <DatasetRelationshipSummary>(json); if (relationship == null) { _logger.Error("A null relationship message was provided to UpdateBuildProjectRelationships"); return(new BadRequestObjectResult("Null relationship provided")); } BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId); IEnumerable <Models.Calcs.Calculation> calculations = await _calculationsRepository.GetCalculationsBySpecificationId(specificationId); buildProject.Build = _sourceCodeService.Compile(buildProject, calculations ?? Enumerable.Empty <Models.Calcs.Calculation>()); if (!_featureToggle.IsDynamicBuildProjectEnabled()) { await _buildProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectsRepository.UpdateBuildProject(buildProject)); } await _sourceCodeService.SaveAssembly(buildProject); return(new OkObjectResult(buildProject)); }
private async Task PersistDataset(TableLoadResult loadResult, Dataset dataset, DatasetDefinition datasetDefinition, BuildProject buildProject, string specificationId, string relationshipId, int version, Reference user) { IEnumerable <ProviderSummary> providerSummaries = await _providerService.FetchCoreProviderData(); Guard.IsNullOrWhiteSpace(relationshipId, nameof(relationshipId)); IList <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>(); if (buildProject.DatasetRelationships == null) { _logger.Error($"No dataset relationships found for build project with id : '{buildProject.Id}' for specification '{specificationId}'"); return; } DatasetRelationshipSummary relationshipSummary = buildProject.DatasetRelationships.FirstOrDefault(m => m.Relationship.Id == relationshipId); if (relationshipSummary == null) { _logger.Error($"No dataset relationship found for build project with id : {buildProject.Id} with data definition id {datasetDefinition.Id} and relationshipId '{relationshipId}'"); return; } ConcurrentDictionary <string, ProviderSourceDataset> existingCurrent = new ConcurrentDictionary <string, ProviderSourceDataset>(); IEnumerable <ProviderSourceDataset> existingCurrentDatasets = await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providersResultsRepository.GetCurrentProviderSourceDatasets(specificationId, relationshipId)); if (existingCurrentDatasets.AnyWithNullCheck()) { foreach (ProviderSourceDataset currentDataset in existingCurrentDatasets) { existingCurrent.TryAdd(currentDataset.ProviderId, currentDataset); } } ConcurrentDictionary <string, ProviderSourceDataset> resultsByProviderId = new ConcurrentDictionary <string, ProviderSourceDataset>(); ConcurrentDictionary <string, ProviderSourceDataset> updateCurrentDatasets = new ConcurrentDictionary <string, ProviderSourceDataset>(); Parallel.ForEach(loadResult.Rows, (RowLoadResult row) => { IEnumerable <string> allProviderIds = GetProviderIdsForIdentifier(datasetDefinition, row, providerSummaries); foreach (string providerId in allProviderIds) { if (!resultsByProviderId.TryGetValue(providerId, out ProviderSourceDataset sourceDataset)) { sourceDataset = new ProviderSourceDataset { DataGranularity = relationshipSummary.DataGranularity, SpecificationId = specificationId, DefinesScope = relationshipSummary.DefinesScope, DataRelationship = new Reference(relationshipSummary.Relationship.Id, relationshipSummary.Relationship.Name), DatasetRelationshipSummary = new Reference(relationshipSummary.Id, relationshipSummary.Name), ProviderId = providerId }; sourceDataset.Current = new ProviderSourceDatasetVersion { Rows = new List <Dictionary <string, object> >(), Dataset = new VersionReference(dataset.Id, dataset.Name, version), Date = DateTimeOffset.Now.ToLocalTime(), ProviderId = providerId, Version = 1, PublishStatus = Models.Versioning.PublishStatus.Draft, ProviderSourceDatasetId = sourceDataset.Id, Author = user }; if (!resultsByProviderId.TryAdd(providerId, sourceDataset)) { resultsByProviderId.TryGetValue(providerId, out sourceDataset); } } if (_featureToggle.IsUseFieldDefinitionIdsInSourceDatasetsEnabled()) { sourceDataset.DataDefinitionId = relationshipSummary.DatasetDefinition.Id; Dictionary <string, object> rows = new Dictionary <string, object>(); foreach (KeyValuePair <string, object> rowField in row.Fields) { foreach (TableDefinition tableDefinition in datasetDefinition.TableDefinitions) { FieldDefinition fieldDefinition = tableDefinition.FieldDefinitions.FirstOrDefault(m => m.Name == rowField.Key); if (fieldDefinition != null) { rows.Add(fieldDefinition.Id, rowField.Value); } } } sourceDataset.Current.Rows.Add(rows); } else { sourceDataset.DataDefinition = new Reference(relationshipSummary.DatasetDefinition.Id, relationshipSummary.DatasetDefinition.Name); sourceDataset.Current.Rows.Add(row.Fields); } } }); ConcurrentBag <ProviderSourceDatasetVersion> historyToSave = new ConcurrentBag <ProviderSourceDatasetVersion>(); List <Task> historySaveTasks = new List <Task>(resultsByProviderId.Count); SemaphoreSlim throttler = new SemaphoreSlim(initialCount: 15); foreach (KeyValuePair <string, ProviderSourceDataset> providerSourceDataset in resultsByProviderId) { await throttler.WaitAsync(); historySaveTasks.Add( Task.Run(async() => { try { string providerId = providerSourceDataset.Key; ProviderSourceDataset sourceDataset = providerSourceDataset.Value; ProviderSourceDatasetVersion newVersion = null; if (existingCurrent.ContainsKey(providerId)) { newVersion = existingCurrent[providerId].Current.Clone() as ProviderSourceDatasetVersion; string existingDatasetJson = JsonConvert.SerializeObject(existingCurrent[providerId].Current.Rows); string latestDatasetJson = JsonConvert.SerializeObject(sourceDataset.Current.Rows); if (existingDatasetJson != latestDatasetJson) { newVersion = await _sourceDatasetsVersionRepository.CreateVersion(newVersion, existingCurrent[providerId].Current, providerId); newVersion.Author = user; newVersion.Rows = sourceDataset.Current.Rows; sourceDataset.Current = newVersion; updateCurrentDatasets.TryAdd(providerId, sourceDataset); historyToSave.Add(newVersion); } existingCurrent.TryRemove(providerId, out ProviderSourceDataset existingProviderSourceDataset); } else { newVersion = sourceDataset.Current; updateCurrentDatasets.TryAdd(providerId, sourceDataset); historyToSave.Add(newVersion); } } finally { throttler.Release(); } })); } await TaskHelper.WhenAllAndThrow(historySaveTasks.ToArray()); if (updateCurrentDatasets.Count > 0) { _logger.Information($"Saving {updateCurrentDatasets.Count()} updated source datasets"); await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providersResultsRepository.UpdateCurrentProviderSourceDatasets(updateCurrentDatasets.Values)); } if (_featureToggle.IsProviderResultsSpecificationCleanupEnabled() && existingCurrent.Any()) { _logger.Information($"Removing {existingCurrent.Count()} missing source datasets"); await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providersResultsRepository.DeleteCurrentProviderSourceDatasets(existingCurrent.Values)); foreach (IEnumerable <ProviderSourceDataset> providerSourceDataSets in existingCurrent.Values.Partition <ProviderSourceDataset>(1000)) { await SendProviderSourceDatasetCleanupMessageToTopic(specificationId, ServiceBusConstants.TopicNames.ProviderSourceDatasetCleanup, providerSourceDataSets); } } if (historyToSave.Any()) { _logger.Information($"Saving {historyToSave.Count()} items to history"); await _sourceDatasetsVersionRepository.SaveVersions(historyToSave); } Reference relationshipReference = new Reference(relationshipSummary.Relationship.Id, relationshipSummary.Relationship.Name); DatasetAggregations datasetAggregations = GenerateAggregations(datasetDefinition, loadResult, specificationId, relationshipReference); if (!datasetAggregations.Fields.IsNullOrEmpty()) { await _datasetsAggregationsRepository.CreateDatasetAggregations(datasetAggregations); } await _cacheProvider.RemoveAsync <List <CalculationAggregation> >($"{CacheKeys.DatasetAggregationsForSpecification}{specificationId}"); await PopulateProviderSummariesForSpecification(specificationId, providerSummaries); }
public Task <IActionResult> UpdateBuildProjectRelationships([FromQuery] string specificationId, [FromBody] DatasetRelationshipSummary relationship) => _buildProjectsService.UpdateBuildProjectRelationships(specificationId, relationship);
public async Task <ApiResponse <BuildProject> > UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary datasetRelationshipSummary) { Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId)); Guard.ArgumentNotNull(datasetRelationshipSummary, nameof(datasetRelationshipSummary)); string url = $"{UrlRoot}/update-buildproject-relationships?specificationId={specificationId}"; return(await PostAsync <BuildProject, DatasetRelationshipSummary>(url, datasetRelationshipSummary)); }
public async Task <IActionResult> CreateRelationship(HttpRequest request) { string json = await request.GetRawBodyStringAsync(); CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json); if (model == null) { _logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship"); return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided")); } BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState(); if (validationResult != null) { return(validationResult); } DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId); if (definition == null) { _logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}"); return(new StatusCodeResult(412)); } SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId); if (specification == null) { _logger.Error($"Specification was not found for id {model.SpecificationId}"); return(new StatusCodeResult(412)); } string relationshipId = Guid.NewGuid().ToString(); DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship { Name = model.Name, DatasetDefinition = new Reference(definition.Id, definition.Name), Specification = new Reference(specification.Id, specification.Name), Description = model.Description, Id = relationshipId, IsSetAsProviderData = model.IsSetAsProviderData, UsedInDataAggregations = model.UsedInDataAggregations }; HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } IDictionary <string, string> properties = request.BuildMessageProperties(); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification, new AssignDefinitionRelationshipMessage { SpecificationId = specification.Id, RelationshipId = relationshipId }, properties); DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary { Name = relationship.Name, Id = Guid.NewGuid().ToString(), Relationship = new Reference(relationship.Id, relationship.Name), DatasetDefinition = definition, DatasetDefinitionId = definition.Id, DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider, DefinesScope = relationship.IsSetAsProviderData }; BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary); await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}"); return(new OkObjectResult(relationship)); }
public async Task <BuildProject> UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary datasetRelationshipSummary) { if (string.IsNullOrWhiteSpace(specificationId)) { throw new ArgumentNullException(nameof(specificationId)); } Guard.ArgumentNotNull(datasetRelationshipSummary, nameof(datasetRelationshipSummary)); string url = $"{updateRelationshipsUrl}{specificationId}"; return(await _apiClient.PostAsync <BuildProject, DatasetRelationshipSummary>(url, datasetRelationshipSummary)); }
public async Task <BuildProject> UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary datasetRelationshipSummary) { if (string.IsNullOrWhiteSpace(specificationId)) { throw new ArgumentNullException(nameof(specificationId)); } Guard.ArgumentNotNull(datasetRelationshipSummary, nameof(datasetRelationshipSummary)); ApiResponse <Common.ApiClient.Calcs.Models.BuildProject> apiResponse = await _apiClient.UpdateBuildProjectRelationships(specificationId, _mapper.Map <Common.ApiClient.Calcs.Models.DatasetRelationshipSummary>(datasetRelationshipSummary)); return(_mapper.Map <BuildProject>(apiResponse?.Content)); }
public async Task <IActionResult> UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary relationship) { if (string.IsNullOrWhiteSpace(specificationId)) { _logger.Error("No specification Id was provided to UpdateBuildProjectRelationships"); return(new BadRequestObjectResult("Null or empty specification Id provided")); } if (relationship == null) { _logger.Error("A null relationship message was provided to UpdateBuildProjectRelationships"); return(new BadRequestObjectResult("Null relationship provided")); } BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId); IEnumerable <Models.Calcs.Calculation> calculations = await _calculationsRepository.GetCalculationsBySpecificationId(specificationId); buildProject.Build = _sourceCodeService.Compile(buildProject, calculations ?? Enumerable.Empty <Models.Calcs.Calculation>()); if (!_featureToggle.IsDynamicBuildProjectEnabled()) { await _buildProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectsRepository.UpdateBuildProject(buildProject)); } await _sourceCodeService.SaveAssembly(buildProject); return(new OkObjectResult(buildProject)); }
public async Task <IActionResult> CreateRelationship(CreateDefinitionSpecificationRelationshipModel model, Reference author, string correlationId) { if (model == null) { _logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship"); return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided")); } BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState(); if (validationResult != null) { return(validationResult); } DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId); if (definition == null) { _logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}"); return(new StatusCodeResult(412)); } ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(model.SpecificationId)); if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null) { _logger.Error($"Specification was not found for id {model.SpecificationId}"); return(new StatusCodeResult(412)); } SpecModel.SpecificationSummary specification = specificationApiResponse.Content; string relationshipId = Guid.NewGuid().ToString(); DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship { Name = model.Name, DatasetDefinition = new Reference(definition.Id, definition.Name), Specification = new Reference(specification.Id, specification.Name), Description = model.Description, Id = relationshipId, IsSetAsProviderData = model.IsSetAsProviderData, UsedInDataAggregations = model.UsedInDataAggregations, Author = author, LastUpdated = _dateTimeProvider.UtcNow, }; HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, author); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification, new AssignDefinitionRelationshipMessage { SpecificationId = specification.Id, RelationshipId = relationshipId }, properties); DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary { Name = relationship.Name, Id = Guid.NewGuid().ToString(), Relationship = new Reference(relationship.Id, relationship.Name), DatasetDefinition = definition, DatasetDefinitionId = definition.Id, DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider, DefinesScope = relationship.IsSetAsProviderData }; await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary); await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}"); return(new OkObjectResult(relationship)); }