public ProviderSourceDatasetBuilder WithCurrent(ProviderSourceDatasetVersion current) { _current = current; return(this); }
private async Task PersistDataset(TableLoadResult loadResult, Dataset dataset, DatasetDefinition datasetDefinition, BuildProject buildProject, string specificationId, string relationshipId, int version, Reference user) { IEnumerable <ProviderSummary> providerSummaries = await _providerService.FetchCoreProviderData(); Guard.IsNullOrWhiteSpace(relationshipId, nameof(relationshipId)); IList <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>(); if (buildProject.DatasetRelationships == null) { _logger.Error($"No dataset relationships found for build project with id : '{buildProject.Id}' for specification '{specificationId}'"); return; } DatasetRelationshipSummary relationshipSummary = buildProject.DatasetRelationships.FirstOrDefault(m => m.Relationship.Id == relationshipId); if (relationshipSummary == null) { _logger.Error($"No dataset relationship found for build project with id : {buildProject.Id} with data definition id {datasetDefinition.Id} and relationshipId '{relationshipId}'"); return; } ConcurrentDictionary <string, ProviderSourceDataset> existingCurrent = new ConcurrentDictionary <string, ProviderSourceDataset>(); IEnumerable <ProviderSourceDataset> existingCurrentDatasets = await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providersResultsRepository.GetCurrentProviderSourceDatasets(specificationId, relationshipId)); if (existingCurrentDatasets.AnyWithNullCheck()) { foreach (ProviderSourceDataset currentDataset in existingCurrentDatasets) { existingCurrent.TryAdd(currentDataset.ProviderId, currentDataset); } } ConcurrentDictionary <string, ProviderSourceDataset> resultsByProviderId = new ConcurrentDictionary <string, ProviderSourceDataset>(); ConcurrentDictionary <string, ProviderSourceDataset> updateCurrentDatasets = new ConcurrentDictionary <string, ProviderSourceDataset>(); Parallel.ForEach(loadResult.Rows, (RowLoadResult row) => { IEnumerable <string> allProviderIds = GetProviderIdsForIdentifier(datasetDefinition, row, providerSummaries); foreach (string providerId in allProviderIds) { if (!resultsByProviderId.TryGetValue(providerId, out ProviderSourceDataset sourceDataset)) { sourceDataset = new ProviderSourceDataset { DataGranularity = relationshipSummary.DataGranularity, SpecificationId = specificationId, DefinesScope = relationshipSummary.DefinesScope, DataRelationship = new Reference(relationshipSummary.Relationship.Id, relationshipSummary.Relationship.Name), DatasetRelationshipSummary = new Reference(relationshipSummary.Id, relationshipSummary.Name), ProviderId = providerId }; sourceDataset.Current = new ProviderSourceDatasetVersion { Rows = new List <Dictionary <string, object> >(), Dataset = new VersionReference(dataset.Id, dataset.Name, version), Date = DateTimeOffset.Now.ToLocalTime(), ProviderId = providerId, Version = 1, PublishStatus = Models.Versioning.PublishStatus.Draft, ProviderSourceDatasetId = sourceDataset.Id, Author = user }; if (!resultsByProviderId.TryAdd(providerId, sourceDataset)) { resultsByProviderId.TryGetValue(providerId, out sourceDataset); } } if (_featureToggle.IsUseFieldDefinitionIdsInSourceDatasetsEnabled()) { sourceDataset.DataDefinitionId = relationshipSummary.DatasetDefinition.Id; Dictionary <string, object> rows = new Dictionary <string, object>(); foreach (KeyValuePair <string, object> rowField in row.Fields) { foreach (TableDefinition tableDefinition in datasetDefinition.TableDefinitions) { FieldDefinition fieldDefinition = tableDefinition.FieldDefinitions.FirstOrDefault(m => m.Name == rowField.Key); if (fieldDefinition != null) { rows.Add(fieldDefinition.Id, rowField.Value); } } } sourceDataset.Current.Rows.Add(rows); } else { sourceDataset.DataDefinition = new Reference(relationshipSummary.DatasetDefinition.Id, relationshipSummary.DatasetDefinition.Name); sourceDataset.Current.Rows.Add(row.Fields); } } }); ConcurrentBag <ProviderSourceDatasetVersion> historyToSave = new ConcurrentBag <ProviderSourceDatasetVersion>(); List <Task> historySaveTasks = new List <Task>(resultsByProviderId.Count); SemaphoreSlim throttler = new SemaphoreSlim(initialCount: 15); foreach (KeyValuePair <string, ProviderSourceDataset> providerSourceDataset in resultsByProviderId) { await throttler.WaitAsync(); historySaveTasks.Add( Task.Run(async() => { try { string providerId = providerSourceDataset.Key; ProviderSourceDataset sourceDataset = providerSourceDataset.Value; ProviderSourceDatasetVersion newVersion = null; if (existingCurrent.ContainsKey(providerId)) { newVersion = existingCurrent[providerId].Current.Clone() as ProviderSourceDatasetVersion; string existingDatasetJson = JsonConvert.SerializeObject(existingCurrent[providerId].Current.Rows); string latestDatasetJson = JsonConvert.SerializeObject(sourceDataset.Current.Rows); if (existingDatasetJson != latestDatasetJson) { newVersion = await _sourceDatasetsVersionRepository.CreateVersion(newVersion, existingCurrent[providerId].Current, providerId); newVersion.Author = user; newVersion.Rows = sourceDataset.Current.Rows; sourceDataset.Current = newVersion; updateCurrentDatasets.TryAdd(providerId, sourceDataset); historyToSave.Add(newVersion); } existingCurrent.TryRemove(providerId, out ProviderSourceDataset existingProviderSourceDataset); } else { newVersion = sourceDataset.Current; updateCurrentDatasets.TryAdd(providerId, sourceDataset); historyToSave.Add(newVersion); } } finally { throttler.Release(); } })); } await TaskHelper.WhenAllAndThrow(historySaveTasks.ToArray()); if (updateCurrentDatasets.Count > 0) { _logger.Information($"Saving {updateCurrentDatasets.Count()} updated source datasets"); await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providersResultsRepository.UpdateCurrentProviderSourceDatasets(updateCurrentDatasets.Values)); } if (_featureToggle.IsProviderResultsSpecificationCleanupEnabled() && existingCurrent.Any()) { _logger.Information($"Removing {existingCurrent.Count()} missing source datasets"); await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providersResultsRepository.DeleteCurrentProviderSourceDatasets(existingCurrent.Values)); foreach (IEnumerable <ProviderSourceDataset> providerSourceDataSets in existingCurrent.Values.Partition <ProviderSourceDataset>(1000)) { await SendProviderSourceDatasetCleanupMessageToTopic(specificationId, ServiceBusConstants.TopicNames.ProviderSourceDatasetCleanup, providerSourceDataSets); } } if (historyToSave.Any()) { _logger.Information($"Saving {historyToSave.Count()} items to history"); await _sourceDatasetsVersionRepository.SaveVersions(historyToSave); } Reference relationshipReference = new Reference(relationshipSummary.Relationship.Id, relationshipSummary.Relationship.Name); DatasetAggregations datasetAggregations = GenerateAggregations(datasetDefinition, loadResult, specificationId, relationshipReference); if (!datasetAggregations.Fields.IsNullOrEmpty()) { await _datasetsAggregationsRepository.CreateDatasetAggregations(datasetAggregations); } await _cacheProvider.RemoveAsync <List <CalculationAggregation> >($"{CacheKeys.DatasetAggregationsForSpecification}{specificationId}"); await PopulateProviderSummariesForSpecification(specificationId, providerSummaries); }