private async Task PersistDataset(TableLoadResult loadResult, Dataset dataset, DatasetDefinition datasetDefinition, BuildProject buildProject, string specificationId, string relationshipId, int version, Reference user)
{
IEnumerable <ProviderSummary> providerSummaries = await _providerService.FetchCoreProviderData();
Guard.IsNullOrWhiteSpace(relationshipId, nameof(relationshipId));
IList <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>();
if (buildProject.DatasetRelationships == null)
{
_logger.Error($"No dataset relationships found for build project with id : '{buildProject.Id}' for specification '{specificationId}'");
return;
}
DatasetRelationshipSummary relationshipSummary = buildProject.DatasetRelationships.FirstOrDefault(m => m.Relationship.Id == relationshipId);
if (relationshipSummary == null)
{
_logger.Error($"No dataset relationship found for build project with id : {buildProject.Id} with data definition id {datasetDefinition.Id} and relationshipId '{relationshipId}'");
return;
}
ConcurrentDictionary <string, ProviderSourceDataset> existingCurrent = new ConcurrentDictionary <string, ProviderSourceDataset>();
IEnumerable <ProviderSourceDataset> existingCurrentDatasets = await _providerResultsRepositoryPolicy.ExecuteAsync(() =>
_providersResultsRepository.GetCurrentProviderSourceDatasets(specificationId, relationshipId));
if (existingCurrentDatasets.AnyWithNullCheck())
{
foreach (ProviderSourceDataset currentDataset in existingCurrentDatasets)
{
existingCurrent.TryAdd(currentDataset.ProviderId, currentDataset);
}
}
ConcurrentDictionary <string, ProviderSourceDataset> resultsByProviderId = new ConcurrentDictionary <string, ProviderSourceDataset>();
ConcurrentDictionary <string, ProviderSourceDataset> updateCurrentDatasets = new ConcurrentDictionary <string, ProviderSourceDataset>();
Parallel.ForEach(loadResult.Rows, (RowLoadResult row) =>
{
IEnumerable <string> allProviderIds = GetProviderIdsForIdentifier(datasetDefinition, row, providerSummaries);
foreach (string providerId in allProviderIds)
{
if (!resultsByProviderId.TryGetValue(providerId, out ProviderSourceDataset sourceDataset))
{
sourceDataset = new ProviderSourceDataset
{
DataGranularity = relationshipSummary.DataGranularity,
SpecificationId = specificationId,
DefinesScope = relationshipSummary.DefinesScope,
DataRelationship = new Reference(relationshipSummary.Relationship.Id, relationshipSummary.Relationship.Name),
DatasetRelationshipSummary = new Reference(relationshipSummary.Id, relationshipSummary.Name),
ProviderId = providerId
};
sourceDataset.Current = new ProviderSourceDatasetVersion
{
Rows = new List <Dictionary <string, object> >(),
Dataset = new VersionReference(dataset.Id, dataset.Name, version),
Date = DateTimeOffset.Now.ToLocalTime(),
ProviderId = providerId,
Version = 1,
PublishStatus = Models.Versioning.PublishStatus.Draft,
ProviderSourceDatasetId = sourceDataset.Id,
Author = user
};
if (!resultsByProviderId.TryAdd(providerId, sourceDataset))
{
resultsByProviderId.TryGetValue(providerId, out sourceDataset);
}
}
if (_featureToggle.IsUseFieldDefinitionIdsInSourceDatasetsEnabled())
{
sourceDataset.DataDefinitionId = relationshipSummary.DatasetDefinition.Id;
Dictionary <string, object> rows = new Dictionary <string, object>();
foreach (KeyValuePair <string, object> rowField in row.Fields)
{
foreach (TableDefinition tableDefinition in datasetDefinition.TableDefinitions)
{
FieldDefinition fieldDefinition = tableDefinition.FieldDefinitions.FirstOrDefault(m => m.Name == rowField.Key);
if (fieldDefinition != null)
{
rows.Add(fieldDefinition.Id, rowField.Value);
}
}
}
sourceDataset.Current.Rows.Add(rows);
}
else
{
sourceDataset.DataDefinition = new Reference(relationshipSummary.DatasetDefinition.Id, relationshipSummary.DatasetDefinition.Name);
sourceDataset.Current.Rows.Add(row.Fields);
}
}
});
ConcurrentBag <ProviderSourceDatasetVersion> historyToSave = new ConcurrentBag <ProviderSourceDatasetVersion>();
List <Task> historySaveTasks = new List <Task>(resultsByProviderId.Count);
SemaphoreSlim throttler = new SemaphoreSlim(initialCount: 15);
foreach (KeyValuePair <string, ProviderSourceDataset> providerSourceDataset in resultsByProviderId)
{
await throttler.WaitAsync();
historySaveTasks.Add(
Task.Run(async() =>
{
try
{
string providerId = providerSourceDataset.Key;
ProviderSourceDataset sourceDataset = providerSourceDataset.Value;
ProviderSourceDatasetVersion newVersion = null;
if (existingCurrent.ContainsKey(providerId))
{
newVersion = existingCurrent[providerId].Current.Clone() as ProviderSourceDatasetVersion;
string existingDatasetJson = JsonConvert.SerializeObject(existingCurrent[providerId].Current.Rows);
string latestDatasetJson = JsonConvert.SerializeObject(sourceDataset.Current.Rows);
if (existingDatasetJson != latestDatasetJson)
{
newVersion = await _sourceDatasetsVersionRepository.CreateVersion(newVersion, existingCurrent[providerId].Current, providerId);
newVersion.Author = user;
newVersion.Rows = sourceDataset.Current.Rows;
sourceDataset.Current = newVersion;
updateCurrentDatasets.TryAdd(providerId, sourceDataset);
historyToSave.Add(newVersion);
}
existingCurrent.TryRemove(providerId, out ProviderSourceDataset existingProviderSourceDataset);
}
else
{
newVersion = sourceDataset.Current;
updateCurrentDatasets.TryAdd(providerId, sourceDataset);
historyToSave.Add(newVersion);
}
}
finally
{
throttler.Release();
}
}));
}
await TaskHelper.WhenAllAndThrow(historySaveTasks.ToArray());
if (updateCurrentDatasets.Count > 0)
{
_logger.Information($"Saving {updateCurrentDatasets.Count()} updated source datasets");
await _providerResultsRepositoryPolicy.ExecuteAsync(() =>
_providersResultsRepository.UpdateCurrentProviderSourceDatasets(updateCurrentDatasets.Values));
}
if (_featureToggle.IsProviderResultsSpecificationCleanupEnabled() && existingCurrent.Any())
{
_logger.Information($"Removing {existingCurrent.Count()} missing source datasets");
await _providerResultsRepositoryPolicy.ExecuteAsync(() =>
_providersResultsRepository.DeleteCurrentProviderSourceDatasets(existingCurrent.Values));
foreach (IEnumerable <ProviderSourceDataset> providerSourceDataSets in existingCurrent.Values.Partition <ProviderSourceDataset>(1000))
{
await SendProviderSourceDatasetCleanupMessageToTopic(specificationId, ServiceBusConstants.TopicNames.ProviderSourceDatasetCleanup, providerSourceDataSets);
}
}
if (historyToSave.Any())
{
_logger.Information($"Saving {historyToSave.Count()} items to history");
await _sourceDatasetsVersionRepository.SaveVersions(historyToSave);
}
Reference relationshipReference = new Reference(relationshipSummary.Relationship.Id, relationshipSummary.Relationship.Name);
DatasetAggregations datasetAggregations = GenerateAggregations(datasetDefinition, loadResult, specificationId, relationshipReference);
if (!datasetAggregations.Fields.IsNullOrEmpty())
{
await _datasetsAggregationsRepository.CreateDatasetAggregations(datasetAggregations);
}
await _cacheProvider.RemoveAsync <List <CalculationAggregation> >($"{CacheKeys.DatasetAggregationsForSpecification}{specificationId}");
await PopulateProviderSummariesForSpecification(specificationId, providerSummaries);
}
