public async Task <IActionResult> GetRelationshipBySpecificationIdAndName(string specificationId, string name)
{
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("The specification id was not provided to GetRelationshipsBySpecificationIdAndName");
return(new BadRequestObjectResult("No specification id was provided to GetRelationshipsBySpecificationIdAndName"));
}
if (string.IsNullOrWhiteSpace(name))
{
_logger.Error("The name was not provided to GetRelationshipsBySpecificationIdAndName");
return(new BadRequestObjectResult("No name was provided to GetRelationshipsBySpecificationIdAndName"));
}
DefinitionSpecificationRelationship relationship = await _datasetRepository.GetRelationshipBySpecificationIdAndName(specificationId, name);
if (relationship != null)
{
return(new OkObjectResult(relationship));
}
return(new NotFoundResult());
}
public async Task <IActionResult> GetRelationshipBySpecificationIdAndName(HttpRequest request)
{
request.Query.TryGetValue("specificationId", out Microsoft.Extensions.Primitives.StringValues specId);
string specificationId = specId.FirstOrDefault();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("The specification id was not provided to GetRelationshipsBySpecificationIdAndName");
return(new BadRequestObjectResult("No specification id was provided to GetRelationshipsBySpecificationIdAndName"));
}
request.Query.TryGetValue("name", out Microsoft.Extensions.Primitives.StringValues name);
string relationshipName = name.FirstOrDefault();
if (string.IsNullOrWhiteSpace(relationshipName))
{
_logger.Error("The name was not provided to GetRelationshipsBySpecificationIdAndName");
return(new BadRequestObjectResult("No name was provided to GetRelationshipsBySpecificationIdAndName"));
}
DefinitionSpecificationRelationship relationship = await _datasetRepository.GetRelationshipBySpecificationIdAndName(specificationId, name);
if (relationship != null)
{
return(new OkObjectResult(relationship));
}
return(new NotFoundResult());
}
public async Task <IActionResult> GetDataSourcesByRelationshipId(string relationshipId)
{
if (string.IsNullOrWhiteSpace(relationshipId))
{
_logger.Error("The relationshipId id was not provided to GetDataSourcesByRelationshipId");
return(new BadRequestObjectResult("No relationship id was provided"));
}
SelectDatasourceModel selectDatasourceModel = new SelectDatasourceModel();
DefinitionSpecificationRelationship relationship = await _datasetRepository.GetDefinitionSpecificationRelationshipById(relationshipId);
if (relationship == null)
{
return(new StatusCodeResult(412));
}
selectDatasourceModel.RelationshipId = relationship.Id;
selectDatasourceModel.RelationshipName = relationship.Name;
selectDatasourceModel.SpecificationId = relationship.Specification.Id;
selectDatasourceModel.SpecificationName = relationship.Specification.Name;
selectDatasourceModel.DefinitionId = relationship.DatasetDefinition.Id;
selectDatasourceModel.DefinitionName = relationship.DatasetDefinition.Name;
IEnumerable <Dataset> datasets = await _datasetRepository.GetDatasetsByQuery(m => m.Content.Definition.Id == relationship.DatasetDefinition.Id);
if (!datasets.IsNullOrEmpty())
{
if (selectDatasourceModel.Datasets == null)
{
selectDatasourceModel.Datasets = Enumerable.Empty <DatasetVersions>();
}
foreach (Dataset dataset in datasets)
{
selectDatasourceModel.Datasets = selectDatasourceModel.Datasets.Concat(new[] {
new DatasetVersions
{
Id = dataset.Id,
Name = dataset.Name,
Description = dataset.Description,
SelectedVersion = (relationship.DatasetVersion != null && relationship.DatasetVersion.Id == dataset.Id) ? relationship.DatasetVersion.Version : null as int?,
Versions = dataset.History.Select(m => new DatasetVersionModel
{
Id = m.Id,
Version = m.Version,
Date = m.Date,
Author = m.Author,
Comment = m.Comment
})
}
});
}
}
return(new OkObjectResult(selectDatasourceModel));
}
public async Task <IActionResult> GetCurrentRelationshipsBySpecificationId(string specificationId)
{
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error($"No specification id was provided to GetCurrentRelationshipsBySpecificationId");
return(new BadRequestObjectResult($"Null or empty {nameof(specificationId)} provided"));
}
ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(specificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
_logger.Error($"Failed to find specification for id: {specificationId}");
return(new StatusCodeResult(412));
}
IEnumerable <DefinitionSpecificationRelationship> relationships =
(await _datasetRepository.GetDefinitionSpecificationRelationshipsByQuery(m => m.Content.Specification.Id == specificationId)).ToList();
if (relationships.IsNullOrEmpty())
{
relationships = new DefinitionSpecificationRelationship[0];
}
IList <DatasetSpecificationRelationshipViewModel> relationshipViewModels = new List <DatasetSpecificationRelationshipViewModel>();
if (relationships.IsNullOrEmpty())
{
return(new OkObjectResult(relationshipViewModels));
}
IEnumerable <KeyValuePair <string, int> > datasetLatestVersions =
await _datasetRepository.GetDatasetLatestVersions(relationships.Select(_ => _.DatasetVersion?.Id));
IList <Task <DatasetSpecificationRelationshipViewModel> > tasks = new List <Task <DatasetSpecificationRelationshipViewModel> >();
foreach (DefinitionSpecificationRelationship relationship in relationships)
{
KeyValuePair <string, int> datasetLatestVersion = datasetLatestVersions.SingleOrDefault(_ => _.Key == relationship.DatasetVersion?.Id);
Task <DatasetSpecificationRelationshipViewModel> task = CreateViewModel(relationship, datasetLatestVersion);
tasks.Add(task);
}
await Task.WhenAll(tasks);
foreach (Task <DatasetSpecificationRelationshipViewModel> task in tasks)
{
relationshipViewModels.Add(task.Result);
}
return(new OkObjectResult(tasks.Select(m => m.Result)));
}
public async Task <IActionResult> GetCurrentRelationshipsBySpecificationId(HttpRequest request)
{
request.Query.TryGetValue("specificationId", out Microsoft.Extensions.Primitives.StringValues specId);
string specificationId = specId.FirstOrDefault();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error($"No specification id was provided to GetCurrentRelationshipsBySpecificationId");
return(new BadRequestObjectResult($"Null or empty {nameof(specificationId)} provided"));
}
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(specificationId);
if (specification == null)
{
_logger.Error($"Failed to find specification for id: {specificationId}");
return(new StatusCodeResult(412));
}
IEnumerable <DefinitionSpecificationRelationship> relationships =
(await _datasetRepository.GetDefinitionSpecificationRelationshipsByQuery(m => m.Specification.Id == specificationId)).ToList();
if (relationships.IsNullOrEmpty())
{
relationships = new DefinitionSpecificationRelationship[0];
}
IList <DatasetSpecificationRelationshipViewModel> relationshipViewModels = new List <DatasetSpecificationRelationshipViewModel>();
if (relationships.IsNullOrEmpty())
{
return(new OkObjectResult(relationshipViewModels));
}
IList <Task <DatasetSpecificationRelationshipViewModel> > tasks = new List <Task <DatasetSpecificationRelationshipViewModel> >();
foreach (DefinitionSpecificationRelationship relationship in relationships)
{
Task <DatasetSpecificationRelationshipViewModel> task = CreateViewModel(relationship);
tasks.Add(task);
}
await Task.WhenAll(tasks);
foreach (Task <DatasetSpecificationRelationshipViewModel> task in tasks)
{
relationshipViewModels.Add(task.Result);
}
return(new OkObjectResult(tasks.Select(m => m.Result)));
}
public async Task <IActionResult> GetDataSourcesByRelationshipId(HttpRequest request)
{
request.Query.TryGetValue("relationshipId", out Microsoft.Extensions.Primitives.StringValues relId);
string relationshipId = relId.FirstOrDefault();
if (string.IsNullOrWhiteSpace(relationshipId))
{
_logger.Error("The relationshipId id was not provided to GetDataSourcesByRelationshipId");
return(new BadRequestObjectResult("No relationship id was provided"));
}
SelectDatasourceModel selectDatasourceModel = new SelectDatasourceModel();
DefinitionSpecificationRelationship relationship = await _datasetRepository.GetDefinitionSpecificationRelationshipById(relationshipId);
if (relationship == null)
{
return(new StatusCodeResult(412));
}
selectDatasourceModel.RelationshipId = relationship.Id;
selectDatasourceModel.RelationshipName = relationship.Name;
selectDatasourceModel.SpecificationId = relationship.Specification.Id;
selectDatasourceModel.SpecificationName = relationship.Specification.Name;
selectDatasourceModel.DefinitionId = relationship.DatasetDefinition.Id;
selectDatasourceModel.DefinitionName = relationship.DatasetDefinition.Name;
IEnumerable <Dataset> datasets = await _datasetRepository.GetDatasetsByQuery(m => m.Definition.Id == relationship.DatasetDefinition.Id);
if (!datasets.IsNullOrEmpty())
{
if (selectDatasourceModel.Datasets == null)
{
selectDatasourceModel.Datasets = Enumerable.Empty <DatasetVersions>();
}
foreach (Dataset dataset in datasets)
{
selectDatasourceModel.Datasets = selectDatasourceModel.Datasets.Concat(new[] {
new DatasetVersions
{
Id = dataset.Id,
Name = dataset.Name,
SelectedVersion = (relationship.DatasetVersion != null && relationship.DatasetVersion.Id == dataset.Id) ? relationship.DatasetVersion.Version : null as int?,
Versions = dataset.History.Select(m => m.Version)
}
});
}
}
return(new OkObjectResult(selectDatasourceModel));
}
public Task <ApiResponse <DefinitionSpecificationRelationship> > GetDefinitionSpecificationRelationshipById(string relationshipId)
{
DefinitionSpecificationRelationship result = new DefinitionSpecificationRelationship()
{
DatasetDefinition = new Reference("1", "Dataset definition name"),
Description = "Description of the relationship",
Id = "1",
Name = "Test mocked relationship",
Specification = new Reference("SpecificationID", "SpecificationName"),
};
return(Task.FromResult(new ApiResponse <DefinitionSpecificationRelationship>(HttpStatusCode.OK, result)));
}
private async Task <DatasetSpecificationRelationshipViewModel> CreateViewModel(
DefinitionSpecificationRelationship relationship,
KeyValuePair <string, int> datasetLatestVersion)
{
DatasetSpecificationRelationshipViewModel relationshipViewModel = new DatasetSpecificationRelationshipViewModel
{
Id = relationship.Id,
Name = relationship.Name,
RelationshipDescription = relationship.Description,
IsProviderData = relationship.IsSetAsProviderData,
LastUpdatedAuthor = relationship.Author,
LastUpdatedDate = relationship.LastUpdated
};
if (relationship.DatasetVersion != null)
{
Dataset dataset = await _datasetRepository.GetDatasetByDatasetId(relationship.DatasetVersion.Id);
if (dataset != null)
{
relationshipViewModel.DatasetId = relationship.DatasetVersion.Id;
relationshipViewModel.DatasetName = dataset.Name;
relationshipViewModel.Version = relationship.DatasetVersion.Version;
relationshipViewModel.IsLatestVersion = relationship.DatasetVersion.Version == datasetLatestVersion.Value;
}
else
{
_logger.Warning($"Dataset could not be found for Id {relationship.DatasetVersion.Id}");
}
}
if (relationship.DatasetDefinition != null)
{
string definitionId = relationship.DatasetDefinition.Id;
Models.Datasets.Schema.DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(definitionId);
if (definition != null)
{
relationshipViewModel.Definition = new DatasetDefinitionViewModel
{
Id = definition.Id,
Name = definition.Name,
Description = definition.Description
};
}
}
return(relationshipViewModel);
}
public async Task <IActionResult> GetRelationshipsBySpecificationId(string specificationId)
{
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("No specification id was provided to GetRelationshipsBySpecificationId");
return(new BadRequestObjectResult("No specification id was provided to GetRelationshipsBySpecificationId"));
}
IEnumerable <DefinitionSpecificationRelationship> relationships =
await _datasetRepository.GetDefinitionSpecificationRelationshipsByQuery(m => m.Content.Specification.Id == specificationId);
if (relationships.IsNullOrEmpty())
{
relationships = new DefinitionSpecificationRelationship[0];
}
IList <DatasetSpecificationRelationshipViewModel> relationshipViewModels = new List <DatasetSpecificationRelationshipViewModel>();
if (relationships.IsNullOrEmpty())
{
return(new OkObjectResult(relationshipViewModels));
}
IList <Task <DatasetSpecificationRelationshipViewModel> > tasks = new List <Task <DatasetSpecificationRelationshipViewModel> >();
IEnumerable <KeyValuePair <string, int> > datasetLatestVersions =
await _datasetRepository.GetDatasetLatestVersions(relationships.Select(_ => _.DatasetVersion?.Id));
foreach (DefinitionSpecificationRelationship relationship in relationships)
{
KeyValuePair <string, int> datasetLatestVersion = datasetLatestVersions.SingleOrDefault(_ => _.Key == relationship.DatasetVersion?.Id);
Task <DatasetSpecificationRelationshipViewModel> task = CreateViewModel(relationship, datasetLatestVersion);
tasks.Add(task);
}
await Task.WhenAll(tasks);
foreach (Task <DatasetSpecificationRelationshipViewModel> task in tasks)
{
relationshipViewModels.Add(task.Result);
}
return(new OkObjectResult(tasks.Select(m => m.Result)));
}
public async Task <IActionResult> GetCurrentDatasetRelationshipFieldsBySpecificationId(string specificationId)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
IEnumerable <DefinitionSpecificationRelationship> relationships =
(await _datasetRepository.GetDefinitionSpecificationRelationshipsByQuery(m => m.Specification.Id == specificationId)).ToList();
if (relationships.IsNullOrEmpty())
{
relationships = new DefinitionSpecificationRelationship[0];
}
IEnumerable <string> definitionIds = relationships.Select(m => m.DatasetDefinition.Id);
IEnumerable <DatasetDefinition> definitions = await _datasetRepository.GetDatasetDefinitionsByQuery(d => definitionIds.Contains(d.Id));
IList <DatasetSchemaRelationshipModel> schemaRelationshipModels = new List <DatasetSchemaRelationshipModel>();
foreach (DefinitionSpecificationRelationship definitionSpecificationRelationship in relationships)
{
string relationshipName = definitionSpecificationRelationship.Name;
string datasetName = VisualBasicTypeGenerator.GenerateIdentifier(relationshipName);
DatasetDefinition datasetDefinition = definitions.FirstOrDefault(m => m.Id == definitionSpecificationRelationship.DatasetDefinition.Id);
schemaRelationshipModels.Add(new DatasetSchemaRelationshipModel
{
DefinitionId = datasetDefinition.Id,
RelationshipId = definitionSpecificationRelationship.Id,
RelationshipName = relationshipName,
Fields = datasetDefinition.TableDefinitions.SelectMany(m => m.FieldDefinitions.Select(f => new DatasetSchemaRelationshipField
{
Name = f.Name,
SourceName = VisualBasicTypeGenerator.GenerateIdentifier(f.Name),
SourceRelationshipName = datasetName,
IsAggregable = f.IsAggregable
}))
});
}
return(new OkObjectResult(schemaRelationshipModels));
}
public async Task <IActionResult> GetCurrentRelationshipsBySpecificationIdAndDatasetDefinitionId(string specificationId, string datasetDefinitionId)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.IsNullOrWhiteSpace(datasetDefinitionId, nameof(datasetDefinitionId));
IEnumerable <DefinitionSpecificationRelationship> relationships =
await _datasetRepository.GetDefinitionSpecificationRelationshipsByQuery(m => m.Specification.Id == specificationId && m.DatasetDefinition.Id == datasetDefinitionId);
if (relationships.IsNullOrEmpty())
{
relationships = new DefinitionSpecificationRelationship[0];
}
IList <DatasetSpecificationRelationshipViewModel> relationshipViewModels = new List <DatasetSpecificationRelationshipViewModel>();
if (relationships.IsNullOrEmpty())
{
return(new OkObjectResult(relationshipViewModels));
}
IList <Task <DatasetSpecificationRelationshipViewModel> > tasks = new List <Task <DatasetSpecificationRelationshipViewModel> >();
foreach (DefinitionSpecificationRelationship relationship in relationships)
{
Task <DatasetSpecificationRelationshipViewModel> task = CreateViewModel(relationship);
tasks.Add(task);
}
await Task.WhenAll(tasks);
foreach (Task <DatasetSpecificationRelationshipViewModel> task in tasks)
{
relationshipViewModels.Add(task.Result);
}
return(new OkObjectResult(tasks.Select(m => m.Result)));
}
public async Task <IActionResult> GetRelationshipsBySpecificationId(HttpRequest request)
{
request.Query.TryGetValue("specificationId", out Microsoft.Extensions.Primitives.StringValues specId);
string specificationId = specId.FirstOrDefault();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("No specification id was provided to GetRelationshipsBySpecificationId");
return(new BadRequestObjectResult("No specification id was provided to GetRelationshipsBySpecificationId"));
}
IEnumerable <DefinitionSpecificationRelationship> relationships =
await _datasetRepository.GetDefinitionSpecificationRelationshipsByQuery(m => m.Specification.Id == specificationId);
if (relationships.IsNullOrEmpty())
{
relationships = new DefinitionSpecificationRelationship[0];
}
return(new OkObjectResult(relationships));
}
public CreateDefinitionSpecificationRelationshipModelValidator(IDatasetRepository datasetRepository)
{
_datasetRepository = datasetRepository;
RuleFor(model => model.DatasetDefinitionId)
.NotEmpty()
.WithMessage("Missing dataset definition id.");
RuleFor(model => model.SpecificationId)
.NotEmpty()
.WithMessage("Missing specification id.");
RuleFor(model => model.Name)
.Custom((name, context) => {
CreateDefinitionSpecificationRelationshipModel relationshipModel = context.ParentContext.InstanceToValidate as CreateDefinitionSpecificationRelationshipModel;
if (string.IsNullOrWhiteSpace(relationshipModel.Name))
{
context.AddFailure("Missing name provided");
}
else
{
if (!string.IsNullOrWhiteSpace(relationshipModel.SpecificationId))
{
DefinitionSpecificationRelationship relationship = _datasetRepository.GetRelationshipBySpecificationIdAndName(relationshipModel.SpecificationId, relationshipModel.Name).Result;
if (relationship != null)
{
context.AddFailure($"You must give a unique relationship name");
}
}
}
});
RuleFor(model => model.Description)
.NotEmpty()
.WithMessage("Missing description provided.");
}
public async Task <IActionResult> AssignDatasourceVersionToRelationship(AssignDatasourceModel model, Reference user, string correlationId)
{
if (model == null)
{
_logger.Error("Null AssignDatasourceModel was provided to AssignDatasourceVersionToRelationship");
return(new BadRequestObjectResult("Null AssignDatasourceModel was provided"));
}
Dataset dataset = await _datasetRepository.GetDatasetByDatasetId(model.DatasetId);
if (dataset == null)
{
_logger.Error($"Dataset not found for dataset id: {model.DatasetId}");
return(new StatusCodeResult(412));
}
DefinitionSpecificationRelationship relationship = await _datasetRepository.GetDefinitionSpecificationRelationshipById(model.RelationshipId);
if (relationship == null)
{
_logger.Error($"Relationship not found for relationship id: {model.RelationshipId}");
return(new StatusCodeResult(412));
}
relationship.Author = user;
relationship.LastUpdated = _dateTimeProvider.UtcNow;
relationship.DatasetVersion = new DatasetRelationshipVersion
{
Id = model.DatasetId,
Version = model.Version
};
HttpStatusCode statusCode = await _datasetRepository.UpdateDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to assign data source to relationship : {model.RelationshipId} with status code {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IEnumerable <CalculationResponseModel> allCalculations = await _calcsRepository.GetCurrentCalculationsBySpecificationId(relationship.Specification.Id);
bool generateCalculationAggregations = !allCalculations.IsNullOrEmpty() &&
SourceCodeHelpers.HasCalculationAggregateFunctionParameters(allCalculations.Select(m => m.SourceCode));
Trigger trigger = new Trigger
{
EntityId = dataset.Id,
EntityType = nameof(Dataset),
Message = $"Mapping dataset: '{dataset.Id}'"
};
JobCreateModel job = new JobCreateModel
{
InvokerUserDisplayName = user?.Name,
InvokerUserId = user?.Id,
JobDefinitionId = JobConstants.DefinitionNames.MapDatasetJob,
MessageBody = JsonConvert.SerializeObject(dataset),
Properties = new Dictionary <string, string>
{
{ "specification-id", relationship.Specification.Id },
{ "relationship-id", relationship.Id },
{ "user-id", user?.Id },
{ "user-name", user?.Name },
},
SpecificationId = relationship.Specification.Id,
Trigger = trigger,
CorrelationId = correlationId
};
Job parentJob = null;
if (relationship.IsSetAsProviderData)
{
string parentJobDefinition = generateCalculationAggregations ?
JobConstants.DefinitionNames.MapScopedDatasetJobWithAggregation :
JobConstants.DefinitionNames.MapScopedDatasetJob;
parentJob = await _jobManagement.QueueJob(new JobCreateModel
{
InvokerUserDisplayName = user?.Name,
InvokerUserId = user?.Id,
JobDefinitionId = parentJobDefinition,
Properties = new Dictionary <string, string>
{
{ "specification-id", relationship.Specification.Id },
{ "provider-cache-key", $"{CacheKeys.ScopedProviderSummariesPrefix}{relationship.Specification.Id}" },
{ "specification-summary-cache-key", $"{CacheKeys.SpecificationSummaryById}{relationship.Specification.Id}" }
},
SpecificationId = relationship.Specification.Id,
Trigger = trigger,
CorrelationId = correlationId
});
}
if (parentJob != null)
{
job.ParentJobId = parentJob.Id;
job.Properties.Add("parentJobId", parentJob.Id);
}
await _jobManagement.QueueJob(job);
return(new NoContentResult());
}
public async Task ProcessDataset(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
IDictionary <string, object> properties = message.UserProperties;
Dataset dataset = message.GetPayloadAsInstanceOf <Dataset>();
string jobId = message.UserProperties["jobId"].ToString();
await UpdateJobStatus(jobId, null, 0);
if (dataset == null)
{
_logger.Error("A null dataset was provided to ProcessData");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - null dataset provided");
return;
}
if (!message.UserProperties.ContainsKey("specification-id"))
{
_logger.Error("Specification Id key is missing in ProcessDataset message properties");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - specification id not provided");
return;
}
string specificationId = message.UserProperties["specification-id"].ToString();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("A null or empty specification id was provided to ProcessData");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - specification if is null or empty");
return;
}
if (!message.UserProperties.ContainsKey("relationship-id"))
{
_logger.Error("Relationship Id key is missing in ProcessDataset message properties");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship id not provided");
return;
}
string relationshipId = message.UserProperties["relationship-id"].ToString();
if (string.IsNullOrWhiteSpace(relationshipId))
{
_logger.Error("A null or empty relationship id was provided to ProcessDataset");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship id is null or empty");
return;
}
DefinitionSpecificationRelationship relationship = await _datasetRepository.GetDefinitionSpecificationRelationshipById(relationshipId);
if (relationship == null)
{
_logger.Error($"Relationship not found for relationship id: {relationshipId}");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship not found");
return;
}
BuildProject buildProject = null;
Reference user = message.GetUserDetails();
try
{
buildProject = await ProcessDataset(dataset, specificationId, relationshipId, relationship.DatasetVersion.Version, user);
await UpdateJobStatus(jobId, true, 100, "Processed Dataset");
}
catch (NonRetriableException argEx)
{
// This type of exception is not retriable so fail
_logger.Error(argEx, $"Failed to run ProcessDataset with exception: {argEx.Message} for relationship ID '{relationshipId}'");
await UpdateJobStatus(jobId, false, 100, $"Failed to run Process - {argEx.Message}");
return;
}
catch (Exception exception)
{
// Unknown exception occurred so allow it to be retried
_logger.Error(exception, $"Failed to run ProcessDataset with exception: {exception.Message} for relationship ID '{relationshipId}'");
throw;
}
if (buildProject != null && !buildProject.DatasetRelationships.IsNullOrEmpty() && buildProject.DatasetRelationships.Any(m => m.DefinesScope))
{
string userId = user != null ? user.Id : "";
string userName = user != null ? user.Name : "";
try
{
HttpStatusCode statusCode = await _calcsRepository.CompileAndSaveAssembly(specificationId);
if (!statusCode.IsSuccess())
{
string errorMessage = $"Failed to compile and save assembly for specification id '{specificationId}' with status code '{statusCode}'";
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
Trigger trigger = new Trigger
{
EntityId = relationshipId,
EntityType = nameof(DefinitionSpecificationRelationship),
Message = $"Processed dataset relationship: '{relationshipId}' for specification: '{specificationId}'"
};
string correlationId = message.GetCorrelationId();
IEnumerable <CalculationCurrentVersion> allCalculations = await _calcsRepository.GetCurrentCalculationsBySpecificationId(specificationId);
bool generateCalculationAggregations = allCalculations.IsNullOrEmpty() ? false : SourceCodeHelpers.HasCalculationAggregateFunctionParameters(allCalculations.Select(m => m.SourceCode));
await SendInstructAllocationsToJobService($"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}", specificationId, userId, userName, trigger, correlationId, generateCalculationAggregations);
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specificationId}'");
throw new Exception($"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specificationId}'", ex);
}
}
}
public async Task <IActionResult> CreateRelationship(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json);
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId);
if (specification == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = request.BuildMessageProperties();
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}
public async Task <IActionResult> AssignDatasourceVersionToRelationship(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
AssignDatasourceModel model = JsonConvert.DeserializeObject <AssignDatasourceModel>(json);
if (model == null)
{
_logger.Error("Null AssignDatasourceModel was provided to AssignDatasourceVersionToRelationship");
return(new BadRequestObjectResult("Null AssignDatasourceModel was provided"));
}
Dataset dataset = await _datasetRepository.GetDatasetByDatasetId(model.DatasetId);
if (dataset == null)
{
_logger.Error($"Dataset not found for dataset id: {model.DatasetId}");
return(new StatusCodeResult(412));
}
DefinitionSpecificationRelationship relationship = await _datasetRepository.GetDefinitionSpecificationRelationshipById(model.RelationshipId);
if (relationship == null)
{
_logger.Error($"Relationship not found for relationship id: {model.RelationshipId}");
return(new StatusCodeResult(412));
}
relationship.DatasetVersion = new DatasetRelationshipVersion
{
Id = model.DatasetId,
Version = model.Version
};
HttpStatusCode statusCode = await _datasetRepository.UpdateDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to assign data source to relationship : {model.RelationshipId} with status code {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
Reference user = request.GetUser();
Trigger trigger = new Trigger
{
EntityId = dataset.Id,
EntityType = nameof(Dataset),
Message = $"Mapping dataset: '{dataset.Id}'"
};
string correlationId = request.GetCorrelationId();
JobCreateModel job = new JobCreateModel
{
InvokerUserDisplayName = user.Name,
InvokerUserId = user.Id,
JobDefinitionId = JobConstants.DefinitionNames.MapDatasetJob,
MessageBody = JsonConvert.SerializeObject(dataset),
Properties = new Dictionary <string, string>
{
{ "specification-id", relationship.Specification.Id },
{ "relationship-id", relationship.Id }
},
SpecificationId = relationship.Specification.Id,
Trigger = trigger,
CorrelationId = correlationId
};
await _jobsApiClientPolicy.ExecuteAsync(() => _jobsApiClient.CreateJob(job));
return(new NoContentResult());
}
public Task <HttpStatusCode> UpdateDefinitionSpecificationRelationship(DefinitionSpecificationRelationship relationship)
{
return(_cosmosRepository.UpdateAsync(relationship));
}
public async Task <IActionResult> CreateRelationship(CreateDefinitionSpecificationRelationshipModel model, Reference author, string correlationId)
{
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(model.SpecificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
SpecModel.SpecificationSummary specification = specificationApiResponse.Content;
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations,
Author = author,
LastUpdated = _dateTimeProvider.UtcNow,
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, author);
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}