public async Task GetDatasetSchemaSasUrl_GivenModelAndDatasetNameContainsSlashes_ReplacesSlashesWithUnderscoreAndReturnsUrl()
{
// Arrange
DatasetSchemaSasUrlRequestModel model = new DatasetSchemaSasUrlRequestModel
{
DatasetDefinitionId = "12345"
};
string json = JsonConvert.SerializeObject(model);
byte[] byteArray = Encoding.UTF8.GetBytes(json);
MemoryStream stream = new MemoryStream(byteArray);
HttpRequest request = Substitute.For <HttpRequest>();
request
.Body
.Returns(stream);
IBlobClient blobClient = CreateBlobClient();
DatasetDefinition datasetDefinition = new DatasetDefinition()
{
Id = "12345",
Name = "TEST/SLASH Definition",
};
IDatasetRepository datasetRepository = CreateDataSetsRepository();
datasetRepository
.GetDatasetDefinition(Arg.Is(model.DatasetDefinitionId))
.Returns(datasetDefinition);
DefinitionsService definitionsService = CreateDefinitionsService(
datasetsRepository: datasetRepository,
blobClient: blobClient);
// Act
IActionResult result = await definitionsService.GetDatasetSchemaSasUrl(request);
// Assert
result
.Should()
.BeOfType <OkObjectResult>();
blobClient
.Received(1)
.GetBlobSasUrl(Arg.Is("schemas/TEST_SLASH Definition.xlsx"), Arg.Any <DateTimeOffset>(), Arg.Any <SharedAccessBlobPermissions>());
}
public async Task SaveDefinition_GivenValidYamlAndDoesContainsExistingItemWithNoModelUpdates_ThenDoesNotAddMessageToTopicAndReturnsOK()
{
//Arrange
string yaml = CreateRawDefinition();
string definitionId = "9183";
byte[] byteArray = Encoding.UTF8.GetBytes(yaml);
MemoryStream stream = new MemoryStream(byteArray);
IHeaderDictionary headerDictionary = new HeaderDictionary();
headerDictionary
.Add("yaml-file", new StringValues(yamlFile));
HttpRequest request = Substitute.For <HttpRequest>();
request
.Headers
.Returns(headerDictionary);
request
.Body
.Returns(stream);
ILogger logger = CreateLogger();
HttpStatusCode statusCode = HttpStatusCode.Created;
DatasetDefinition existingDatasetDefinition = new DatasetDefinition
{
Id = definitionId
};
IDatasetRepository datasetsRepository = CreateDataSetsRepository();
datasetsRepository
.SaveDefinition(Arg.Any <DatasetDefinition>())
.Returns(statusCode);
datasetsRepository
.GetDatasetDefinition(Arg.Is(definitionId))
.Returns(existingDatasetDefinition);
byte[] excelAsBytes = new byte[100];
IExcelWriter <DatasetDefinition> excelWriter = CreateExcelWriter();
excelWriter
.Write(Arg.Any <DatasetDefinition>())
.Returns(excelAsBytes);
ICloudBlob blob = Substitute.For <ICloudBlob>();
IBlobClient blobClient = CreateBlobClient();
blobClient
.GetBlockBlobReference(Arg.Any <string>())
.Returns(blob);
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService();
definitionChangesDetectionService
.DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Is(existingDatasetDefinition))
.Returns(datasetDefinitionChanges);
IMessengerService messengerService = CreateMessengerService();
DefinitionsService service = CreateDefinitionsService(
logger,
datasetsRepository,
excelWriter: excelWriter,
blobClient: blobClient,
definitionChangesDetectionService: definitionChangesDetectionService,
messengerService: messengerService);
//Act
IActionResult result = await service.SaveDefinition(request);
//Assert
result
.Should()
.BeOfType <OkResult>();
await
messengerService
.DidNotReceive()
.SendToTopic(
Arg.Is(ServiceBusConstants.TopicNames.DataDefinitionChanges),
Arg.Any <DatasetDefinitionChanges>(),
Arg.Any <IDictionary <string, string> >());
}
async public Task SaveDefinition_GivenUpdatedYamlWithChangedIdentifierTypeFieldButAlreadyUsedInRelationship_ReturnsBadRequest()
{
//Arrange
IEnumerable <string> specificationIds = new[] { "spec-1" };
string definitionId = "9183";
string yaml = CreateRawDefinition();
byte[] byteArray = Encoding.UTF8.GetBytes(yaml);
MemoryStream stream = new MemoryStream(byteArray);
IHeaderDictionary headerDictionary = new HeaderDictionary();
headerDictionary
.Add("yaml-file", new StringValues(yamlFile));
HttpRequest request = Substitute.For <HttpRequest>();
request
.Headers
.Returns(headerDictionary);
request
.Body
.Returns(stream);
ILogger logger = CreateLogger();
IDatasetRepository datasetRepository = CreateDataSetsRepository();
datasetRepository
.GetDistinctRelationshipSpecificationIdsForDatasetDefinitionId(Arg.Is(definitionId))
.Returns(specificationIds);
datasetRepository
.GetDatasetDefinition(Arg.Is(definitionId))
.Returns(new DatasetDefinition());
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges
{
Id = definitionId,
};
FieldDefinitionChanges fieldDefinitionChanges = new FieldDefinitionChanges();
fieldDefinitionChanges.ChangeTypes.Add(FieldDefinitionChangeType.IdentifierType);
TableDefinitionChanges tableDefinitionChanges = new TableDefinitionChanges();
tableDefinitionChanges.FieldChanges.Add(fieldDefinitionChanges);
datasetDefinitionChanges.TableDefinitionChanges.Add(tableDefinitionChanges);
IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService();
definitionChangesDetectionService
.DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Any <DatasetDefinition>())
.Returns(datasetDefinitionChanges);
DefinitionsService service = CreateDefinitionsService(logger, definitionChangesDetectionService: definitionChangesDetectionService, datasetsRepository: datasetRepository);
//Act
IActionResult result = await service.SaveDefinition(request);
//Assert
result
.Should()
.BeOfType <BadRequestObjectResult>()
.Which
.Value
.Should()
.Be("Unable to change provider identifier as there are currently relationships setup against this schema");
}
public async Task GetDatasetSchemaSasUrl_GivenModelAndDatasetNameDoesNotContainSlashes_GetSasUrl()
{
//Arrange
const string sasUrl = "https://wherever.naf?jhjhjhjhjhhjhjhjhjjhj";
DatasetSchemaSasUrlRequestModel model = new DatasetSchemaSasUrlRequestModel
{
DatasetDefinitionId = "12345"
};
string json = JsonConvert.SerializeObject(model);
byte[] byteArray = Encoding.UTF8.GetBytes(json);
MemoryStream stream = new MemoryStream(byteArray);
HttpRequest request = Substitute.For <HttpRequest>();
request
.Body
.Returns(stream);
IBlobClient blobClient = CreateBlobClient();
blobClient
.GetBlobSasUrl(Arg.Any <string>(), Arg.Any <DateTimeOffset>(), Arg.Any <SharedAccessBlobPermissions>())
.Returns(sasUrl);
DatasetDefinition datasetDefinition = new DatasetDefinition()
{
Id = "12345",
Name = "14 15",
};
IDatasetRepository datasetRepository = CreateDataSetsRepository();
datasetRepository
.GetDatasetDefinition(Arg.Is(model.DatasetDefinitionId))
.Returns(datasetDefinition);
DefinitionsService definitionsService = CreateDefinitionsService(
datasetsRepository: datasetRepository,
blobClient: blobClient);
//Act
IActionResult result = await definitionsService.GetDatasetSchemaSasUrl(request);
//Assert
result
.Should()
.BeOfType <OkObjectResult>();
OkObjectResult okObjectResult = result as OkObjectResult;
DatasetSchemaSasUrlResponseModel responseModel = okObjectResult.Value as DatasetSchemaSasUrlResponseModel;
responseModel
.SchemaUrl
.Should()
.Be(sasUrl);
blobClient
.Received(1)
.GetBlobSasUrl(Arg.Is("schemas/14 15.xlsx"), Arg.Any <DateTimeOffset>(), Arg.Any <SharedAccessBlobPermissions>());
}
async public Task <IActionResult> SaveDefinition(HttpRequest request)
{
string yaml = await request.GetRawBodyStringAsync();
string yamlFilename = request.GetYamlFileNameFromRequest();
if (string.IsNullOrEmpty(yaml))
{
_logger.Error($"Null or empty yaml provided for file: {yamlFilename}");
return(new BadRequestObjectResult($"Invalid yaml was provided for file: {yamlFilename}"));
}
var deserializer = new DeserializerBuilder()
.WithNamingConvention(new CamelCaseNamingConvention())
.Build();
DatasetDefinition definition = null;
try
{
definition = deserializer.Deserialize <DatasetDefinition>(yaml);
}
catch (Exception exception)
{
_logger.Error(exception, $"Invalid yaml was provided for file: {yamlFilename}");
return(new BadRequestObjectResult($"Invalid yaml was provided for file: {yamlFilename}"));
}
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
DatasetDefinition existingDefinition = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(definition.Id));
if (existingDefinition != null)
{
datasetDefinitionChanges = _definitionChangesDetectionService.DetectChanges(definition, existingDefinition);
IEnumerable <string> relationships = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDistinctRelationshipSpecificationIdsForDatasetDefinitionId(datasetDefinitionChanges.Id));
IEnumerable <FieldDefinitionChanges> fieldDefinitionChanges = datasetDefinitionChanges.TableDefinitionChanges.SelectMany(m => m.FieldChanges);
if (!relationships.IsNullOrEmpty() && !fieldDefinitionChanges.IsNullOrEmpty())
{
if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.RemovedField)))
{
return(new BadRequestObjectResult("Unable to remove a field as there are currently relationships setup against this schema"));
}
if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.IdentifierType)))
{
return(new BadRequestObjectResult("Unable to change provider identifier as there are currently relationships setup against this schema"));
}
}
}
try
{
HttpStatusCode result = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.SaveDefinition(definition));
if (!result.IsSuccess())
{
int statusCode = (int)result;
_logger.Error($"Failed to save yaml file: {yamlFilename} to cosmos db with status {statusCode}");
return(new StatusCodeResult(statusCode));
}
await IndexDatasetDefinition(definition);
}
catch (Exception exception)
{
_logger.Error(exception, $"Exception occurred writing to yaml file: {yamlFilename} to cosmos db");
return(new InternalServerErrorResult($"Exception occurred writing to yaml file: {yamlFilename} to cosmos db"));
}
byte[] excelAsBytes = _excelWriter.Write(definition);
if (excelAsBytes == null || excelAsBytes.Length == 0)
{
_logger.Error($"Failed to generate excel file for {definition.Name}");
return(new InternalServerErrorResult($"Failed to generate excel file for {definition.Name}"));
}
try
{
await SaveToBlobStorage(excelAsBytes, definition.Name);
}
catch (Exception ex)
{
return(new InternalServerErrorResult(ex.Message));
}
_logger.Information($"Successfully saved file: {yamlFilename} to cosmos db");
if (existingDefinition != null && datasetDefinitionChanges.HasChanges)
{
IDictionary <string, string> properties = request.BuildMessageProperties();
await _messengerService.SendToTopic(ServiceBusConstants.TopicNames.DataDefinitionChanges, datasetDefinitionChanges, properties);
}
return(new OkResult());
}
public async Task <IActionResult> CreateRelationship(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json);
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId);
if (specification == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = request.BuildMessageProperties();
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}
public async Task <IActionResult> GetDatasetDefinitionById(string datasetDefinitionId)
{
if (string.IsNullOrWhiteSpace(datasetDefinitionId))
{
_logger.Error("No datasetDefinitionId was provided to GetDatasetDefinitionById");
return(new BadRequestObjectResult("Null or empty datasetDefinitionId provided"));
}
DatasetDefinition defintion = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(datasetDefinitionId));
if (defintion == null)
{
return(new NotFoundResult());
}
return(new OkObjectResult(defintion));
}
public async Task <IActionResult> CreateRelationship(CreateDefinitionSpecificationRelationshipModel model, Reference author, string correlationId)
{
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(model.SpecificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
SpecModel.SpecificationSummary specification = specificationApiResponse.Content;
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations,
Author = author,
LastUpdated = _dateTimeProvider.UtcNow,
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, author);
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}