public async Task GetDatasetSchemaSasUrl_GivenModelAndDatasetNameContainsSlashes_ReplacesSlashesWithUnderscoreAndReturnsUrl() { // Arrange DatasetSchemaSasUrlRequestModel model = new DatasetSchemaSasUrlRequestModel { DatasetDefinitionId = "12345" }; string json = JsonConvert.SerializeObject(model); byte[] byteArray = Encoding.UTF8.GetBytes(json); MemoryStream stream = new MemoryStream(byteArray); HttpRequest request = Substitute.For <HttpRequest>(); request .Body .Returns(stream); IBlobClient blobClient = CreateBlobClient(); DatasetDefinition datasetDefinition = new DatasetDefinition() { Id = "12345", Name = "TEST/SLASH Definition", }; IDatasetRepository datasetRepository = CreateDataSetsRepository(); datasetRepository .GetDatasetDefinition(Arg.Is(model.DatasetDefinitionId)) .Returns(datasetDefinition); DefinitionsService definitionsService = CreateDefinitionsService( datasetsRepository: datasetRepository, blobClient: blobClient); // Act IActionResult result = await definitionsService.GetDatasetSchemaSasUrl(request); // Assert result .Should() .BeOfType <OkObjectResult>(); blobClient .Received(1) .GetBlobSasUrl(Arg.Is("schemas/TEST_SLASH Definition.xlsx"), Arg.Any <DateTimeOffset>(), Arg.Any <SharedAccessBlobPermissions>()); }
public async Task SaveDefinition_GivenValidYamlAndDoesContainsExistingItemWithNoModelUpdates_ThenDoesNotAddMessageToTopicAndReturnsOK() { //Arrange string yaml = CreateRawDefinition(); string definitionId = "9183"; byte[] byteArray = Encoding.UTF8.GetBytes(yaml); MemoryStream stream = new MemoryStream(byteArray); IHeaderDictionary headerDictionary = new HeaderDictionary(); headerDictionary .Add("yaml-file", new StringValues(yamlFile)); HttpRequest request = Substitute.For <HttpRequest>(); request .Headers .Returns(headerDictionary); request .Body .Returns(stream); ILogger logger = CreateLogger(); HttpStatusCode statusCode = HttpStatusCode.Created; DatasetDefinition existingDatasetDefinition = new DatasetDefinition { Id = definitionId }; IDatasetRepository datasetsRepository = CreateDataSetsRepository(); datasetsRepository .SaveDefinition(Arg.Any <DatasetDefinition>()) .Returns(statusCode); datasetsRepository .GetDatasetDefinition(Arg.Is(definitionId)) .Returns(existingDatasetDefinition); byte[] excelAsBytes = new byte[100]; IExcelWriter <DatasetDefinition> excelWriter = CreateExcelWriter(); excelWriter .Write(Arg.Any <DatasetDefinition>()) .Returns(excelAsBytes); ICloudBlob blob = Substitute.For <ICloudBlob>(); IBlobClient blobClient = CreateBlobClient(); blobClient .GetBlockBlobReference(Arg.Any <string>()) .Returns(blob); DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges(); IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService(); definitionChangesDetectionService .DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Is(existingDatasetDefinition)) .Returns(datasetDefinitionChanges); IMessengerService messengerService = CreateMessengerService(); DefinitionsService service = CreateDefinitionsService( logger, datasetsRepository, excelWriter: excelWriter, blobClient: blobClient, definitionChangesDetectionService: definitionChangesDetectionService, messengerService: messengerService); //Act IActionResult result = await service.SaveDefinition(request); //Assert result .Should() .BeOfType <OkResult>(); await messengerService .DidNotReceive() .SendToTopic( Arg.Is(ServiceBusConstants.TopicNames.DataDefinitionChanges), Arg.Any <DatasetDefinitionChanges>(), Arg.Any <IDictionary <string, string> >()); }
async public Task SaveDefinition_GivenUpdatedYamlWithChangedIdentifierTypeFieldButAlreadyUsedInRelationship_ReturnsBadRequest() { //Arrange IEnumerable <string> specificationIds = new[] { "spec-1" }; string definitionId = "9183"; string yaml = CreateRawDefinition(); byte[] byteArray = Encoding.UTF8.GetBytes(yaml); MemoryStream stream = new MemoryStream(byteArray); IHeaderDictionary headerDictionary = new HeaderDictionary(); headerDictionary .Add("yaml-file", new StringValues(yamlFile)); HttpRequest request = Substitute.For <HttpRequest>(); request .Headers .Returns(headerDictionary); request .Body .Returns(stream); ILogger logger = CreateLogger(); IDatasetRepository datasetRepository = CreateDataSetsRepository(); datasetRepository .GetDistinctRelationshipSpecificationIdsForDatasetDefinitionId(Arg.Is(definitionId)) .Returns(specificationIds); datasetRepository .GetDatasetDefinition(Arg.Is(definitionId)) .Returns(new DatasetDefinition()); DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges { Id = definitionId, }; FieldDefinitionChanges fieldDefinitionChanges = new FieldDefinitionChanges(); fieldDefinitionChanges.ChangeTypes.Add(FieldDefinitionChangeType.IdentifierType); TableDefinitionChanges tableDefinitionChanges = new TableDefinitionChanges(); tableDefinitionChanges.FieldChanges.Add(fieldDefinitionChanges); datasetDefinitionChanges.TableDefinitionChanges.Add(tableDefinitionChanges); IDefinitionChangesDetectionService definitionChangesDetectionService = CreateChangesDetectionService(); definitionChangesDetectionService .DetectChanges(Arg.Any <DatasetDefinition>(), Arg.Any <DatasetDefinition>()) .Returns(datasetDefinitionChanges); DefinitionsService service = CreateDefinitionsService(logger, definitionChangesDetectionService: definitionChangesDetectionService, datasetsRepository: datasetRepository); //Act IActionResult result = await service.SaveDefinition(request); //Assert result .Should() .BeOfType <BadRequestObjectResult>() .Which .Value .Should() .Be("Unable to change provider identifier as there are currently relationships setup against this schema"); }
public async Task GetDatasetSchemaSasUrl_GivenModelAndDatasetNameDoesNotContainSlashes_GetSasUrl() { //Arrange const string sasUrl = "https://wherever.naf?jhjhjhjhjhhjhjhjhjjhj"; DatasetSchemaSasUrlRequestModel model = new DatasetSchemaSasUrlRequestModel { DatasetDefinitionId = "12345" }; string json = JsonConvert.SerializeObject(model); byte[] byteArray = Encoding.UTF8.GetBytes(json); MemoryStream stream = new MemoryStream(byteArray); HttpRequest request = Substitute.For <HttpRequest>(); request .Body .Returns(stream); IBlobClient blobClient = CreateBlobClient(); blobClient .GetBlobSasUrl(Arg.Any <string>(), Arg.Any <DateTimeOffset>(), Arg.Any <SharedAccessBlobPermissions>()) .Returns(sasUrl); DatasetDefinition datasetDefinition = new DatasetDefinition() { Id = "12345", Name = "14 15", }; IDatasetRepository datasetRepository = CreateDataSetsRepository(); datasetRepository .GetDatasetDefinition(Arg.Is(model.DatasetDefinitionId)) .Returns(datasetDefinition); DefinitionsService definitionsService = CreateDefinitionsService( datasetsRepository: datasetRepository, blobClient: blobClient); //Act IActionResult result = await definitionsService.GetDatasetSchemaSasUrl(request); //Assert result .Should() .BeOfType <OkObjectResult>(); OkObjectResult okObjectResult = result as OkObjectResult; DatasetSchemaSasUrlResponseModel responseModel = okObjectResult.Value as DatasetSchemaSasUrlResponseModel; responseModel .SchemaUrl .Should() .Be(sasUrl); blobClient .Received(1) .GetBlobSasUrl(Arg.Is("schemas/14 15.xlsx"), Arg.Any <DateTimeOffset>(), Arg.Any <SharedAccessBlobPermissions>()); }
async public Task <IActionResult> SaveDefinition(HttpRequest request) { string yaml = await request.GetRawBodyStringAsync(); string yamlFilename = request.GetYamlFileNameFromRequest(); if (string.IsNullOrEmpty(yaml)) { _logger.Error($"Null or empty yaml provided for file: {yamlFilename}"); return(new BadRequestObjectResult($"Invalid yaml was provided for file: {yamlFilename}")); } var deserializer = new DeserializerBuilder() .WithNamingConvention(new CamelCaseNamingConvention()) .Build(); DatasetDefinition definition = null; try { definition = deserializer.Deserialize <DatasetDefinition>(yaml); } catch (Exception exception) { _logger.Error(exception, $"Invalid yaml was provided for file: {yamlFilename}"); return(new BadRequestObjectResult($"Invalid yaml was provided for file: {yamlFilename}")); } DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges(); DatasetDefinition existingDefinition = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(definition.Id)); if (existingDefinition != null) { datasetDefinitionChanges = _definitionChangesDetectionService.DetectChanges(definition, existingDefinition); IEnumerable <string> relationships = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDistinctRelationshipSpecificationIdsForDatasetDefinitionId(datasetDefinitionChanges.Id)); IEnumerable <FieldDefinitionChanges> fieldDefinitionChanges = datasetDefinitionChanges.TableDefinitionChanges.SelectMany(m => m.FieldChanges); if (!relationships.IsNullOrEmpty() && !fieldDefinitionChanges.IsNullOrEmpty()) { if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.RemovedField))) { return(new BadRequestObjectResult("Unable to remove a field as there are currently relationships setup against this schema")); } if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.IdentifierType))) { return(new BadRequestObjectResult("Unable to change provider identifier as there are currently relationships setup against this schema")); } } } try { HttpStatusCode result = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.SaveDefinition(definition)); if (!result.IsSuccess()) { int statusCode = (int)result; _logger.Error($"Failed to save yaml file: {yamlFilename} to cosmos db with status {statusCode}"); return(new StatusCodeResult(statusCode)); } await IndexDatasetDefinition(definition); } catch (Exception exception) { _logger.Error(exception, $"Exception occurred writing to yaml file: {yamlFilename} to cosmos db"); return(new InternalServerErrorResult($"Exception occurred writing to yaml file: {yamlFilename} to cosmos db")); } byte[] excelAsBytes = _excelWriter.Write(definition); if (excelAsBytes == null || excelAsBytes.Length == 0) { _logger.Error($"Failed to generate excel file for {definition.Name}"); return(new InternalServerErrorResult($"Failed to generate excel file for {definition.Name}")); } try { await SaveToBlobStorage(excelAsBytes, definition.Name); } catch (Exception ex) { return(new InternalServerErrorResult(ex.Message)); } _logger.Information($"Successfully saved file: {yamlFilename} to cosmos db"); if (existingDefinition != null && datasetDefinitionChanges.HasChanges) { IDictionary <string, string> properties = request.BuildMessageProperties(); await _messengerService.SendToTopic(ServiceBusConstants.TopicNames.DataDefinitionChanges, datasetDefinitionChanges, properties); } return(new OkResult()); }
public async Task <IActionResult> CreateRelationship(HttpRequest request) { string json = await request.GetRawBodyStringAsync(); CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json); if (model == null) { _logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship"); return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided")); } BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState(); if (validationResult != null) { return(validationResult); } DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId); if (definition == null) { _logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}"); return(new StatusCodeResult(412)); } SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId); if (specification == null) { _logger.Error($"Specification was not found for id {model.SpecificationId}"); return(new StatusCodeResult(412)); } string relationshipId = Guid.NewGuid().ToString(); DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship { Name = model.Name, DatasetDefinition = new Reference(definition.Id, definition.Name), Specification = new Reference(specification.Id, specification.Name), Description = model.Description, Id = relationshipId, IsSetAsProviderData = model.IsSetAsProviderData, UsedInDataAggregations = model.UsedInDataAggregations }; HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } IDictionary <string, string> properties = request.BuildMessageProperties(); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification, new AssignDefinitionRelationshipMessage { SpecificationId = specification.Id, RelationshipId = relationshipId }, properties); DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary { Name = relationship.Name, Id = Guid.NewGuid().ToString(), Relationship = new Reference(relationship.Id, relationship.Name), DatasetDefinition = definition, DatasetDefinitionId = definition.Id, DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider, DefinesScope = relationship.IsSetAsProviderData }; BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary); await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}"); return(new OkObjectResult(relationship)); }
public async Task <IActionResult> GetDatasetDefinitionById(string datasetDefinitionId) { if (string.IsNullOrWhiteSpace(datasetDefinitionId)) { _logger.Error("No datasetDefinitionId was provided to GetDatasetDefinitionById"); return(new BadRequestObjectResult("Null or empty datasetDefinitionId provided")); } DatasetDefinition defintion = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(datasetDefinitionId)); if (defintion == null) { return(new NotFoundResult()); } return(new OkObjectResult(defintion)); }
public async Task <IActionResult> CreateRelationship(CreateDefinitionSpecificationRelationshipModel model, Reference author, string correlationId) { if (model == null) { _logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship"); return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided")); } BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState(); if (validationResult != null) { return(validationResult); } DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId); if (definition == null) { _logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}"); return(new StatusCodeResult(412)); } ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(model.SpecificationId)); if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null) { _logger.Error($"Specification was not found for id {model.SpecificationId}"); return(new StatusCodeResult(412)); } SpecModel.SpecificationSummary specification = specificationApiResponse.Content; string relationshipId = Guid.NewGuid().ToString(); DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship { Name = model.Name, DatasetDefinition = new Reference(definition.Id, definition.Name), Specification = new Reference(specification.Id, specification.Name), Description = model.Description, Id = relationshipId, IsSetAsProviderData = model.IsSetAsProviderData, UsedInDataAggregations = model.UsedInDataAggregations, Author = author, LastUpdated = _dateTimeProvider.UtcNow, }; HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, author); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification, new AssignDefinitionRelationshipMessage { SpecificationId = specification.Id, RelationshipId = relationshipId }, properties); DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary { Name = relationship.Name, Id = Guid.NewGuid().ToString(), Relationship = new Reference(relationship.Id, relationship.Name), DatasetDefinition = definition, DatasetDefinitionId = definition.Id, DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider, DefinesScope = relationship.IsSetAsProviderData }; await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary); await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}"); return(new OkObjectResult(relationship)); }