public void Validate_GivenInvalidModelWithCsvFile_ValidIsFalse() { //Arrange CreateNewDatasetModel model = CreateModel(); model.Filename = "filename.csv"; CreateNewDatasetModelValidator validator = CreateValidator(); //Act ValidationResult result = validator.Validate(model); //Assert result .IsValid .Should() .BeFalse(); result .Errors .Count .Should().Be(1); result .Errors[0] .ErrorMessage .Should().Contain("Check you have the right file format"); }
public void Validate_GivenNameAlreadyExists_ValidIsFalse() { //Arrange CreateNewDatasetModel model = CreateModel(); IEnumerable <Dataset> datasets = new[] { new Dataset() }; IDatasetRepository repository = CreateDatasetsRepository(true, true); repository .GetDatasetsByQuery(Arg.Any <Expression <Func <DocumentEntity <Dataset>, bool> > >()) .Returns(datasets); CreateNewDatasetModelValidator validator = CreateValidator(repository); //Act ValidationResult result = validator.Validate(model); //Assert result .IsValid .Should() .BeFalse(); result .Errors .Count .Should() .Be(1); }
public void Validate_GivenvalidModelWithXlsxFile_ValidIsTrue() { //Arrange CreateNewDatasetModel model = CreateModel(); model.Filename = "filename.XLSX"; CreateNewDatasetModelValidator validator = CreateValidator(); //Act ValidationResult result = validator.Validate(model); //Assert result .IsValid .Should() .BeTrue(); }
public void Validate_GivenDefinitionNotFoundForFundingStreamId_ValidIsFalse() { //Arrange CreateNewDatasetModel model = CreateModel(); IDatasetRepository datasetRepository = CreateDatasetsRepository(false, false); CreateNewDatasetModelValidator validator = CreateValidator(datasetRepository); //Act ValidationResult result = validator.Validate(model); //Assert result .IsValid .Should() .BeFalse(); result .Errors .Count .Should() .Be(1); }
public void Validate_GivenEmptyFundingStreamId_ValidIsFalse() { //Arrange CreateNewDatasetModel model = CreateModel(); model.FundingStreamId = string.Empty; CreateNewDatasetModelValidator validator = CreateValidator(); //Act ValidationResult result = validator.Validate(model); //Assert result .IsValid .Should() .BeFalse(); result .Errors .Count .Should() .Be(1); }
public void Validate_GivenFilenameWithIncorrectExtension_ValidIsFalse() { //Arrange CreateNewDatasetModel model = CreateModel(); model.Filename = "Filename.docx"; CreateNewDatasetModelValidator validator = CreateValidator(); //Act ValidationResult result = validator.Validate(model); //Assert result .IsValid .Should() .BeFalse(); result .Errors .Count .Should() .Be(1); }
public void Validate_GivenInvalidFundingStreamId_ValidIsFalse() { //Arrange CreateNewDatasetModel model = CreateModel(); model.FundingStreamId = "test-invalid-funding-stream-id"; CreateNewDatasetModelValidator validator = CreateValidator(); //Act ValidationResult result = validator.Validate(model); //Assert result .IsValid .Should() .BeFalse(); result .Errors .Count .Should() .Be(2); }
public async Task <ValidatedApiResponse <NewDatasetVersionResponseModel> > CreateNewDataset(CreateNewDatasetModel createNewDatasetModel) { Guard.ArgumentNotNull(createNewDatasetModel, nameof(createNewDatasetModel)); return(await ValidatedPostAsync <NewDatasetVersionResponseModel, CreateNewDatasetModel>(DataSetsUriFor("create-new-dataset"), createNewDatasetModel)); }
public Task <ValidatedApiResponse <NewDatasetVersionResponseModel> > CreateDataset(CreateNewDatasetModel dataset) { Guard.ArgumentNotNull(dataset, nameof(dataset)); return(ValidatedPostAsync <NewDatasetVersionResponseModel, CreateNewDatasetModel>("create-new-dataset", dataset)); }
private async Task GenerateDataset(Specification specification, SpecGeneratorConfiguration configuration) { if (!string.IsNullOrWhiteSpace(configuration.DatasetDefinitionId) && !string.IsNullOrWhiteSpace(configuration.DatasetFilePath)) { _logger.Information("Looking up Dataset with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId); ApiResponse <DatasetDefinition> datasetDefinitionResponse = await _datasetsClient.GetDatasetDefinitionById(configuration.DatasetDefinitionId); if (datasetDefinitionResponse.StatusCode != HttpStatusCode.OK) { _logger.Error("Unable to lookup Dataset Definition with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId); return; } _logger.Information("Found Dataset Definition '{Name}'", datasetDefinitionResponse.Content.Name); string filename = Path.GetFileName(configuration.DatasetFilePath); CreateNewDatasetModel createNewDatasetModel = new CreateNewDatasetModel() { DefinitionId = datasetDefinitionResponse.Content.Id, Description = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Id})", Name = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Name})", Filename = filename, }; _logger.Information("Creating data source with excel spreadsheet located at {DatasetFilePath}", configuration.DatasetFilePath); ValidatedApiResponse <NewDatasetVersionResponseModel> datasetVersionResponse = await this._datasetsClient.CreateDataset(createNewDatasetModel); if (datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.OK && datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.Created) { _logger.Error("Unable to create data source {Content}", datasetDefinitionResponse.Content); return; } // Upload file to blob storage - headers should match the TypeScript attributes to set the metadata on the blob using (HttpClient httpClient = new HttpClient()) { using (StreamReader sr = new StreamReader(configuration.DatasetFilePath)) { using (StreamContent sc = new StreamContent(sr.BaseStream)) { sc.Headers.Add("x-ms-blob-type", "BlockBlob"); sc.Headers.Add("x-ms-meta-dataDefinitionId", datasetVersionResponse.Content.DefinitionId); sc.Headers.Add("x-ms-meta-datasetId", datasetVersionResponse.Content.DatasetId); sc.Headers.Add("x-ms-meta-authorName", "Spec Generator"); sc.Headers.Add("x-ms-meta-authorId", "specGenerator"); sc.Headers.Add("x-ms-meta-filename", filename); sc.Headers.Add("x-ms-meta-name", datasetVersionResponse.Content.Name); sc.Headers.Add("x-ms-meta-description", datasetVersionResponse.Content.Description); HttpResponseMessage fileUploadResponse = await httpClient.PutAsync(datasetVersionResponse.Content.BlobUrl, sc); if (!fileUploadResponse.IsSuccessStatusCode) { _logger.Error("Invalid response received on data source upload {ReasonPhrase}", fileUploadResponse.ReasonPhrase); return; } else { _logger.Information("Uploaded data source file to blob storage {filename}", filename); } } } } ValidateDatasetModel validateDatasetModel = new ValidateDatasetModel() { Comment = "SpecGenerator", DatasetId = datasetVersionResponse.Content.DatasetId, Description = "Spec Generator", Filename = filename, Version = 1, }; _logger.Information("Validating data source"); ValidatedApiResponse <DatasetValidationStatusModel> apiResponse = await _datasetsClient.ValidateDataset(validateDatasetModel); if (apiResponse.ModelState != null && apiResponse.ModelState.Any()) { _logger.Error("Dataset validation errors"); foreach (var error in apiResponse.ModelState) { _logger.Error("Field Key: {Key}", error.Key); foreach (var errorItem in error.Value) { _logger.Error("Message: {errorItem}", errorItem); } } return; } else if (apiResponse.Content != null) { if (!string.IsNullOrWhiteSpace(apiResponse.Content.ErrorMessage)) { _logger.Error("Data source validation failed with message: '{Message}'", apiResponse.Content.ErrorMessage); return; } } else { _logger.Error("Data source validation failed with null response"); return; } // Check status int validationChecks = 0; while (validationChecks < 1000) { ApiResponse <DatasetValidationStatusModel> validationStatusResponse = await _datasetsClient.GetDatasetValidateStatus(apiResponse.Content.OperationId); if (validationStatusResponse.StatusCode == HttpStatusCode.OK) { _logger.Information("Validation status: {CurrentOperation}", validationStatusResponse.Content.CurrentOperation); if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.Validated) { break; } else if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.FailedValidation) { _logger.Error("Validation error:", validationStatusResponse.Content.ErrorMessage); return; } } else { _logger.Warning("Validation response returned {StatusCode}, expected OK", validationStatusResponse.StatusCode); } validationChecks++; Thread.Sleep(2000); } AssignDatasetSchemaModel assignDatasetSchemaModel = new AssignDatasetSchemaModel() { DatasetDefinitionId = datasetDefinitionResponse.Content.Id, Description = $"SpecGenerator - Provider Dataset", IsSetAsProviderData = true, Name = "Provider Dataset", SpecificationId = specification.Id, UsedInDataAggregations = false, }; HttpStatusCode assignStatusCode = await _datasetsClient.AssignDatasetSchema(assignDatasetSchemaModel); _logger.Information("Creating Data Source '{Name}' for providers on specification", assignDatasetSchemaModel.Name); ApiResponse <IEnumerable <DatasetSchemasAssigned> > datasetSchemasAssignedResponse = await _datasetsClient.GetAssignedDatasetSchemasForSpecification(specification.Id); DatasetSchemasAssigned assignedSchema = datasetSchemasAssignedResponse.Content.First(); AssignDatasetVersion assignDatasetVersion = new AssignDatasetVersion() { DatasetId = datasetVersionResponse.Content.DatasetId, RelationshipId = assignedSchema.Id, Version = 1, }; _logger.Information("Assigning uploaded data source to specification"); HttpStatusCode assignStatusVersionToSpecCode = await _datasetsClient.AssignDataSourceVersionToRelationship(assignDatasetVersion); } }
public Task <IActionResult> CreateDataset([FromBody] CreateNewDatasetModel createNewDatasetModel) { Reference user = ControllerContext.HttpContext.Request.GetUserOrDefault(); return(_datasetService.CreateNewDataset(createNewDatasetModel, user)); }