public void Validate_GivenInvalidModelWithCsvFile_ValidIsFalse()
{
//Arrange
CreateNewDatasetModel model = CreateModel();
model.Filename = "filename.csv";
CreateNewDatasetModelValidator validator = CreateValidator();
//Act
ValidationResult result = validator.Validate(model);
//Assert
result
.IsValid
.Should()
.BeFalse();
result
.Errors
.Count
.Should().Be(1);
result
.Errors[0]
.ErrorMessage
.Should().Contain("Check you have the right file format");
}
public void Validate_GivenNameAlreadyExists_ValidIsFalse()
{
//Arrange
CreateNewDatasetModel model = CreateModel();
IEnumerable <Dataset> datasets = new[]
{
new Dataset()
};
IDatasetRepository repository = CreateDatasetsRepository(true, true);
repository
.GetDatasetsByQuery(Arg.Any <Expression <Func <DocumentEntity <Dataset>, bool> > >())
.Returns(datasets);
CreateNewDatasetModelValidator validator = CreateValidator(repository);
//Act
ValidationResult result = validator.Validate(model);
//Assert
result
.IsValid
.Should()
.BeFalse();
result
.Errors
.Count
.Should()
.Be(1);
}
public void Validate_GivenvalidModelWithXlsxFile_ValidIsTrue()
{
//Arrange
CreateNewDatasetModel model = CreateModel();
model.Filename = "filename.XLSX";
CreateNewDatasetModelValidator validator = CreateValidator();
//Act
ValidationResult result = validator.Validate(model);
//Assert
result
.IsValid
.Should()
.BeTrue();
}
public void Validate_GivenDefinitionNotFoundForFundingStreamId_ValidIsFalse()
{
//Arrange
CreateNewDatasetModel model = CreateModel();
IDatasetRepository datasetRepository = CreateDatasetsRepository(false, false);
CreateNewDatasetModelValidator validator = CreateValidator(datasetRepository);
//Act
ValidationResult result = validator.Validate(model);
//Assert
result
.IsValid
.Should()
.BeFalse();
result
.Errors
.Count
.Should()
.Be(1);
}
public void Validate_GivenEmptyFundingStreamId_ValidIsFalse()
{
//Arrange
CreateNewDatasetModel model = CreateModel();
model.FundingStreamId = string.Empty;
CreateNewDatasetModelValidator validator = CreateValidator();
//Act
ValidationResult result = validator.Validate(model);
//Assert
result
.IsValid
.Should()
.BeFalse();
result
.Errors
.Count
.Should()
.Be(1);
}
public void Validate_GivenFilenameWithIncorrectExtension_ValidIsFalse()
{
//Arrange
CreateNewDatasetModel model = CreateModel();
model.Filename = "Filename.docx";
CreateNewDatasetModelValidator validator = CreateValidator();
//Act
ValidationResult result = validator.Validate(model);
//Assert
result
.IsValid
.Should()
.BeFalse();
result
.Errors
.Count
.Should()
.Be(1);
}
public void Validate_GivenInvalidFundingStreamId_ValidIsFalse()
{
//Arrange
CreateNewDatasetModel model = CreateModel();
model.FundingStreamId = "test-invalid-funding-stream-id";
CreateNewDatasetModelValidator validator = CreateValidator();
//Act
ValidationResult result = validator.Validate(model);
//Assert
result
.IsValid
.Should()
.BeFalse();
result
.Errors
.Count
.Should()
.Be(2);
}
public async Task <ValidatedApiResponse <NewDatasetVersionResponseModel> > CreateNewDataset(CreateNewDatasetModel createNewDatasetModel)
{
Guard.ArgumentNotNull(createNewDatasetModel, nameof(createNewDatasetModel));
return(await ValidatedPostAsync <NewDatasetVersionResponseModel, CreateNewDatasetModel>(DataSetsUriFor("create-new-dataset"), createNewDatasetModel));
}
public Task <ValidatedApiResponse <NewDatasetVersionResponseModel> > CreateDataset(CreateNewDatasetModel dataset)
{
Guard.ArgumentNotNull(dataset, nameof(dataset));
return(ValidatedPostAsync <NewDatasetVersionResponseModel, CreateNewDatasetModel>("create-new-dataset", dataset));
}
private async Task GenerateDataset(Specification specification, SpecGeneratorConfiguration configuration)
{
if (!string.IsNullOrWhiteSpace(configuration.DatasetDefinitionId) && !string.IsNullOrWhiteSpace(configuration.DatasetFilePath))
{
_logger.Information("Looking up Dataset with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId);
ApiResponse <DatasetDefinition> datasetDefinitionResponse = await _datasetsClient.GetDatasetDefinitionById(configuration.DatasetDefinitionId);
if (datasetDefinitionResponse.StatusCode != HttpStatusCode.OK)
{
_logger.Error("Unable to lookup Dataset Definition with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId);
return;
}
_logger.Information("Found Dataset Definition '{Name}'", datasetDefinitionResponse.Content.Name);
string filename = Path.GetFileName(configuration.DatasetFilePath);
CreateNewDatasetModel createNewDatasetModel = new CreateNewDatasetModel()
{
DefinitionId = datasetDefinitionResponse.Content.Id,
Description = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Id})",
Name = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Name})",
Filename = filename,
};
_logger.Information("Creating data source with excel spreadsheet located at {DatasetFilePath}", configuration.DatasetFilePath);
ValidatedApiResponse <NewDatasetVersionResponseModel> datasetVersionResponse = await this._datasetsClient.CreateDataset(createNewDatasetModel);
if (datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.OK && datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.Created)
{
_logger.Error("Unable to create data source {Content}", datasetDefinitionResponse.Content);
return;
}
// Upload file to blob storage - headers should match the TypeScript attributes to set the metadata on the blob
using (HttpClient httpClient = new HttpClient())
{
using (StreamReader sr = new StreamReader(configuration.DatasetFilePath))
{
using (StreamContent sc = new StreamContent(sr.BaseStream))
{
sc.Headers.Add("x-ms-blob-type", "BlockBlob");
sc.Headers.Add("x-ms-meta-dataDefinitionId", datasetVersionResponse.Content.DefinitionId);
sc.Headers.Add("x-ms-meta-datasetId", datasetVersionResponse.Content.DatasetId);
sc.Headers.Add("x-ms-meta-authorName", "Spec Generator");
sc.Headers.Add("x-ms-meta-authorId", "specGenerator");
sc.Headers.Add("x-ms-meta-filename", filename);
sc.Headers.Add("x-ms-meta-name", datasetVersionResponse.Content.Name);
sc.Headers.Add("x-ms-meta-description", datasetVersionResponse.Content.Description);
HttpResponseMessage fileUploadResponse = await httpClient.PutAsync(datasetVersionResponse.Content.BlobUrl, sc);
if (!fileUploadResponse.IsSuccessStatusCode)
{
_logger.Error("Invalid response received on data source upload {ReasonPhrase}", fileUploadResponse.ReasonPhrase);
return;
}
else
{
_logger.Information("Uploaded data source file to blob storage {filename}", filename);
}
}
}
}
ValidateDatasetModel validateDatasetModel = new ValidateDatasetModel()
{
Comment = "SpecGenerator",
DatasetId = datasetVersionResponse.Content.DatasetId,
Description = "Spec Generator",
Filename = filename,
Version = 1,
};
_logger.Information("Validating data source");
ValidatedApiResponse <DatasetValidationStatusModel> apiResponse = await _datasetsClient.ValidateDataset(validateDatasetModel);
if (apiResponse.ModelState != null && apiResponse.ModelState.Any())
{
_logger.Error("Dataset validation errors");
foreach (var error in apiResponse.ModelState)
{
_logger.Error("Field Key: {Key}", error.Key);
foreach (var errorItem in error.Value)
{
_logger.Error("Message: {errorItem}", errorItem);
}
}
return;
}
else if (apiResponse.Content != null)
{
if (!string.IsNullOrWhiteSpace(apiResponse.Content.ErrorMessage))
{
_logger.Error("Data source validation failed with message: '{Message}'", apiResponse.Content.ErrorMessage);
return;
}
}
else
{
_logger.Error("Data source validation failed with null response");
return;
}
// Check status
int validationChecks = 0;
while (validationChecks < 1000)
{
ApiResponse <DatasetValidationStatusModel> validationStatusResponse = await _datasetsClient.GetDatasetValidateStatus(apiResponse.Content.OperationId);
if (validationStatusResponse.StatusCode == HttpStatusCode.OK)
{
_logger.Information("Validation status: {CurrentOperation}", validationStatusResponse.Content.CurrentOperation);
if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.Validated)
{
break;
}
else if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.FailedValidation)
{
_logger.Error("Validation error:", validationStatusResponse.Content.ErrorMessage);
return;
}
}
else
{
_logger.Warning("Validation response returned {StatusCode}, expected OK", validationStatusResponse.StatusCode);
}
validationChecks++;
Thread.Sleep(2000);
}
AssignDatasetSchemaModel assignDatasetSchemaModel = new AssignDatasetSchemaModel()
{
DatasetDefinitionId = datasetDefinitionResponse.Content.Id,
Description = $"SpecGenerator - Provider Dataset",
IsSetAsProviderData = true,
Name = "Provider Dataset",
SpecificationId = specification.Id,
UsedInDataAggregations = false,
};
HttpStatusCode assignStatusCode = await _datasetsClient.AssignDatasetSchema(assignDatasetSchemaModel);
_logger.Information("Creating Data Source '{Name}' for providers on specification", assignDatasetSchemaModel.Name);
ApiResponse <IEnumerable <DatasetSchemasAssigned> > datasetSchemasAssignedResponse = await _datasetsClient.GetAssignedDatasetSchemasForSpecification(specification.Id);
DatasetSchemasAssigned assignedSchema = datasetSchemasAssignedResponse.Content.First();
AssignDatasetVersion assignDatasetVersion = new AssignDatasetVersion()
{
DatasetId = datasetVersionResponse.Content.DatasetId,
RelationshipId = assignedSchema.Id,
Version = 1,
};
_logger.Information("Assigning uploaded data source to specification");
HttpStatusCode assignStatusVersionToSpecCode = await _datasetsClient.AssignDataSourceVersionToRelationship(assignDatasetVersion);
}
}
public Task <IActionResult> CreateDataset([FromBody] CreateNewDatasetModel createNewDatasetModel)
{
Reference user = ControllerContext.HttpContext.Request.GetUserOrDefault();
return(_datasetService.CreateNewDataset(createNewDatasetModel, user));
}
