public async Task ValidateDataset_GivenViewModelButResponseIsBadRequest_ReturnsStatusCode400()
{
// Arrange
ValidateDatasetModel viewModel = new ValidateDatasetModel();
ValidatedApiResponse <DatasetValidationStatusModel> response = new ValidatedApiResponse <DatasetValidationStatusModel>(HttpStatusCode.BadRequest);
response.ModelState = new Dictionary <string, IEnumerable <string> >();
IDatasetsApiClient apiClient = CreateApiClient();
apiClient
.ValidateDataset(Arg.Is(viewModel))
.Returns(response);
ILogger logger = CreateLogger();
DatasetController controller = CreateController(apiClient, logger);
// Act
IActionResult result = await controller.ValidateDataset(viewModel);
// Assert
result
.Should()
.BeOfType <InternalServerErrorResult>();
logger
.Received(1)
.Warning(Arg.Is("Failed to validate dataset with status code: {statusCode}"), Arg.Is(HttpStatusCode.BadRequest));
}
async public Task <IActionResult> ValidateDataset([FromBody] ValidateDatasetModel vm)
{
Guard.ArgumentNotNull(vm, nameof(vm));
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
ValidatedApiResponse <DatasetValidationStatusModel> apiResponse = await _datasetApiClient.ValidateDataset(vm);
if (apiResponse == null)
{
_logger.Warning("Validate Dataset API response was null");
return(new InternalServerErrorResult("Validate Dataset API response was null"));
}
if (!apiResponse.StatusCode.IsSuccess())
{
_logger.Warning("Failed to validate dataset with status code: {statusCode}", apiResponse.StatusCode);
if (apiResponse.StatusCode == HttpStatusCode.BadRequest && !apiResponse.ModelState.Values.IsNullOrEmpty())
{
return(new BadRequestObjectResult(apiResponse.ModelState));
}
return(new InternalServerErrorResult("Validate Dataset API response failed with status code: {statusCode}" + apiResponse.StatusCode));
}
DatasetValidationStatusViewModel result = _mapper.Map <DatasetValidationStatusViewModel>(apiResponse.Content);
return(new OkObjectResult(result));
}
public async Task <IActionResult> ValidateDataset([FromBody] ValidateDatasetModel vm)
{
Guard.ArgumentNotNull(vm, nameof(vm));
FundingStreamPermission permissions = await _authorizationHelper.GetUserFundingStreamPermissions(User, vm.FundingStreamId);
if (permissions?.CanUploadDataSourceFiles != true)
{
_logger.Error($"User [{User?.Identity?.Name}] has insufficient permissions to upload a dataset file for {vm.FundingStreamId}");
return(Forbid(new AuthenticationProperties()));
}
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
ValidatedApiResponse <DatasetValidationStatusModel> apiResponse = await _datasetApiClient.ValidateDataset(
new GetDatasetBlobModel
{
DatasetId = vm.DatasetId,
Version = vm.Version,
Filename = vm.Filename,
Description = vm.Description,
Comment = vm.Comment,
FundingStreamId = vm.FundingStreamId,
MergeExistingVersion = vm.MergeExistingVersion
});
if (apiResponse == null)
{
_logger.Warning("Validate Dataset API response was null");
return(new InternalServerErrorResult("Validate Dataset API response was null"));
}
if (!apiResponse.StatusCode.IsSuccess())
{
_logger.Warning("Failed to validate dataset with status code: {statusCode}", apiResponse.StatusCode);
if (apiResponse.IsBadRequest(out BadRequestObjectResult badRequest))
{
return(badRequest);
}
return(new InternalServerErrorResult(
"Validate Dataset API response failed with status code: {statusCode}" + apiResponse.StatusCode));
}
DatasetValidationStatusViewModel
result = _mapper.Map <DatasetValidationStatusViewModel>(apiResponse.Content);
return(new OkObjectResult(result));
}
public async Task ValidateDataset_GivenViewModelAndResponseIsSuccess_ReturnsStatusModel()
{
string fundingStreamId = Guid.NewGuid().ToString();
// Arrange
ValidateDatasetModel viewModel = new ValidateDatasetModel
{
FundingStreamId = fundingStreamId
};
GivenTheUserHasPermissionToUploadDataSourceFilesForFundingStream(fundingStreamId, new FundingStreamPermission
{
CanUploadDataSourceFiles = true
});
DatasetValidationStatusModel statusModel = new DatasetValidationStatusModel
{
ValidationFailures = new Dictionary <string, IEnumerable <string> >(),
CurrentOperation = DatasetValidationStatus.Processing,
DatasetId = "datasetId",
ErrorMessage = "errorMessage",
OperationId = "operationId"
};
ValidatedApiResponse <DatasetValidationStatusModel> response = new ValidatedApiResponse <DatasetValidationStatusModel>(HttpStatusCode.OK, statusModel);
DatasetValidationStatusViewModel resultViewModel = new DatasetValidationStatusViewModel
{
ValidationFailures = new Dictionary <string, IEnumerable <string> >(),
CurrentOperation = DatasetValidationStatusOperationViewModel.Processing,
DatasetId = "datasetId",
ErrorMessage = "errorMessage",
OperationId = "operationId"
};
_apiClient
.ValidateDataset(Arg.Any <GetDatasetBlobModel>())
.Returns(response);
// Act
IActionResult result = await _controller.ValidateDataset(viewModel);
// Assert
result
.Should()
.BeOfType <OkObjectResult>()
.Which
.Value
.Should()
.BeEquivalentTo(resultViewModel);
}
public void ValidateDatasett_GivenViewModelIsNull_ThowsArgumentNullException()
{
// Arrange
ValidateDatasetModel viewModel = null;
DatasetController controller = CreateController();
// Act
Func <Task> test = async() => await controller.ValidateDataset(viewModel);
// Assert
test
.Should()
.ThrowExactly <ArgumentNullException>();
}
public async Task ValidateDataset_GivenViewModelAndResponseIsSuccess_ReturnsStatusModel()
{
// Arrange
ValidateDatasetModel viewModel = new ValidateDatasetModel();
DatasetValidationStatusModel statusModel = new DatasetValidationStatusModel()
{
ValidationFailures = new Dictionary <string, IEnumerable <string> >(),
CurrentOperation = DatasetValidationStatusOperation.Processing,
DatasetId = "datasetId",
ErrorMessage = "errorMessage",
OperationId = "operationId",
};
ValidatedApiResponse <DatasetValidationStatusModel> response = new ValidatedApiResponse <DatasetValidationStatusModel>(HttpStatusCode.OK, statusModel);
DatasetValidationStatusViewModel resultViewModel = new DatasetValidationStatusViewModel()
{
ValidationFailures = new Dictionary <string, IEnumerable <string> >(),
CurrentOperation = DatasetValidationStatusOperationViewModel.Processing,
DatasetId = "datasetId",
ErrorMessage = "errorMessage",
OperationId = "operationId",
};
IDatasetsApiClient apiClient = CreateApiClient();
apiClient
.ValidateDataset(Arg.Is(viewModel))
.Returns(response);
ILogger logger = CreateLogger();
DatasetController controller = CreateController(apiClient, logger);
// Act
IActionResult result = await controller.ValidateDataset(viewModel);
// Assert
result
.Should()
.BeOfType <OkObjectResult>()
.Which
.Value
.Should()
.BeEquivalentTo(resultViewModel);
}
public async Task ValidateDataset_GivenViewModelButResponseIsBadRequestAndHasModelState_ReturnsBadRequestObjectResult()
{
string fundingStreamId = Guid.NewGuid().ToString();
// Arrange
ValidateDatasetModel viewModel = new ValidateDatasetModel
{
FundingStreamId = fundingStreamId
};
GivenTheUserHasPermissionToUploadDataSourceFilesForFundingStream(fundingStreamId, new FundingStreamPermission
{
CanUploadDataSourceFiles = true
});
IDictionary <string, IEnumerable <string> > modelState = new Dictionary <string, IEnumerable <string> >();
modelState.Add("error",
new List <string>
{
"an error occured"
});
ValidatedApiResponse <DatasetValidationStatusModel> response = new ValidatedApiResponse <DatasetValidationStatusModel>(HttpStatusCode.BadRequest)
{
ModelState = modelState
};
_apiClient
.ValidateDataset(Arg.Any <GetDatasetBlobModel>())
.Returns(response);
// Act
IActionResult result = await _controller.ValidateDataset(viewModel);
// Assert
result
.Should()
.BeOfType <BadRequestObjectResult>();
_logger
.Received(1)
.Warning(Arg.Is("Failed to validate dataset with status code: {statusCode}"), Arg.Is(HttpStatusCode.BadRequest));
}
public Task <ValidatedApiResponse <DatasetValidationStatusModel> > ValidateDataset(ValidateDatasetModel model)
{
Guard.ArgumentNotNull(model, nameof(model));
return(ValidatedPostAsync <DatasetValidationStatusModel, ValidateDatasetModel>("validate-dataset", model, timeout: TimeSpan.FromSeconds(300)));
}
private async Task GenerateDataset(Specification specification, SpecGeneratorConfiguration configuration)
{
if (!string.IsNullOrWhiteSpace(configuration.DatasetDefinitionId) && !string.IsNullOrWhiteSpace(configuration.DatasetFilePath))
{
_logger.Information("Looking up Dataset with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId);
ApiResponse <DatasetDefinition> datasetDefinitionResponse = await _datasetsClient.GetDatasetDefinitionById(configuration.DatasetDefinitionId);
if (datasetDefinitionResponse.StatusCode != HttpStatusCode.OK)
{
_logger.Error("Unable to lookup Dataset Definition with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId);
return;
}
_logger.Information("Found Dataset Definition '{Name}'", datasetDefinitionResponse.Content.Name);
string filename = Path.GetFileName(configuration.DatasetFilePath);
CreateNewDatasetModel createNewDatasetModel = new CreateNewDatasetModel()
{
DefinitionId = datasetDefinitionResponse.Content.Id,
Description = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Id})",
Name = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Name})",
Filename = filename,
};
_logger.Information("Creating data source with excel spreadsheet located at {DatasetFilePath}", configuration.DatasetFilePath);
ValidatedApiResponse <NewDatasetVersionResponseModel> datasetVersionResponse = await this._datasetsClient.CreateDataset(createNewDatasetModel);
if (datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.OK && datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.Created)
{
_logger.Error("Unable to create data source {Content}", datasetDefinitionResponse.Content);
return;
}
// Upload file to blob storage - headers should match the TypeScript attributes to set the metadata on the blob
using (HttpClient httpClient = new HttpClient())
{
using (StreamReader sr = new StreamReader(configuration.DatasetFilePath))
{
using (StreamContent sc = new StreamContent(sr.BaseStream))
{
sc.Headers.Add("x-ms-blob-type", "BlockBlob");
sc.Headers.Add("x-ms-meta-dataDefinitionId", datasetVersionResponse.Content.DefinitionId);
sc.Headers.Add("x-ms-meta-datasetId", datasetVersionResponse.Content.DatasetId);
sc.Headers.Add("x-ms-meta-authorName", "Spec Generator");
sc.Headers.Add("x-ms-meta-authorId", "specGenerator");
sc.Headers.Add("x-ms-meta-filename", filename);
sc.Headers.Add("x-ms-meta-name", datasetVersionResponse.Content.Name);
sc.Headers.Add("x-ms-meta-description", datasetVersionResponse.Content.Description);
HttpResponseMessage fileUploadResponse = await httpClient.PutAsync(datasetVersionResponse.Content.BlobUrl, sc);
if (!fileUploadResponse.IsSuccessStatusCode)
{
_logger.Error("Invalid response received on data source upload {ReasonPhrase}", fileUploadResponse.ReasonPhrase);
return;
}
else
{
_logger.Information("Uploaded data source file to blob storage {filename}", filename);
}
}
}
}
ValidateDatasetModel validateDatasetModel = new ValidateDatasetModel()
{
Comment = "SpecGenerator",
DatasetId = datasetVersionResponse.Content.DatasetId,
Description = "Spec Generator",
Filename = filename,
Version = 1,
};
_logger.Information("Validating data source");
ValidatedApiResponse <DatasetValidationStatusModel> apiResponse = await _datasetsClient.ValidateDataset(validateDatasetModel);
if (apiResponse.ModelState != null && apiResponse.ModelState.Any())
{
_logger.Error("Dataset validation errors");
foreach (var error in apiResponse.ModelState)
{
_logger.Error("Field Key: {Key}", error.Key);
foreach (var errorItem in error.Value)
{
_logger.Error("Message: {errorItem}", errorItem);
}
}
return;
}
else if (apiResponse.Content != null)
{
if (!string.IsNullOrWhiteSpace(apiResponse.Content.ErrorMessage))
{
_logger.Error("Data source validation failed with message: '{Message}'", apiResponse.Content.ErrorMessage);
return;
}
}
else
{
_logger.Error("Data source validation failed with null response");
return;
}
// Check status
int validationChecks = 0;
while (validationChecks < 1000)
{
ApiResponse <DatasetValidationStatusModel> validationStatusResponse = await _datasetsClient.GetDatasetValidateStatus(apiResponse.Content.OperationId);
if (validationStatusResponse.StatusCode == HttpStatusCode.OK)
{
_logger.Information("Validation status: {CurrentOperation}", validationStatusResponse.Content.CurrentOperation);
if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.Validated)
{
break;
}
else if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.FailedValidation)
{
_logger.Error("Validation error:", validationStatusResponse.Content.ErrorMessage);
return;
}
}
else
{
_logger.Warning("Validation response returned {StatusCode}, expected OK", validationStatusResponse.StatusCode);
}
validationChecks++;
Thread.Sleep(2000);
}
AssignDatasetSchemaModel assignDatasetSchemaModel = new AssignDatasetSchemaModel()
{
DatasetDefinitionId = datasetDefinitionResponse.Content.Id,
Description = $"SpecGenerator - Provider Dataset",
IsSetAsProviderData = true,
Name = "Provider Dataset",
SpecificationId = specification.Id,
UsedInDataAggregations = false,
};
HttpStatusCode assignStatusCode = await _datasetsClient.AssignDatasetSchema(assignDatasetSchemaModel);
_logger.Information("Creating Data Source '{Name}' for providers on specification", assignDatasetSchemaModel.Name);
ApiResponse <IEnumerable <DatasetSchemasAssigned> > datasetSchemasAssignedResponse = await _datasetsClient.GetAssignedDatasetSchemasForSpecification(specification.Id);
DatasetSchemasAssigned assignedSchema = datasetSchemasAssignedResponse.Content.First();
AssignDatasetVersion assignDatasetVersion = new AssignDatasetVersion()
{
DatasetId = datasetVersionResponse.Content.DatasetId,
RelationshipId = assignedSchema.Id,
Version = 1,
};
_logger.Information("Assigning uploaded data source to specification");
HttpStatusCode assignStatusVersionToSpecCode = await _datasetsClient.AssignDataSourceVersionToRelationship(assignDatasetVersion);
}
}
