private async Task GenerateDataset(Specification specification, SpecGeneratorConfiguration configuration)
{
if (!string.IsNullOrWhiteSpace(configuration.DatasetDefinitionId) && !string.IsNullOrWhiteSpace(configuration.DatasetFilePath))
{
_logger.Information("Looking up Dataset with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId);
ApiResponse <DatasetDefinition> datasetDefinitionResponse = await _datasetsClient.GetDatasetDefinitionById(configuration.DatasetDefinitionId);
if (datasetDefinitionResponse.StatusCode != HttpStatusCode.OK)
{
_logger.Error("Unable to lookup Dataset Definition with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId);
return;
}
_logger.Information("Found Dataset Definition '{Name}'", datasetDefinitionResponse.Content.Name);
string filename = Path.GetFileName(configuration.DatasetFilePath);
CreateNewDatasetModel createNewDatasetModel = new CreateNewDatasetModel()
{
DefinitionId = datasetDefinitionResponse.Content.Id,
Description = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Id})",
Name = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Name})",
Filename = filename,
};
_logger.Information("Creating data source with excel spreadsheet located at {DatasetFilePath}", configuration.DatasetFilePath);
ValidatedApiResponse <NewDatasetVersionResponseModel> datasetVersionResponse = await this._datasetsClient.CreateDataset(createNewDatasetModel);
if (datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.OK && datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.Created)
{
_logger.Error("Unable to create data source {Content}", datasetDefinitionResponse.Content);
return;
}
// Upload file to blob storage - headers should match the TypeScript attributes to set the metadata on the blob
using (HttpClient httpClient = new HttpClient())
{
using (StreamReader sr = new StreamReader(configuration.DatasetFilePath))
{
using (StreamContent sc = new StreamContent(sr.BaseStream))
{
sc.Headers.Add("x-ms-blob-type", "BlockBlob");
sc.Headers.Add("x-ms-meta-dataDefinitionId", datasetVersionResponse.Content.DefinitionId);
sc.Headers.Add("x-ms-meta-datasetId", datasetVersionResponse.Content.DatasetId);
sc.Headers.Add("x-ms-meta-authorName", "Spec Generator");
sc.Headers.Add("x-ms-meta-authorId", "specGenerator");
sc.Headers.Add("x-ms-meta-filename", filename);
sc.Headers.Add("x-ms-meta-name", datasetVersionResponse.Content.Name);
sc.Headers.Add("x-ms-meta-description", datasetVersionResponse.Content.Description);
HttpResponseMessage fileUploadResponse = await httpClient.PutAsync(datasetVersionResponse.Content.BlobUrl, sc);
if (!fileUploadResponse.IsSuccessStatusCode)
{
_logger.Error("Invalid response received on data source upload {ReasonPhrase}", fileUploadResponse.ReasonPhrase);
return;
}
else
{
_logger.Information("Uploaded data source file to blob storage {filename}", filename);
}
}
}
}
ValidateDatasetModel validateDatasetModel = new ValidateDatasetModel()
{
Comment = "SpecGenerator",
DatasetId = datasetVersionResponse.Content.DatasetId,
Description = "Spec Generator",
Filename = filename,
Version = 1,
};
_logger.Information("Validating data source");
ValidatedApiResponse <DatasetValidationStatusModel> apiResponse = await _datasetsClient.ValidateDataset(validateDatasetModel);
if (apiResponse.ModelState != null && apiResponse.ModelState.Any())
{
_logger.Error("Dataset validation errors");
foreach (var error in apiResponse.ModelState)
{
_logger.Error("Field Key: {Key}", error.Key);
foreach (var errorItem in error.Value)
{
_logger.Error("Message: {errorItem}", errorItem);
}
}
return;
}
else if (apiResponse.Content != null)
{
if (!string.IsNullOrWhiteSpace(apiResponse.Content.ErrorMessage))
{
_logger.Error("Data source validation failed with message: '{Message}'", apiResponse.Content.ErrorMessage);
return;
}
}
else
{
_logger.Error("Data source validation failed with null response");
return;
}
// Check status
int validationChecks = 0;
while (validationChecks < 1000)
{
ApiResponse <DatasetValidationStatusModel> validationStatusResponse = await _datasetsClient.GetDatasetValidateStatus(apiResponse.Content.OperationId);
if (validationStatusResponse.StatusCode == HttpStatusCode.OK)
{
_logger.Information("Validation status: {CurrentOperation}", validationStatusResponse.Content.CurrentOperation);
if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.Validated)
{
break;
}
else if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.FailedValidation)
{
_logger.Error("Validation error:", validationStatusResponse.Content.ErrorMessage);
return;
}
}
else
{
_logger.Warning("Validation response returned {StatusCode}, expected OK", validationStatusResponse.StatusCode);
}
validationChecks++;
Thread.Sleep(2000);
}
AssignDatasetSchemaModel assignDatasetSchemaModel = new AssignDatasetSchemaModel()
{
DatasetDefinitionId = datasetDefinitionResponse.Content.Id,
Description = $"SpecGenerator - Provider Dataset",
IsSetAsProviderData = true,
Name = "Provider Dataset",
SpecificationId = specification.Id,
UsedInDataAggregations = false,
};
HttpStatusCode assignStatusCode = await _datasetsClient.AssignDatasetSchema(assignDatasetSchemaModel);
_logger.Information("Creating Data Source '{Name}' for providers on specification", assignDatasetSchemaModel.Name);
ApiResponse <IEnumerable <DatasetSchemasAssigned> > datasetSchemasAssignedResponse = await _datasetsClient.GetAssignedDatasetSchemasForSpecification(specification.Id);
DatasetSchemasAssigned assignedSchema = datasetSchemasAssignedResponse.Content.First();
AssignDatasetVersion assignDatasetVersion = new AssignDatasetVersion()
{
DatasetId = datasetVersionResponse.Content.DatasetId,
RelationshipId = assignedSchema.Id,
Version = 1,
};
_logger.Information("Assigning uploaded data source to specification");
HttpStatusCode assignStatusVersionToSpecCode = await _datasetsClient.AssignDataSourceVersionToRelationship(assignDatasetVersion);
}
}
public Task <HttpStatusCode> AssignDatasetSchema(AssignDatasetSchemaModel datasetSchema)
{
Guard.ArgumentNotNull(datasetSchema, nameof(datasetSchema));
return(PostAsync("create-definitionspecification-relationship", datasetSchema));
}
public async Task <IActionResult> OnPostAsync(string specificationId)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
SpecificationId = specificationId;
ApiResponse <SpecificationSummary> specificationResponse = await _specsClient.GetSpecificationSummary(specificationId);
if (specificationResponse == null || specificationResponse.StatusCode == HttpStatusCode.NotFound)
{
return(new NotFoundObjectResult($"Unable to get specification response. Specification Id value = {SpecificationId}"));
}
if (specificationResponse.StatusCode == HttpStatusCode.OK && specificationResponse.Content == null)
{
throw new InvalidOperationException($"Unable to retrieve specification model from the response. Specification Id value = {SpecificationId}");
}
IsAuthorizedToEdit = await _authorizationHelper.DoesUserHavePermission(User, specificationResponse.Content, SpecificationActionTypes.CanEditSpecification);
if (!IsAuthorizedToEdit)
{
return(new ForbidResult());
}
if (!string.IsNullOrWhiteSpace(AssignDatasetSchemaViewModel.Name))
{
ApiResponse <DatasetSchemasAssigned> existingRelationshipResponse = await _datasetsClient.GetAssignedDatasetSchemasForSpecificationAndRelationshipName(specificationId, AssignDatasetSchemaViewModel.Name);
if (existingRelationshipResponse.StatusCode != HttpStatusCode.NotFound)
{
this.ModelState.AddModelError($"{nameof(AssignDatasetSchemaViewModel)}.{nameof(AssignDatasetSchemaViewModel.Name)}", ValidationMessages.RelationshipNameAlreadyExists);
}
}
if (!ModelState.IsValid)
{
if (specificationResponse.StatusCode == HttpStatusCode.OK)
{
SpecificationSummary specContent = specificationResponse.Content;
ApiResponse <IEnumerable <DatasetDefinition> > datasetResponse = await _datasetsClient.GetDataDefinitions();
if (datasetResponse == null || datasetResponse.StatusCode == HttpStatusCode.NotFound)
{
return(new NotFoundObjectResult(ErrorMessages.DatasetDefinitionNotFoundInDatasetService));
}
if (datasetResponse.StatusCode == HttpStatusCode.OK)
{
IEnumerable <DatasetDefinition> datasetDefinitionList = datasetResponse.Content;
if (datasetDefinitionList == null)
{
throw new InvalidOperationException($"Unable to retrieve Dataset definition from the response. Specification Id value = {SpecificationId}");
}
SpecificationName = specContent.Name;
SpecificationDescription = specContent.Description;
FundingPeriodId = specContent.FundingPeriod.Id;
FundingPeriodName = specContent.FundingPeriod.Name;
PopulateDatasetSchemas(datasetDefinitionList);
return(Page());
}
else
{
return(new StatusCodeResult(500));
}
}
}
AssignDatasetSchemaModel datasetSchema = _mapper.Map <AssignDatasetSchemaModel>(AssignDatasetSchemaViewModel);
datasetSchema.SpecificationId = specificationId;
HttpStatusCode newAssignDatasetResponse = await _datasetsClient.AssignDatasetSchema(datasetSchema);
if (newAssignDatasetResponse.Equals(HttpStatusCode.OK))
{
return(Redirect($"/datasets/ListDatasetSchemas/{specificationId}"));
}
else
{
return(new StatusCodeResult(500));
}
}
