public async Task <IActionResult> OnPostAsync(string relationshipId, string specificationId, string datasetVersion = null)
{
Guard.IsNullOrWhiteSpace(relationshipId, nameof(relationshipId));
ApiResponse <SelectDataSourceModel> sourcesResponse = await _datasetClient.GetDatasourcesByRelationshipId(relationshipId);
if (sourcesResponse.StatusCode != HttpStatusCode.OK || sourcesResponse.Content == null)
{
_logger.Error($"Failed to fetch data sources with status code {sourcesResponse.StatusCode.ToString()}");
return(NotFound());
}
IsAuthorizedToMap = await _authorizationHelper.DoesUserHavePermission(User, sourcesResponse.Content, SpecificationActionTypes.CanMapDatasets);
if (!IsAuthorizedToMap)
{
return(new ForbidResult());
}
if (string.IsNullOrWhiteSpace(datasetVersion))
{
this.ModelState.AddModelError($"{nameof(ViewModel)}.{nameof(datasetVersion)}", "");
SelectDataSourceViewModel viewModel = PopulateViewModel(sourcesResponse.Content);
if (viewModel == null)
{
return(new StatusCodeResult(500));
}
ViewModel = viewModel;
return(Page());
}
string[] datasetVersionArray = datasetVersion.Split("_");
if (datasetVersionArray.Length != 2)
{
_logger.Error($"Dataset version: {datasetVersion} is invalid");
return(new StatusCodeResult(500));
}
AssignDatasetVersion assignDatasetVersion = new AssignDatasetVersion
{
RelationshipId = relationshipId,
DatasetId = datasetVersionArray[0],
Version = Convert.ToInt32(datasetVersionArray[1])
};
HttpStatusCode httpStatusCode = await _datasetClient.AssignDataSourceVersionToRelationship(assignDatasetVersion);
if (httpStatusCode.IsSuccess())
{
return(Redirect($"/datasets/specificationrelationships?specificationId={specificationId}&wasSuccess=true"));
}
_logger.Error($"Failed to assign dataset version with status code: {httpStatusCode.ToString()}");
return(new StatusCodeResult(500));
}
public Task <HttpStatusCode> AssignDataSourceVersionToRelationship(AssignDatasetVersion datasetVersion)
{
Guard.ArgumentNotNull(datasetVersion, nameof(datasetVersion));
return(PostAsync("assign-datasource-to-relationship", datasetVersion));
}
private async Task GenerateDataset(Specification specification, SpecGeneratorConfiguration configuration)
{
if (!string.IsNullOrWhiteSpace(configuration.DatasetDefinitionId) && !string.IsNullOrWhiteSpace(configuration.DatasetFilePath))
{
_logger.Information("Looking up Dataset with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId);
ApiResponse <DatasetDefinition> datasetDefinitionResponse = await _datasetsClient.GetDatasetDefinitionById(configuration.DatasetDefinitionId);
if (datasetDefinitionResponse.StatusCode != HttpStatusCode.OK)
{
_logger.Error("Unable to lookup Dataset Definition with ID {DatasetDefinitionId}", configuration.DatasetDefinitionId);
return;
}
_logger.Information("Found Dataset Definition '{Name}'", datasetDefinitionResponse.Content.Name);
string filename = Path.GetFileName(configuration.DatasetFilePath);
CreateNewDatasetModel createNewDatasetModel = new CreateNewDatasetModel()
{
DefinitionId = datasetDefinitionResponse.Content.Id,
Description = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Id})",
Name = $"{configuration.SpecificationName} ({datasetDefinitionResponse.Content.Name})",
Filename = filename,
};
_logger.Information("Creating data source with excel spreadsheet located at {DatasetFilePath}", configuration.DatasetFilePath);
ValidatedApiResponse <NewDatasetVersionResponseModel> datasetVersionResponse = await this._datasetsClient.CreateDataset(createNewDatasetModel);
if (datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.OK && datasetDefinitionResponse.StatusCode != System.Net.HttpStatusCode.Created)
{
_logger.Error("Unable to create data source {Content}", datasetDefinitionResponse.Content);
return;
}
// Upload file to blob storage - headers should match the TypeScript attributes to set the metadata on the blob
using (HttpClient httpClient = new HttpClient())
{
using (StreamReader sr = new StreamReader(configuration.DatasetFilePath))
{
using (StreamContent sc = new StreamContent(sr.BaseStream))
{
sc.Headers.Add("x-ms-blob-type", "BlockBlob");
sc.Headers.Add("x-ms-meta-dataDefinitionId", datasetVersionResponse.Content.DefinitionId);
sc.Headers.Add("x-ms-meta-datasetId", datasetVersionResponse.Content.DatasetId);
sc.Headers.Add("x-ms-meta-authorName", "Spec Generator");
sc.Headers.Add("x-ms-meta-authorId", "specGenerator");
sc.Headers.Add("x-ms-meta-filename", filename);
sc.Headers.Add("x-ms-meta-name", datasetVersionResponse.Content.Name);
sc.Headers.Add("x-ms-meta-description", datasetVersionResponse.Content.Description);
HttpResponseMessage fileUploadResponse = await httpClient.PutAsync(datasetVersionResponse.Content.BlobUrl, sc);
if (!fileUploadResponse.IsSuccessStatusCode)
{
_logger.Error("Invalid response received on data source upload {ReasonPhrase}", fileUploadResponse.ReasonPhrase);
return;
}
else
{
_logger.Information("Uploaded data source file to blob storage {filename}", filename);
}
}
}
}
ValidateDatasetModel validateDatasetModel = new ValidateDatasetModel()
{
Comment = "SpecGenerator",
DatasetId = datasetVersionResponse.Content.DatasetId,
Description = "Spec Generator",
Filename = filename,
Version = 1,
};
_logger.Information("Validating data source");
ValidatedApiResponse <DatasetValidationStatusModel> apiResponse = await _datasetsClient.ValidateDataset(validateDatasetModel);
if (apiResponse.ModelState != null && apiResponse.ModelState.Any())
{
_logger.Error("Dataset validation errors");
foreach (var error in apiResponse.ModelState)
{
_logger.Error("Field Key: {Key}", error.Key);
foreach (var errorItem in error.Value)
{
_logger.Error("Message: {errorItem}", errorItem);
}
}
return;
}
else if (apiResponse.Content != null)
{
if (!string.IsNullOrWhiteSpace(apiResponse.Content.ErrorMessage))
{
_logger.Error("Data source validation failed with message: '{Message}'", apiResponse.Content.ErrorMessage);
return;
}
}
else
{
_logger.Error("Data source validation failed with null response");
return;
}
// Check status
int validationChecks = 0;
while (validationChecks < 1000)
{
ApiResponse <DatasetValidationStatusModel> validationStatusResponse = await _datasetsClient.GetDatasetValidateStatus(apiResponse.Content.OperationId);
if (validationStatusResponse.StatusCode == HttpStatusCode.OK)
{
_logger.Information("Validation status: {CurrentOperation}", validationStatusResponse.Content.CurrentOperation);
if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.Validated)
{
break;
}
else if (validationStatusResponse.Content.CurrentOperation == DatasetValidationStatus.FailedValidation)
{
_logger.Error("Validation error:", validationStatusResponse.Content.ErrorMessage);
return;
}
}
else
{
_logger.Warning("Validation response returned {StatusCode}, expected OK", validationStatusResponse.StatusCode);
}
validationChecks++;
Thread.Sleep(2000);
}
AssignDatasetSchemaModel assignDatasetSchemaModel = new AssignDatasetSchemaModel()
{
DatasetDefinitionId = datasetDefinitionResponse.Content.Id,
Description = $"SpecGenerator - Provider Dataset",
IsSetAsProviderData = true,
Name = "Provider Dataset",
SpecificationId = specification.Id,
UsedInDataAggregations = false,
};
HttpStatusCode assignStatusCode = await _datasetsClient.AssignDatasetSchema(assignDatasetSchemaModel);
_logger.Information("Creating Data Source '{Name}' for providers on specification", assignDatasetSchemaModel.Name);
ApiResponse <IEnumerable <DatasetSchemasAssigned> > datasetSchemasAssignedResponse = await _datasetsClient.GetAssignedDatasetSchemasForSpecification(specification.Id);
DatasetSchemasAssigned assignedSchema = datasetSchemasAssignedResponse.Content.First();
AssignDatasetVersion assignDatasetVersion = new AssignDatasetVersion()
{
DatasetId = datasetVersionResponse.Content.DatasetId,
RelationshipId = assignedSchema.Id,
Version = 1,
};
_logger.Information("Assigning uploaded data source to specification");
HttpStatusCode assignStatusVersionToSpecCode = await _datasetsClient.AssignDataSourceVersionToRelationship(assignDatasetVersion);
}
}
